You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/10/28 19:12:54 UTC
svn commit: r1711088 [2/4] - in /uima/sandbox/uima-ducc/trunk:
src/main/admin/ src/main/assembly/ src/main/resources/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/per...
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java Wed Oct 28 18:12:53 2015
@@ -18,292 +18,51 @@
*/
package org.apache.uima.ducc.database;
-import java.io.IOException;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.uima.ducc.common.DuccNode;
-import org.apache.uima.ducc.common.IIdentity;
-import org.apache.uima.ducc.common.Node;
-import org.apache.uima.ducc.common.NodeIdentity;
-import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.utils.DuccLogger;
-import org.apache.uima.ducc.common.utils.DuccProperties;
-import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.common.utils.id.IDuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
-import org.apache.uima.ducc.database.DbConstants.DbEdge;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
-import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent;
-import org.apache.uima.ducc.transport.cmdline.ICommandLine;
-import org.apache.uima.ducc.transport.event.common.IDuccPerWorkItemStatistics;
-import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
-import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
-import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccStandardInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccUimaAggregateComponent;
-import org.apache.uima.ducc.transport.event.common.IDuccUimaDeployableConfiguration;
-import org.apache.uima.ducc.transport.event.common.IRationale;
-import org.apache.uima.ducc.transport.event.common.ITimeWindow;
-import org.apache.uima.ducc.transport.event.common.JdReservationBean;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.InstanceCreator;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-import com.google.gson.TypeAdapter;
-import com.google.gson.TypeAdapterFactory;
-import com.google.gson.reflect.TypeToken;
-import com.google.gson.stream.JsonReader;
-import com.google.gson.stream.JsonToken;
-import com.google.gson.stream.JsonWriter;
-import com.orientechnologies.orient.core.record.impl.ODocument;
-import com.orientechnologies.orient.core.sql.OCommandSQL;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import com.tinkerpop.blueprints.impls.orient.OrientBaseGraph;
-import com.tinkerpop.blueprints.impls.orient.OrientEdge;
-import com.tinkerpop.blueprints.impls.orient.OrientVertex;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.SimpleStatement;
public class DbHandle
{
private DuccLogger logger = null;
DbManager manager;
- public OrientBaseGraph graphDb;
- DbHandle(DbManager manager, OrientBaseGraph graphDb)
+ DbHandle(DbManager manager)
{
- if ( DuccService.getDuccLogger() == null ) {
- // not running within a ducc service - just get a regular logger
- logger = DuccLogger.getLogger(DbHandle.class, "DB");
- } else {
- // running within a ducc service - get the component logger
- logger = DuccService.getDuccLogger(DbHandle.class.getName());
- }
+ logger = DuccLogger.getLogger(DbHandle.class, "DB");
+// if ( DuccService.getDuccLogger() == null ) {
+// // not running within a ducc service - just get a regular logger
+// logger = DuccLogger.getLogger(DbHandle.class, "DB");
+// } else {
+// // running within a ducc service - get the component logger
+// logger = DuccService.getDuccLogger(DbHandle.class.getName());
+// }
this.manager = manager;
- this.graphDb = graphDb;
- }
-
-
- public void close()
- {
- if ( graphDb == null ) return;
- graphDb.shutdown();
- graphDb = null;
- }
-
- /**
- * We use id + isHistory + vertex class = primary key
- * @param id The numeric duccid of the 'thing'
- * @param type the enum representing the database class of the object
- * @param isHistory 'true' if we're searching history, 'false' otherwise
- */
- public boolean thingInDatabase(long id, DbVertex type, DbCategory category)
- throws Exception
- {
- String methodName = "thingInDatabase";
- Iterable<Vertex> vs = null;
- if ( category == DbCategory.Any ) {
- vs = select("SELECT count(*) FROM " + type.pname() +
- " WHERE " + DbConstants.DUCCID + "=" + id);
- } else {
- vs = select("SELECT count(*) FROM " + type.pname() +
- " WHERE " + DbConstants.DUCCID + "=" + id +
- " AND " + DbConstants.DUCC_DBCAT + "='" + category.pname() + "'");
- }
-
- try {
- for ( Vertex v : vs ) {
- OrientVertex ov = (OrientVertex) v;
- ODocument doc = ov.getRecord();
- long did = doc.field("count");
- return (did > 0);
- }
- } catch (Throwable e) {
- logger.error(methodName, null, e);
- }
- return false;
- }
-
- /**
- * Find all DUCCIDs associated with the incoming type.
- *
- * @param type The type enum for the database class, e.g. Job
- * @param isHistory 'true' to search history, 'false' otherwise
- *
- * @return List of human-reaable (aka "friendly" ducc ids as strings.
- */
- public List<Long> listObjectsOfType(DbVertex type, DbCategory dbcat)
- {
- Iterable<Vertex> objs = null;
- if ( dbcat == DbCategory.Any ) {
- objs = graphDb.getVertices(type.pname(),
- new String[] {"@class"},
- new Object[]{type.pname()});
- } else {
- graphDb.getVertices(type.pname(),
- new String[] {"@class", DbConstants.DUCC_DBCAT},
- new Object[]{type.pname(), dbcat.pname()});
- }
- List<Long> ret = new ArrayList<Long>();
-
- for ( Vertex v : objs ) {
- OrientVertex ov = (OrientVertex) v;
- ret.add((Long)ov.getProperty(DbConstants.DUCCID));
- }
-
- return ret;
- }
-
- /**
- * Convert the vertex property set to a DuccPropeties, scrubbing the
- * ducc metadata.
- */
- Properties vertexToProps(OrientVertex v)
- {
- Properties ret = new DuccProperties();
- Set<String> keys = v.getPropertyKeys();
- for (String k : keys) {
- if ( k.equals(DbConstants.DUCCID) ) continue;
- if ( k.equals(DbConstants.DUCC_DBCAT) ) continue;
- ret.put(k, v.getProperty(k));
- }
- return ret;
- }
-
- /**
- * Find the objects of the given type and return their Properties, indexed by duccid.
- *
- * NOTE This returns the properties as DuccProperties so you can safely cast and use the
- * extended function of that class if you want.
- *
- * @param type The type enum for the object class (e.g. Job)
- * @param isHistory 'true' to search history, 'false' otherwise
- */
- public Map<Long, Properties> getPropertiesForType(DbVertex type, DbCategory dbcat)
- throws Exception
- {
- String methodName = "getPropertiesForType";
-
- Iterable<Vertex> vs = null;
- if ( dbcat == DbCategory.Any ) {
- vs = graphDb.getVertices(type.pname(),
- new String[] {"@class"},
- new Object[]{type.pname()});
-
- } else {
- String a = type.pname();
- String[] b = new String[] {"@class", DbConstants.DUCC_DBCAT};
- Object[] c = new Object[]{type.pname(), dbcat.pname()};
- vs = graphDb.getVertices(a, b, c);
-
-
- }
-
- Map<Long, Properties> ret = new HashMap<Long, Properties>();
-
- try {
- for ( Vertex v : vs ) {
- OrientVertex ov = (OrientVertex) v;
- Properties props = vertexToProps(ov);
- Long did = ov.getProperty(DbConstants.DUCCID);
- ret.put(did, props);
- }
- } catch (Throwable e) {
- logger.error(methodName, null, "Database access error: ", e);
- }
-
- return ret;
}
-
- public Map<Long, Properties> getPropertiesForTypeSel(DbVertex type, DbCategory dbcat)
- throws Exception
- {
- String methodName = "getPropertiesForType";
- Iterable<Vertex> vs = null;
- if ( dbcat == DbCategory.Any ) {
- vs = graphDb.getVertices(type.pname(),
- new String[] {"@class"},
- new Object[]{type.pname()});
-
- } else {
- vs = select("SELECT FROM " + type.pname() + " WHERE " + DbConstants.DUCC_DBCAT + "='" + dbcat.pname()+"'");
- }
-
- Map<Long, Properties> ret = new HashMap<Long, Properties>();
-
- try {
- for ( Vertex v : vs ) {
- OrientVertex ov = (OrientVertex) v;
- Properties props = vertexToProps(ov);
- Long did = ov.getProperty(DbConstants.DUCCID);
- ret.put(did, props);
- }
- } catch (Throwable e) {
- logger.error(methodName, null, "Database access error: ", e);
- }
-
- return ret;
- }
-
- /**
- * Use this for selecting, it returns a set of stuff
- */
- public Iterable<Vertex> select(String sql)
+ public ResultSet execute(String sql)
throws Exception
{
- String methodName = "select";
- long now = System.currentTimeMillis();
- logger.info(methodName, null, "SQL", sql);
- logger.info(methodName, null, "Time to select", System.currentTimeMillis() - now);
-
- return graphDb.command(new OCommandSQL(sql)).execute();
- }
-
- /**
- * Use this for just executing stuff that returns an int rc
- */
- public int execute(String sql)
- {
- String methodName = "execute";
+ String methodName = "execute";
long now = System.currentTimeMillis();
- logger.info(methodName, null, "SQL", sql);
+ ResultSet ret = manager.execute(sql);
logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now);
-
- return graphDb.command(new OCommandSQL(sql)).execute();
+
+ return ret;
}
- public void commit()
+ public ResultSet execute(SimpleStatement s)
{
- String methodName = "commit";
- long now = System.currentTimeMillis();
- if ( graphDb != null ) graphDb.commit();
- logger.info(methodName, null, "Time to commit", System.currentTimeMillis() - now);
+ return manager.execute(s);
}
- public void rollback()
+ public ResultSet execute(BoundStatement s)
{
- String methodName = "rollback";
- logger.warn(methodName, null, "ROLLBACK");
- if ( graphDb != null ) graphDb.
- rollback();
+ return manager.execute(s);
}
/**
@@ -339,24 +98,24 @@ public class DbHandle
* @param obj The json-ified object to save
* @param isHistory 'true' if we save to history, 'false' otherwise
*/
- public OrientVertex saveObject(DbVertex type, Long duccid, String obj, DbCategory dbcat)
- {
- //String methodName = "saveObject";
+// public OrientVertex saveObject(DbVertex type, Long duccid, String obj, DbCategory dbcat)
+// {
+// //String methodName = "saveObject";
+
+// //String typename = type.pname();
+
+// OrientVertex ret = null;
+// ODocument document = null;
+// document = new ODocument(type.pname());
+// ret = new OrientVertex(graphDb, document);
+
+// document.fromJSON(obj);
+// document.field(DbConstants.DUCCID, duccid);
+// document.field(DbConstants.DUCC_DBCAT, dbcat.pname());
+// graphDb.getRawGraph().save(document);
- //String typename = type.pname();
-
- OrientVertex ret = null;
- ODocument document = null;
- document = new ODocument(type.pname());
- ret = new OrientVertex(graphDb, document);
-
- document.fromJSON(obj);
- document.field(DbConstants.DUCCID, duccid);
- document.field(DbConstants.DUCC_DBCAT, dbcat.pname());
- graphDb.getRawGraph().save(document);
-
- return ret;
- }
+// return ret;
+// }
/**
@@ -373,27 +132,27 @@ public class DbHandle
* @TODO must research and use it as it will likely be safer and more efficient.
*
*/
- private DbObject traverseObject(OrientVertex v, boolean read_all)
- {
- //String methodName = "traverseObject";
- ODocument doc = v.getRecord();
- String doc_json = doc.toJSON();
- String stype = v.getProperty("@class");
- DbVertex type = manager.vertexType(stype);
-
- DbObject ret = new DbObject(doc_json, type);
-
- if ( read_all ) {
- Iterable<Edge> ed = v.getEdges(Direction.OUT);
- for ( Edge e : ed ) {
- OrientEdge oe = (OrientEdge) e;
- OrientVertex ov = oe.getVertex(Direction.IN);
- //logger.info(methodName, null, "Found edge connected to vertex of type", ov.getProperty("@class"), "duccid", ov.getProperty(DUCCID));
- ret.addEmbedded(traverseObject(ov, read_all));
- }
- }
- return ret;
- }
+// private DbObject traverseObject(OrientVertex v, boolean read_all)
+// {
+// //String methodName = "traverseObject";
+// ODocument doc = v.getRecord();
+// String doc_json = doc.toJSON();
+// String stype = v.getProperty("@class");
+// DbVertex type = manager.vertexType(stype);
+
+// DbObject ret = new DbObject(doc_json, type);
+
+// if ( read_all ) {
+// Iterable<Edge> ed = v.getEdges(Direction.OUT);
+// for ( Edge e : ed ) {
+// OrientEdge oe = (OrientEdge) e;
+// OrientVertex ov = oe.getVertex(Direction.IN);
+// //logger.info(methodName, null, "Found edge connected to vertex of type", ov.getProperty("@class"), "duccid", ov.getProperty(DUCCID));
+// ret.addEmbedded(traverseObject(ov, read_all));
+// }
+// }
+// return ret;
+// }
/**
* Read a database object, optionally chasing edges to get all the various bits.
@@ -402,374 +161,106 @@ public class DbHandle
* @param duccid The numeric id of the object
* @param read_all 'true' to recursively chase down the edges, 'false' otherwise.
*/
- public DbObject readObject(DbVertex type, Long duccid, boolean read_all)
- {
- String methodName = "readObject";
- Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCCID}, new Object[]{type.pname(), duccid});
- int count = 0;
- OrientVertex fv = null;
- for (Vertex v : s) {
- fv = (OrientVertex) v;
- //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
- count++;
- }
- if ( count > 1 ) {
- logger.error(methodName, null, "Expected unique object, found", count, "objects. Returning only the first object.");
- }
-
- //logger.info(methodName, null, "Traversing", fv.getProperty("@class"), "duccid", fv.getProperty(DUCCID));
- return traverseObject(fv, read_all);
- }
-
- /**
- * Link the 'from' to all the 'to''s' with edges of the right type. Everything is expected
- * to be in the db already, but maybe not yet committed, according to the caller.
- *
- * @param from the source object
- * @param to a list of the things the source is pointing/linking to
- * @param type The type enum for edge, e.g. 'process'
+// public DbObject readObject(DbVertex type, Long duccid, boolean read_all)
+// {
+// String methodName = "readObject";
+// Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCCID}, new Object[]{type.pname(), duccid});
+// int count = 0;
+// OrientVertex fv = null;
+// for (Vertex v : s) {
+// fv = (OrientVertex) v;
+// //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
+// count++;
+// }
+// if ( count > 1 ) {
+// logger.error(methodName, null, "Expected unique object, found", count, "objects. Returning only the first object.");
+// }
+
+// //logger.info(methodName, null, "Traversing", fv.getProperty("@class"), "duccid", fv.getProperty(DUCCID));
+// return traverseObject(fv, read_all);
+// }
+
+ /**
+ * This views a row as a set of properties. We update a single column in the table.
+ *
+ * @param table The table name.
+ * @param row The key to search on for update. Caller must fully-form it e.g. "name=bob", if the key
+ * field is called 'name'.
+ * @param propkey The name of the column.
+ * @param propval The value to replace the existing value.
*/
- public void addEdges(OrientVertex from, List<OrientVertex> to, DbEdge type)
+ public boolean updateProperty(String table, String row, String propkey, Object propval)
+ throws Exception
{
- //String methodName = "addEdges";
- OrientVertex fv = (OrientVertex) from;
- String etype = type.pname();
-
- for ( Object o : to ) {
- OrientVertex tv = (OrientVertex) o;
- fv.addEdge(etype, tv);
+ String methodName = "updateProperty";
+ long now = System.currentTimeMillis();
+
+ String cql = "UPDATE " + table + " SET " + propkey + "=" + propval + " WHERE " + row;
+ try {
+ manager.execute(cql);
+ return true;
+ } finally {
+ logger.info(methodName, null, "Time to update one property", propkey, System.currentTimeMillis() - now);
}
-
}
-
/**
- * Link the 'from' to a single 'to' with an e right type. Everything is expected
- * to be in the db already, but maybe not yet committed, according to the caller.
- *
- * @param from the source object
- * @param a single object to point to
- * @param type The type enum for edge, e.g. 'process'
- */
- public void addEdge(OrientVertex from, OrientVertex to, DbEdge type)
- {
- //String methodName = "addEdges";
- OrientVertex fv = (OrientVertex) from;
- OrientVertex tv = (OrientVertex) to;
- String etype = type.pname();
-
- fv.addEdge(etype, tv);
- }
-
- // public void addEdges(DbVertex typeFrom, Long fid, DbVertex typeTo, List<Long> tids)
- // {
- // String methodName = "addEdges";
-
- // Iterable<Vertex> s = graphDb.getVertices(typeFrom.pname(), new String[] {"@class", DUCCID}, new Object[]{typeFrom.pname(), fid});
- // int count = 0;
- // OrientVertex fv = null;
- // for (Vertex v : s) {
- // fv = (OrientVertex) v;
- // //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
- // count++;
- // }
-
- // for ( Long tid : tids ) {
- // Iterable<Vertex> ss = graphDb.getVertices(typeTo.pname(), new String[] {"@class", DUCCID}, new Object[]{typeTo.pname(), tid});
- // count = 0;
- // for (Vertex v : ss) {
- // OrientVertex tv = (OrientVertex) v;
- // // logger.info(methodName, null, "to vertex", count, tv.getIdentity());
- // count++;
-
- // fv.addEdge(DbEdge.Edge.pname(), tv);
- // }
- // }
-
-
- // }
-
- // public void addEdge(DbVertex typeFrom, Long fid, DbVertex typeTo, Long tid)
- // {
- // String methodName = "addEdge";
-
- // Iterable<Vertex> s = graphDb.getVertices(typeFrom.pname(), new String[] {"@class", DUCCID}, new Object[]{typeFrom.pname(), fid});
- // int count = 0;
- // OrientVertex fv = null;
- // for (Vertex v : s) {
- // fv = (OrientVertex) v;
- // //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
- // count++;
- // }
-
- // Iterable<Vertex> ss = graphDb.getVertices(typeTo.pname(), new String[] {"@class", DUCCID}, new Object[]{typeTo.pname(), tid});
- // count = 0;
- // for (Vertex v : ss) {
- // OrientVertex tv = (OrientVertex) v;
- // // logger.info(methodName, null, "to vertex", count, tv.getIdentity());
- // count++;
-
- // fv.addEdge(DbEdge.Edge.pname(), tv);
- // }
-
-
- // }
-
- public void changeCategory(OrientVertex obj, DbCategory to)
- {
- String methodName = "changeCategory";
- Long id = obj.getProperty(DbConstants.DUCCID);
- String oldcat = obj.getProperty(DbConstants.DUCC_DBCAT);
- String type = obj.getProperty("@CLASS");
- logger.info(methodName, null, "Chnage category obj of type", type, "id", id, "from", oldcat, "to", to.pname());
- obj.setProperty(DbConstants.DUCC_DBCAT, to.pname());
- }
-
- /**
- * Update a single property. The object must be unique and must already exist.
+ * This views a row as a set of properties. We update multiple columns in a single row of the table.
*
- * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
- * @param key Holds the value of the primary key
- * @param type The vertex type
- * @param dbcat The category, e.g. history, or checkpoint, or rmstate
- * @param props The properties object to be placed in the db.
+ * @param table The table name.
+ * @param row The key to search on for update. Caller must fully-form it e.g. "name=bob", if the key
+ * field is called 'name'.
+ * @param props This is a list of propertes where every even-numbered object is a column name and
+ * every odd-numbered object is its value.
+ *
+ * @throws IllegalArgumentException if the number of 'props' is not even. Lower layers may throw
+ * unchecked exceptions also.
*/
- public OrientVertex updateProperty(String keyid, Object key, DbVertex type, DbCategory dbcat, String propkey, Object propval)
+ public void updateProperties(String table, String row, Object... props)
+ throws Exception
{
- String methodName = "updateProperty";
+ String methodName = "updateProperties";
long now = System.currentTimeMillis();
- Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
- logger.info(methodName, null, "Time to search on " + type.pname() + " where category " + dbcat.pname() + " and " + keyid + " propkey " + key, System.currentTimeMillis() - now);
-
- OrientVertex ov = null;
- int count = 0; // some sanity checking, we're not allowed more than one
- for (Vertex v : s) {
- ov = (OrientVertex) v;
- count++;
- }
-
- if ( count > 1 ) {
- throw new IllegalStateException("Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey " + propkey + " propval " + propval);
+ if ( ( props.length % 2 ) != 0 ) {
+ throw new IllegalArgumentException("mkUpdate: need even number of props to form (k,v) list. Found " + props.length + " props.");
}
- if ( ov == null ) {
- throw new IllegalStateException("No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey " + propkey + " propval " + propval);
+ String cql = DbUtil.mkUpdate(table, row, props);
+ try {
+ manager.execute(cql);
+ } finally {
+ logger.info(methodName, null, "Total time to update properties", System.currentTimeMillis() - now);
}
-
-
- ov.setProperty(propkey, propval);
- ov.save();
- logger.info(methodName, null, "Time to update one property", System.currentTimeMillis() - now);
-
- return ov;
}
- public OrientVertex updateProperty(Object dbid, String propkey, Object propval)
+ PreparedStatement prepare(String cql)
{
- String methodName = "updateProperty";
- long now = System.currentTimeMillis();
-
- OrientVertex ov = graphDb.getVertex(dbid);
- logger.info(methodName, null, "Time to search on " + dbid.toString(), System.currentTimeMillis() - now);
-
- if ( ov == null ) {
- throw new IllegalStateException("No object in db: Id " + dbid.toString());
- }
-
- ov.setProperty(propkey, propval);
- ov.save();
- logger.info(methodName, null, "Time to update one property for", dbid.toString(), System.currentTimeMillis() - now);
-
- return ov;
+ String methodName = "prepare";
+ logger.info(methodName, null, "Preparing", cql);
+ return manager.prepare(cql);
}
- /**
- * Update the properties on an object in the db with the incoming properties. The object must exist.
- * There is no attempt to synchronize values - the new props either replace, or enrich the object.
- *
- * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
- * @param key Holds the value of the primary key
- * @param type The vertex type
- * @param dbcat The category, e.g. history, or checkpoint, or rmstate
- * @param props A list of (k,v) pairs which will replace their counterparts in the object.
- */
- public OrientVertex updateProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Object... props)
+ void truncate(String table)
+ throws Exception
{
- String methodName = "updateProperties";
- long now = System.currentTimeMillis();
- Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
- logger.info(methodName, null, "Time to search on " + type.pname() + " where category " + dbcat.pname() + " and " + keyid + " propkey " + key, System.currentTimeMillis() - now);
-
- OrientVertex ov = null;
- int count = 0; // some sanity checking, we're not allowed more than one
- for (Vertex v : s) {
- ov = (OrientVertex) v;
- count++;
- }
-
- if ( count > 1 ) {
- throw new IllegalStateException("updateProperties: Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + keyid + " propkey " + key);
- }
-
- if ( ov == null ) {
- throw new IllegalStateException("updateProperties: No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + keyid + " propkey " + key);
- }
- logger.info(methodName, null, "Time to find a vertex", System.currentTimeMillis() - now);
-
- long now2 = System.currentTimeMillis();
- ov.setProperties(props);
- ov.save();
- logger.info(methodName, null, "Time to modify a vertex", System.currentTimeMillis() - now2);
-
- logger.info(methodName, null, "Time to update several properties", System.currentTimeMillis() - now);
-
- return ov;
+ manager.truncate(table);
}
- public OrientVertex updateProperties(Object dbid, Object... props)
+
+ void saveObject(PreparedStatement ps, Object ... fields)
+ throws Exception
{
- String methodName = "updateProperties";
+ String methodName = "saveObject";
long now = System.currentTimeMillis();
-
-
- OrientVertex ov = graphDb.getVertex(dbid);
-
- if ( ov == null ) {
- throw new IllegalStateException("updateProperties: No object in db: id " + dbid.toString());
- }
- logger.info(methodName, null, "Time to find a vertex", dbid.toString(), System.currentTimeMillis() - now);
-
- long now2 = System.currentTimeMillis();
- ov.setProperties(props);
- ov.save();
- logger.info(methodName, null, "Time to modify vertex", dbid.toString(), System.currentTimeMillis() - now2);
- logger.info(methodName, null, "Total time to update vertex", dbid.toString(), System.currentTimeMillis() - now);
-
- return ov;
+ BoundStatement boundStatement = new BoundStatement(ps);
+ BoundStatement bound = boundStatement.bind(fields);
+ execute(bound);
+ logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now);
}
- // /**
- // * Synchronize the properties in an object with the incoming properties. THe object must be unique
- // * and already exist.
- // *
- // * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
- // * @param key Holds the value of the primary key
- // * @param type The vertex type
- // * @param dbcat The category, e.g. history, or checkpoint, or rmstate
- // * @param props The properties object to be placed in the db.
- // */
- // public OrientVertex synchronizeProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Properties props)
- // {
- // String methodName = "synchronizeProperties";
- // long now = System.currentTimeMillis();
-
- // Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
- // logger.info(methodName, null, "Time to search " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now);
-
-
- // long now2 = System.currentTimeMillis();
-
- // OrientVertex ov = null;
- // int count = 0; // some sanity checking, we're not allowed more than one
- // for (Vertex v : s) {
- // ov = (OrientVertex) v;
- // count++;
- // }
-
- // if ( count > 1 ) {
- // throw new IllegalStateException("updateProperties: Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey " + key);
- // }
-
- // if ( ov == null ) {
- // throw new IllegalStateException("updateProperties: No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey " + key);
- // }
- // logger.info(methodName, null, "Time to itarate " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now2);
- // now2 = System.currentTimeMillis();
-
- // logger.info(methodName, null, "Update db record of type", type.pname(), "category", dbcat.pname(), "key", key);
- // Set<String> keys = ov.getPropertyKeys();
- // for (String k : keys) {
- // if ( k.equals(DbConstants.DUCCID) ) continue; // must bypass schema things
- // if ( k.equals(DbConstants.DUCC_DBCAT) ) continue;
- // if ( k.equals(DbConstants.DUCC_DBNODE) ) continue;
-
- // Object val1 = ov.getProperty(k);
- // Object val2 = props.get(k);
-
- // if ( val2 == null ) { // the property is removed
- // logger.info(methodName, null, "Removed property", k);
- // ov.removeProperty(k);
- // continue;
- // }
-
- // if ( !val2.equals(val1) ) { // replace/add a property. val2 is known not null
- // logger.info(methodName, null, "Replaced/added property", k, "value", val1, "with", val2);
- // ov.setProperty(k, val2);
- // continue;
- // }
- // }
-
- // ov.setProperty(keyid, key);
- // ov.save();
- // logger.info(methodName, null, "Time to update " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now2);
-
- // logger.info(methodName, null, "Time to synchronize properties", System.currentTimeMillis() - now);
-
- // return ov;
- // }
-
- // public OrientVertex synchronizeProperties(Object dbid, Properties props)
- // {
- // String methodName = "synchronizeProperties";
- // long now = System.currentTimeMillis();
-
-
-
- // long now2 = System.currentTimeMillis();
-
- // OrientVertex ov = graphDb.getVertex(dbid);
- // if ( ov == null ) {
- // throw new IllegalStateException("updateProperties: No object in db. Id " + dbid);
- // }
- // logger.info(methodName, null, "Time to search for ", dbid, System.currentTimeMillis() - now);
-
- // now2 = System.currentTimeMillis();
- // logger.info(methodName, null, "Update db record ", dbid);
- // Set<String> keys = ov.getPropertyKeys();
- // for (String k : keys) {
- // if ( k.equals(DbConstants.DUCCID) ) continue; // must bypass schema things
- // if ( k.equals(DbConstants.DUCC_DBCAT) ) continue;
- // if ( k.equals(DbConstants.DUCC_DBNODE) ) continue;
-
- // if ( k.equals("svc_dbid") ) continue;
- // if ( k.equals("meta_dbid") ) continue;
-
- // Object val1 = ov.getProperty(k);
- // Object val2 = props.get(k);
-
- // if ( val2 == null ) { // the property is removed
- // logger.info(methodName, null, "Removed property", k);
- // ov.removeProperty(k);
- // continue;
- // }
-
- // if ( !val2.equals(val1) ) { // replace/add a property. val2 is known not null
- // logger.info(methodName, null, "Replaced/added property", k, "value", val1, "with", val2);
- // ov.setProperty(k, val2);
- // continue;
- // }
- // }
-
- // ov.save();
- // logger.info(methodName, null, "Time to update record", dbid, System.currentTimeMillis() - now2);
- // logger.info(methodName, null, "Total time to synchronize record", dbid, System.currentTimeMillis() - now);
-
- // return ov;
- // }
-
/**
* Create an object in the db from a properties object. The caller must do the the checking to insure
* the object already exists (or not, e.g. for a db loader).
@@ -779,222 +270,223 @@ public class DbHandle
* @param duccid The numeric id of the object
* @param isHistory 'True' if it is to be placed in history, 'false' otherwise.
*/
- public OrientVertex createProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Properties props)
- {
- String methodName = "createPropertiesObject";
- String typeName = type.pname();
- OrientVertex ov = null;
-
- logger.info(methodName, null, "Create new properties object of type", type.pname(), "category", dbcat.pname(), "key", key);
- ov = graphDb.addVertex("class:" + typeName, keyid, key, DbConstants.DUCC_DBCAT, dbcat.pname());
- ov.setProperties(props);
- return ov;
- }
-
- static Gson gson = null;
- static Gson mkGsonForJob()
- {
- synchronized(DbHandle.class) {
- if ( gson != null ) return gson;
+// public OrientVertex createProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Properties props)
+// {
+// String methodName = "createPropertiesObject";
+// String typeName = type.pname();
+// OrientVertex ov = null;
+
+// logger.info(methodName, null, "Create new properties object of type", type.pname(), "category", dbcat.pname(), "key", key);
+// ov = graphDb.addVertex("class:" + typeName, keyid, key, DbConstants.DUCC_DBCAT, dbcat.pname());
+// ov.setProperties(props);
+// return ov;
+// }
+
+// static Gson gson = null;
+// static Gson mkGsonForJob()
+// {
+// synchronized(DbHandle.class) {
+// if ( gson != null ) return gson;
- // We need to define Instance creators and such so we do it in a common place
- GsonBuilder gb = new GsonBuilder();
+// // We need to define Instance creators and such so we do it in a common place
+// GsonBuilder gb = new GsonBuilder();
- GenericInterfaceAdapter customAdapter = new GenericInterfaceAdapter();
- gb.serializeSpecialFloatingPointValues().setPrettyPrinting();
- gb.enableComplexMapKeySerialization();
+// GenericInterfaceAdapter customAdapter = new GenericInterfaceAdapter();
+// gb.serializeSpecialFloatingPointValues().setPrettyPrinting();
+// gb.enableComplexMapKeySerialization();
- gb.registerTypeAdapter(Node.class, new NodeInstanceCreator());
- gb.registerTypeAdapter(NodeIdentity.class, new NodeIdentityCreator());
+// gb.registerTypeAdapter(Node.class, new NodeInstanceCreator());
+// gb.registerTypeAdapter(NodeIdentity.class, new NodeIdentityCreator());
- //gb.registerTypeAdapter(IIdentity.class, new IdentityInstanceCreator());
- gb.registerTypeAdapter(IIdentity.class, customAdapter);
+// //gb.registerTypeAdapter(IIdentity.class, new IdentityInstanceCreator());
+// gb.registerTypeAdapter(IIdentity.class, customAdapter);
- gb.registerTypeAdapter(IDuccId.class, customAdapter);
- gb.registerTypeAdapter(ICommandLine.class, customAdapter);
- gb.registerTypeAdapter(ITimeWindow.class, customAdapter);
- gb.registerTypeAdapter(IDuccProcessWorkItems.class, customAdapter);
- gb.registerTypeAdapter(IDuccUimaAggregateComponent.class, customAdapter);
- gb.registerTypeAdapter(IUimaPipelineAEComponent.class, customAdapter);
- gb.registerTypeAdapter(IRationale.class, customAdapter);
- gb.registerTypeAdapter(IDuccUimaDeployableConfiguration.class, customAdapter);
- gb.registerTypeAdapter(IDuccStandardInfo.class, customAdapter);
- gb.registerTypeAdapter(IDuccSchedulingInfo.class, customAdapter);
- gb.registerTypeAdapter(IDuccPerWorkItemStatistics.class, customAdapter);
- gb.registerTypeAdapter(IDuccReservationMap.class, customAdapter);
- gb.registerTypeAdapter(JdReservationBean.class, customAdapter);
+// gb.registerTypeAdapter(IDuccId.class, customAdapter);
+// gb.registerTypeAdapter(ICommandLine.class, customAdapter);
+// gb.registerTypeAdapter(ITimeWindow.class, customAdapter);
+// gb.registerTypeAdapter(IDuccProcessWorkItems.class, customAdapter);
+// gb.registerTypeAdapter(IDuccUimaAggregateComponent.class, customAdapter);
+// gb.registerTypeAdapter(IUimaPipelineAEComponent.class, customAdapter);
+// gb.registerTypeAdapter(IRationale.class, customAdapter);
+// gb.registerTypeAdapter(IDuccUimaDeployableConfiguration.class, customAdapter);
+// gb.registerTypeAdapter(IDuccStandardInfo.class, customAdapter);
+// gb.registerTypeAdapter(IDuccSchedulingInfo.class, customAdapter);
+// gb.registerTypeAdapter(IDuccPerWorkItemStatistics.class, customAdapter);
+// gb.registerTypeAdapter(IDuccReservationMap.class, customAdapter);
+// gb.registerTypeAdapter(JdReservationBean.class, customAdapter);
- //ConcurrentHashMap<DuccId, Long> x = new ConcurrentHashMap<DuccId, Long>();
- //gb.registerTypeAdapter(x.getClass(), new MapAdaptor());
+// //ConcurrentHashMap<DuccId, Long> x = new ConcurrentHashMap<DuccId, Long>();
+// //gb.registerTypeAdapter(x.getClass(), new MapAdaptor());
- //gb.registerTypeAdapterFactory(new DuccTypeFactory());
- //Object obj = new ArrayList<IJdReservation>();
- //gb.registerTypeAdapter(obj.getClass(), customAdapter);
- Gson g = gb.create();
- return g;
- }
- }
-
- // ----------------------------------------------------------------------------------------------------
- // Instance creators and adaptors for GSON
- // ----------------------------------------------------------------------------------------------------
-
- // We need these for the DuccNode and NodeIdentity because they don't have no-arg
- // Constructors.
- //
- // @TODO after merge, consult with Jerry about adding in those constructors
- private static class NodeInstanceCreator implements InstanceCreator<Node> {
- public Node createInstance(Type type) {
- // System.out.println("DuccNode");
- return new DuccNode(null, null, false);
- }
- }
-
- private static class NodeIdentityCreator implements InstanceCreator<NodeIdentity> {
- public NodeIdentity createInstance(Type type) {
- // System.out.println("DuccNodeIdentity");
- try { return new NodeIdentity(null, null); } catch ( Exception e ) {}
- return null;
- }
- }
+// //gb.registerTypeAdapterFactory(new DuccTypeFactory());
+// //Object obj = new ArrayList<IJdReservation>();
+// //gb.registerTypeAdapter(obj.getClass(), customAdapter);
+// Gson g = gb.create();
+// return g;
+// }
+// }
+
+// // ----------------------------------------------------------------------------------------------------
+// // Instance creators and adaptors for GSON
+// // ----------------------------------------------------------------------------------------------------
+
+// // We need these for the DuccNode and NodeIdentity because they don't have no-arg
+// // Constructors.
+// //
+// // @TODO after merge, consult with Jerry about adding in those constructors
+// private static class NodeInstanceCreator implements InstanceCreator<Node> {
+// public Node createInstance(Type type) {
+// // System.out.println("DuccNode");
+// return new DuccNode(null, null, false);
+// }
+// }
+
+// private static class NodeIdentityCreator implements InstanceCreator<NodeIdentity> {
+// public NodeIdentity createInstance(Type type) {
+// // System.out.println("DuccNodeIdentity");
+// try { return new NodeIdentity(null, null); } catch ( Exception e ) {}
+// return null;
+// }
+// }
+
+// /**
+// * JSON helper for our complex objects. Gson doesn't save type information in the json so
+// * it doesn't know how to construct things declared as interfaces.
+// *
+// * This class is a Gson adapter that saves the actual object type in the json on serialization,
+// * and uses that information on deserialization to construct the right thing.
+// */
+// private static class GenericInterfaceAdapter
+// implements
+// JsonSerializer<Object>,
+// JsonDeserializer<Object>
+// {
- /**
- * JSON helper for our complex objects. Gson doesn't save type information in the json so
- * it doesn't know how to construct things declared as interfaces.
- *
- * This class is a Gson adapter that saves the actual object type in the json on serialization,
- * and uses that information on deserialization to construct the right thing.
- */
- private static class GenericInterfaceAdapter
- implements
- JsonSerializer<Object>,
- JsonDeserializer<Object>
- {
-
- private static final String DUCC_META_CLASS = "DUCC_META_CLASS";
+// private static final String DUCC_META_CLASS = "DUCC_META_CLASS";
- @Override
- public Object deserialize(JsonElement jsonElement,
- Type type,
- JsonDeserializationContext jsonDeserializationContext)
- throws JsonParseException
- {
- // Reconstitute the "right" class based on the actual class it came from as
- // found in metadata
- JsonObject obj = jsonElement.getAsJsonObject();
- JsonElement clElem= obj.get(DUCC_META_CLASS);
-
- if ( clElem== null ) {
- throw new IllegalStateException("Cannot determine concrete class for " + type + ". Must register explicit type adapter for it.");
- }
- String clName = clElem.getAsString();
-
- //System.out.println("----- elem: " + clName + " clElem: " + obj);
- try {
- Class<?> clz = Class.forName(clName);
- return jsonDeserializationContext.deserialize(jsonElement, clz);
- } catch (ClassNotFoundException e) {
- throw new JsonParseException(e);
- }
- }
+// @Override
+// public Object deserialize(JsonElement jsonElement,
+// Type type,
+// JsonDeserializationContext jsonDeserializationContext)
+// throws JsonParseException
+// {
+// // Reconstitute the "right" class based on the actual class it came from as
+// // found in metadata
+// JsonObject obj = jsonElement.getAsJsonObject();
+// JsonElement clElem= obj.get(DUCC_META_CLASS);
+
+// if ( clElem== null ) {
+// throw new IllegalStateException("Cannot determine concrete class for " + type + ". Must register explicit type adapter for it.");
+// }
+// String clName = clElem.getAsString();
+
+// //System.out.println("----- elem: " + clName + " clElem: " + obj);
+// try {
+// Class<?> clz = Class.forName(clName);
+// return jsonDeserializationContext.deserialize(jsonElement, clz);
+// } catch (ClassNotFoundException e) {
+// throw new JsonParseException(e);
+// }
+// }
- @Override
- public JsonElement serialize(Object object,
- Type type,
- JsonSerializationContext jsonSerializationContext)
- {
- // Add the mete element indicating what kind of concrete class is this came from
- //String n = object.getClass().getCanonicalName();
- //System.out.println("**** Serialize object A " + n + " of type " + type);
- //if ( n.contains("Concurrent") ) {
- // int stop = 1;
- // stop++;
- //}
-
- JsonElement ele = jsonSerializationContext.serialize(object, object.getClass());
- //System.out.println("**** Serialize object B " + object.getClass().getCanonicalName() + " of type " + type + " : ele " + ele);
- ele.getAsJsonObject().addProperty(DUCC_META_CLASS, object.getClass().getCanonicalName());
- return ele;
- }
- }
-
- @SuppressWarnings("unused")
- private class DuccTypeFactory
- implements TypeAdapterFactory
- {
+// @Override
+// public JsonElement serialize(Object object,
+// Type type,
+// JsonSerializationContext jsonSerializationContext)
+// {
+// // Add the mete element indicating what kind of concrete class is this came from
+// //String n = object.getClass().getCanonicalName();
+// //System.out.println("**** Serialize object A " + n + " of type " + type);
+// //if ( n.contains("Concurrent") ) {
+// // int stop = 1;
+// // stop++;
+// //}
+
+// JsonElement ele = jsonSerializationContext.serialize(object, object.getClass());
+// //System.out.println("**** Serialize object B " + object.getClass().getCanonicalName() + " of type " + type + " : ele " + ele);
+// ele.getAsJsonObject().addProperty(DUCC_META_CLASS, object.getClass().getCanonicalName());
+// return ele;
+// }
+// }
+
+// @SuppressWarnings("unused")
+// private class DuccTypeFactory
+// implements TypeAdapterFactory
+// {
+
+// public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken)
+// {
+// //System.out.println("TYPETOKEN: " + typeToken + " raw type: " + typeToken.getRawType().getName());
+// Class<?> cl = typeToken.getRawType();
+// //System.out.println(" Canonical name: " + cl.getCanonicalName());
+// Type type = typeToken.getType();
+// if ( typeToken.getRawType() != ConcurrentHashMap.class ) {
+// //System.out.println("Skipping type " + typeToken);
+// return null;
+// }
- public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken)
- {
- //System.out.println("TYPETOKEN: " + typeToken + " raw type: " + typeToken.getRawType().getName());
- Class<?> cl = typeToken.getRawType();
- //System.out.println(" Canonical name: " + cl.getCanonicalName());
- Type type = typeToken.getType();
- if ( typeToken.getRawType() != ConcurrentHashMap.class ) {
- //System.out.println("Skipping type " + typeToken);
- return null;
- }
-
- if ( type instanceof ParameterizedType ) {
+// if ( type instanceof ParameterizedType ) {
- ParameterizedType pt = (ParameterizedType) type;
- Type[] types = pt.getActualTypeArguments();
- //for ( Type tt : types ) {
- // System.out.println(" TYPE ARGUMENTS: " + tt);
- //}
- Type tt = types[0];
- Class<?> cll = (Class<?>) tt;
+// ParameterizedType pt = (ParameterizedType) type;
+// Type[] types = pt.getActualTypeArguments();
+// //for ( Type tt : types ) {
+// // System.out.println(" TYPE ARGUMENTS: " + tt);
+// //}
+// Type tt = types[0];
+// Class<?> cll = (Class<?>) tt;
- }
- return null;
- }
-
- }
-
- @SuppressWarnings("unused")
- private class MapAdaptor
- extends TypeAdapter<ConcurrentHashMap<DuccId, Long>>
- {
+// }
+// return null;
+// }
+
+// }
+
+// @SuppressWarnings("unused")
+// private class MapAdaptor
+// extends TypeAdapter<ConcurrentHashMap<DuccId, Long>>
+// {
- public void write(JsonWriter out, ConcurrentHashMap<DuccId, Long> map) throws IOException {
- System.out.println("***************** Writing");
- if (map == null) {
- out.nullValue();
- return;
- }
-
- out.beginArray();
- for (DuccId k : map.keySet() ) {
- out.beginObject();
- out.value(k.getFriendly());
- out.value(k.getUnique());
- out.value(map.get(k));
- out.endObject();
- }
- out.endArray();
- }
+// public void write(JsonWriter out, ConcurrentHashMap<DuccId, Long> map) throws IOException {
+// System.out.println("***************** Writing");
+// if (map == null) {
+// out.nullValue();
+// return;
+// }
+
+// out.beginArray();
+// for (DuccId k : map.keySet() ) {
+// out.beginObject();
+// out.value(k.getFriendly());
+// out.value(k.getUnique());
+// out.value(map.get(k));
+// out.endObject();
+// }
+// out.endArray();
+// }
- public ConcurrentHashMap<DuccId, Long> read(JsonReader in) throws IOException {
- System.out.println("***************** reading");
- if (in.peek() == JsonToken.NULL) {
- in.nextNull();
- return null;
- }
+// public ConcurrentHashMap<DuccId, Long> read(JsonReader in) throws IOException {
+// System.out.println("***************** reading");
+// if (in.peek() == JsonToken.NULL) {
+// in.nextNull();
+// return null;
+// }
- ConcurrentHashMap<DuccId, Long> ret = new ConcurrentHashMap<DuccId, Long>();
- in.beginArray();
- while (in.hasNext()) {
- in.beginObject();
- Long friendly = in.nextLong();
- String unique = in.nextString();
-
- Long val = in.nextLong();
- in.endObject();
- DuccId id = new DuccId(friendly);
- id.setUUID(UUID.fromString(unique));
- ret.put(id, val);
- }
- in.endArray();
- return ret;
- }
- }
+// ConcurrentHashMap<DuccId, Long> ret = new ConcurrentHashMap<DuccId, Long>();
+// in.beginArray();
+// while (in.hasNext()) {
+// in.beginObject();
+// Long friendly = in.nextLong();
+// String unique = in.nextString();
+
+// Long val = in.nextLong();
+// in.endObject();
+// DuccId id = new DuccId(friendly);
+// id.setUUID(UUID.fromString(unique));
+// ret.put(id, val);
+// }
+// in.endArray();
+// return ret;
+// }
+// }
}
+
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java Wed Oct 28 18:12:53 2015
@@ -23,9 +23,11 @@ import java.io.FileInputStream;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
@@ -33,15 +35,17 @@ import java.util.concurrent.atomic.Atomi
import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.persistence.services.IStateServices;
+import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcMetaProps;
+import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcRegProps;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
-import com.orientechnologies.orient.core.config.OGlobalConfiguration;
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
/**
* Toy orientdb loader to load a historydb from ducc history
@@ -51,7 +55,10 @@ public class DbLoader
{
DuccLogger logger = DuccLogger.getLogger(DbLoader.class, "DBLOAD");
String DUCC_HOME;
+ String SVC_HISTORY_KEY = SvcRegProps.is_archived.columnName();
+ String META_HISTORY_KEY = SvcMetaProps.is_archived.columnName();
+ DbManager dbManager = null;
boolean archive = true; // for debug and test, bypass archive-rename
HistoryManagerDb hmd = null;
StateServicesDb ssd = null;
@@ -76,32 +83,37 @@ public class DbLoader
String serviceRegistry = "/state/services";
String checkpointFile = "/state/orchestrator.ckpt";
- String archive_key = IStateServices.archive_key;
- String archive_flag = IStateServices.archive_flag;
int nthreads = 20;
AtomicInteger counter = new AtomicInteger(0);
- //int joblimit = 10000;
- //int reservationlimit = 10000;
- //int servicelimit = 10000;
- //int registrylimit = 10000;
+ //int joblimit = 10;
+ //int reservationlimit = 10;
+ //int servicelimit = 10;
+ //int registrylimit = 1;
int joblimit = Integer.MAX_VALUE;
int reservationlimit = Integer.MAX_VALUE;
int servicelimit = Integer.MAX_VALUE;
int registrylimit = Integer.MAX_VALUE;
- boolean dojobs = true;
- boolean doreservations = true;
- boolean doservices = true;
+ boolean dojobs = false;
+ boolean doreservations = false;
+ boolean doservices = false;
boolean doregistry = true;
- boolean docheckpoint = true;
+ boolean docheckpoint = false;
+
+ long jobBytes = 0;
+ long resBytes = 0;
+ long svcBytes = 0;
+ long svcRegBytes= 0;
+
+ AtomicInteger skippedServices = new AtomicInteger(0);
public DbLoader(String from, String to)
throws Exception
{
- String methodName = "<ctr>";
+ //String methodName = "<ctr>";
DUCC_HOME = System.getProperty("DUCC_HOME");
if ( DUCC_HOME == null ) {
System.out.println("System proprety -DDUCC_HOME must be set.");
@@ -130,10 +142,10 @@ public class DbLoader
}
String databasedir = to + "/database/databases";
- String databasename = databasedir + "/DuccState";
+
// We always use a non-networked version for loading
//state_url = "plocal:" + databasedir + "/DuccState";
- state_url = "remote:bluej538/DuccState";
+ state_url = "bluej538";
System.setProperty("ducc.state.database.url", state_url);
if ( state_url.startsWith("plocal") ) {
@@ -160,6 +172,7 @@ public class DbLoader
}
public void loadJobs()
+ throws Exception
{
String methodName = "loadJobs";
@@ -234,6 +247,7 @@ public class DbLoader
}
public void loadReservations()
+ throws Exception
{
String methodName = "loadReservations";
@@ -309,6 +323,7 @@ public class DbLoader
public void loadServices()
+ throws Exception
{
String methodName = "loadServices";
@@ -390,7 +405,13 @@ public class DbLoader
int c = 0;
File dir = new File(registry);
+ if ( ! dir.isDirectory() ) {
+ logger.error(methodName, null, registry, "is not a directory and cannot be loaded.");
+ return;
+ }
+
File[] files = dir.listFiles();
+
if ( files.length == 0 ) {
if ( isHistory ) {
logger.info(methodName, null, "Nothing in service registry history to move to database");
@@ -504,84 +525,120 @@ public class DbLoader
}
}
+ void test()
+ throws Exception
+ {
+ String methodName = "foo";
+ DbHandle h = dbManager.open();
+ SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDb.JOB_TABLE + " limit 5000");
+ //SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDb.RES_TABLE + " limit 5000");
+ //SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDbB.SVC_TABLE + " limit 5000");
+ logger.info(methodName, null, "Fetch size", s.getFetchSize());
+ s.setFetchSize(100);
+ long now = System.currentTimeMillis();
+
+ try {
+ int counter = 0;
+ int nbytes = 0;
+ ResultSet rs = h.execute(s);
+ for ( Row r : rs ) {
+ counter++;
+ ByteBuffer b = r.getBytes("work");
+ nbytes += b.array().length;
+ logger.info(methodName, null, "found", r.getLong("ducc_dbid"), "of type", r.getString("type"));
+ }
+
+ logger.info(methodName, null, "Found", counter, "results. Total bytes", nbytes);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ logger.info(methodName, null, "Time to select:", System.currentTimeMillis() - now);
+ }
+
+
void run()
throws Exception
{
String methodName = "run";
long now = System.currentTimeMillis();
- DbManager dbm = new DbManager(state_url, logger);
- if ( dbm.checkForDatabase() ) {
- dbm.init();
- dbm.drop();
- dbm.shutdown();
- }
+ if ( false ) {
+ try {
+ dbManager = new DbManager(state_url, logger);
+ dbManager.init();
+
+ test();
+ if ( true ) return;
+ } finally {
+ dbManager.shutdown();
+ }
+ return;
- DbCreate cr = new DbCreate(state_url, logger);
- if ( state_url.startsWith("plocal") ) {
- cr.createPlocalDatabase();
- } else {
- cr.createDatabase();
}
- logger.info(methodName, null, "storage.useWAL", System.getProperty("storage.useWAL"));
- logger.info(methodName, null, "tx.useLog", System.getProperty("tx.useLog"));
- if ( true ) {
- try {
+ dbManager = new DbManager(state_url, logger);
+ dbManager.init();
- OGlobalConfiguration.USE_WAL.setValue(false);
- // OGlobalConfiguration.USE_LOG.setValue(false);
+// DbCreate cr = new DbCreate(state_url, logger);
+// if ( state_url.startsWith("plocal") ) {
+// cr.createPlocalDatabase();
+// } else {
+// cr.createDatabase();
+// }
+
- OGlobalConfiguration.dumpConfiguration(System.out);
- hmd = new HistoryManagerDb(logger);
+ if ( true ) {
+ try {
+ hmd = new HistoryManagerDb();
+ hmd.init(logger, dbManager);
+
long nowt = System.currentTimeMillis();
if ( docheckpoint ) loadCheckpoint();
logger.info(methodName, null, "***** Time to load checkpoint A ****", System.currentTimeMillis() - nowt);
- OGlobalConfiguration.dumpConfiguration(System.out);
-
-
// ---------- Load job history
nowt = System.currentTimeMillis();
if ( dojobs ) loadJobs();
- logger.info(methodName, null, "**** Time to load jobs**** ", System.currentTimeMillis() - nowt);
+ logger.info(methodName, null, "**** Time to load jobs**** ", System.currentTimeMillis() - nowt, "Total bytes loaded:", jobBytes);
// ---------- Load reservation history
nowt = System.currentTimeMillis();
if ( doreservations ) loadReservations();
- logger.info(methodName, null, "**** Time to load reservations ****", System.currentTimeMillis() - nowt);
+ logger.info(methodName, null, "**** Time to load reservations ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", resBytes);
// ---------- Load service isntance and AP history
nowt = System.currentTimeMillis();
if ( doservices ) loadServices();
- logger.info(methodName, null, "**** Time to load service instances ****", System.currentTimeMillis() - nowt);
+ logger.info(methodName, null, "**** Time to load services instances ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcBytes);
+
// ---------- Load service registry
+ long totalSvcBytes = 0;
if ( doregistry ) {
nowt = System.currentTimeMillis();
ssd = new StateServicesDb();
- ssd.init(logger);
+ ssd.init(logger,dbManager);
loadServiceRegistry(serviceRegistry, false);
- try {
- ssd.shutdown();
- } catch ( Exception e ) {
- e.printStackTrace();
- }
- logger.info(methodName, null, "**** Time to load Service registry ****", System.currentTimeMillis() - nowt);
+ logger.info(methodName, null, "**** Time to load Service registry ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcRegBytes);
+ totalSvcBytes = svcRegBytes;
// ---------- Load service registry history
+ svcRegBytes = 0;
nowt = System.currentTimeMillis();
- ssd = new StateServicesDb();
- ssd.init(logger);
loadServiceRegistry(serviceRegistryHistory, true);
- logger.info(methodName, null, "**** Time to load Service history ****", System.currentTimeMillis() - nowt);
+ logger.info(methodName, null, "**** Time to load Service history ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcRegBytes);
+ totalSvcBytes = svcRegBytes;
+
+ logger.info(methodName, null, "**** Skipped services:", skippedServices);
+ // don't shutdown the ssm. we'll close the db in the 'finally' below
}
nowt = System.currentTimeMillis();
- logger.info(methodName, null, "**** Total load time ****", System.currentTimeMillis() - now);
+ logger.info(methodName, null, "**** Total load time ****", System.currentTimeMillis() - now, "Total bytes loaded:", (jobBytes + resBytes + svcBytes + totalSvcBytes));
if ( docheckpoint ) loadCheckpoint();
logger.info(methodName, null, "**** Time to reload checkpoint B ****", System.currentTimeMillis() - nowt);
@@ -590,8 +647,7 @@ public class DbLoader
logger.error(methodName, null, e);
} finally {
- if ( hmd != null ) hmd.shutdown();
- if ( ssd != null ) ssd.shutdown();
+ if ( dbManager!= null ) dbManager.shutdown();
}
}
@@ -627,12 +683,17 @@ public class DbLoader
class JobLoader
implements Runnable
{
+ PreparedStatement statement = null;
BlockingQueue<File> queue;
List<Long> ids;
JobLoader(BlockingQueue<File> queue, List<Long> ids)
+ throws Exception
{
this.queue = queue;
this.ids = ids;
+
+ DbHandle h = dbManager.open();
+ statement = h.prepare("INSERT INTO " + HistoryManagerDb.JOB_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");
}
public void run()
@@ -640,22 +701,44 @@ public class DbLoader
String methodName = "JobLoader.run";
while ( true ) {
File f = null;
- IDuccWorkJob job = null;
+ long nbytes = 0;
+ long duccid = 0;
+ DuccId did = null;
try {
f = queue.take();
-
FileInputStream fis = null;
ObjectInputStream in = null;
try {
long now = System.currentTimeMillis();
+
+ String s = f.getName();
+ int ndx = s.indexOf(".");
+ duccid = Long.parseLong(s.substring(0, ndx));
+ did = new DuccId(duccid);
+ nbytes = f.length();
+ if ( nbytes > 16*1024*1024) {
+ logger.warn(methodName, did, "Skipping outsized job", duccid, "length=", nbytes);
+ nbytes = 0;
+ continue;
+ }
+
+ byte[] buf = new byte[(int)nbytes];
fis = new FileInputStream(f);
- in = new ObjectInputStream(fis);
- job = (IDuccWorkJob) in.readObject();
- logger.info(methodName, job.getDuccId(), "Time to read job:", System.currentTimeMillis() - now);
- hmd.saveJobUnsafe(job);
+ fis.read(buf);
+
+ ByteBuffer bb = ByteBuffer.wrap(buf);
+ logger.info(methodName, did, "Time to read job:", System.currentTimeMillis() - now+" MS", "bytes:", nbytes);
+ logger.info(methodName, did, "Job", duccid, "Store CQL:", statement.getQueryString());
+ long now1 = System.currentTimeMillis();
+ BoundStatement boundStatement = new BoundStatement(statement);
+ BoundStatement bound = boundStatement.bind(duccid, "job", true, bb);
+ DbHandle h = dbManager.open();
+
+ h.execute(bound);
+ logger.info(methodName, did, "Time to store job", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
} catch(Exception e) {
- logger.info(methodName, null, e);
+ logger.info(methodName, did, e);
} finally {
closeStream(in);
closeStream(fis);
@@ -666,11 +749,13 @@ public class DbLoader
return;
} catch(Exception e) {
logger.info(methodName, null, e);
- }
+ } finally {
- synchronized(ids) {
- if ( job != null ) {
- ids.add(job.getDuccId().getFriendly());
+ synchronized(ids) {
+ if ( nbytes > 0 ) {
+ ids.add(duccid);
+ jobBytes += nbytes;
+ }
}
}
}
@@ -681,20 +766,26 @@ public class DbLoader
class ServiceLoader
implements Runnable
{
+ PreparedStatement statement = null;
BlockingQueue<File> queue;
List<Long> ids;
ServiceLoader(BlockingQueue<File> queue, List<Long> ids)
+ throws Exception
{
this.queue = queue;
this.ids = ids;
+ DbHandle h = dbManager.open();
+ statement = h.prepare("INSERT INTO " + HistoryManagerDb.SVC_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");
}
public void run()
{
String methodName = "ServiceLoader.run";
while ( true ) {
- IDuccWorkService svc = null;
File f = null;
+ long nbytes = 0;
+ long duccid = 0;
+ DuccId did = null;
try {
f = queue.take();
FileInputStream fis = null;
@@ -702,30 +793,50 @@ public class DbLoader
try {
long now = System.currentTimeMillis();
+
+ String s = f.getName();
+ int ndx = s.indexOf(".");
+ duccid = Long.parseLong(s.substring(0, ndx));
+ did = new DuccId(duccid);
+ nbytes = f.length();
+ if ( nbytes > 16*1024*1024) {
+ logger.warn(methodName, did, "Skipping outsized service", duccid, "length=", nbytes);
+ nbytes = 0;
+ continue;
+ }
+
+ byte[] buf = new byte[(int)nbytes];
fis = new FileInputStream(f);
- in = new ObjectInputStream(fis);
- svc = (IDuccWorkService) in.readObject();
- logger.info(methodName, svc.getDuccId(), "Time to read service:", System.currentTimeMillis() - now);
- hmd.saveServiceUnsafe(svc);
+ fis.read(buf);
+
+ ByteBuffer bb = ByteBuffer.wrap(buf);
+ logger.info(methodName, did, "Time to read service", duccid, ":", System.currentTimeMillis() - now + " MS", "bytes:", nbytes);
+ logger.info(methodName, did, "Service", duccid, "Store CQL:", statement.getQueryString());
+ long now1 = System.currentTimeMillis();
+ BoundStatement boundStatement = new BoundStatement(statement);
+ BoundStatement bound = boundStatement.bind(duccid, "service", true, bb);
+ DbHandle h = dbManager.open();
+
+ h.execute(bound);
+ logger.info(methodName, did, "Time to store service", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
} catch(Exception e) {
- logger.info(methodName, null, "Error reading or saving service:", f);
- logger.info(methodName, null, e);
+ logger.info(methodName, did, e);
} finally {
closeStream(in);
closeStream(fis);
- counter.getAndDecrement();
+ counter.getAndDecrement();
}
-
} catch ( InterruptedException e ) {
return;
- } catch ( Exception e ){
- logger.info(methodName, null, "Error reading or saving service:", f);
- logger.info(methodName, null, e);
- }
+ } catch(Exception e) {
+ logger.info(methodName, did, e);
+ } finally {
- synchronized(ids) {
- if ( svc != null ) {
- ids.add(svc.getDuccId().getFriendly());
+ synchronized(ids) {
+ if ( nbytes > 0 ) {
+ ids.add(duccid);
+ svcBytes += nbytes;
+ }
}
}
}
@@ -735,20 +846,26 @@ public class DbLoader
class ReservationLoader
implements Runnable
{
+ PreparedStatement statement = null;
BlockingQueue<File> queue;
List<Long> ids;
ReservationLoader(BlockingQueue<File> queue, List<Long> ids)
+ throws Exception
{
this.queue = queue;
- this.ids = ids;
+ this.ids = ids;
+ DbHandle h = dbManager.open();
+ statement = h.prepare("INSERT INTO " + HistoryManagerDb.RES_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");
}
public void run()
{
String methodName = "ReservationLoader.run";
while ( true ) {
- IDuccWorkReservation res = null;
File f = null;
+ long nbytes = 0;
+ long duccid = 0;
+ DuccId did = null;
try {
f = queue.take();
FileInputStream fis = null;
@@ -756,13 +873,35 @@ public class DbLoader
try {
long now = System.currentTimeMillis();
+
+ String s = f.getName();
+ int ndx = s.indexOf(".");
+ duccid = Long.parseLong(s.substring(0, ndx));
+ did = new DuccId(duccid);
+ nbytes = f.length();
+ if ( nbytes > 16*1024*1024) {
+ logger.warn(methodName, did, "Skipping outsized reservation", duccid, "length=", nbytes);
+ nbytes = 0;
+ continue;
+ }
+
+ byte[] buf = new byte[(int)nbytes];
fis = new FileInputStream(f);
- in = new ObjectInputStream(fis);
- res = (IDuccWorkReservation) in.readObject();
- logger.info(methodName, res.getDuccId(), "Time to read reservation:", System.currentTimeMillis() - now);
- hmd.saveReservationUnsafe(res);
+ fis.read(buf);
+
+ ByteBuffer bb = ByteBuffer.wrap(buf);
+ logger.info(methodName, did, "Time to read reservation", duccid, ":", System.currentTimeMillis() - now+" MS", "bytes:", nbytes);
+ logger.info(methodName, did, "Reservation", duccid, "Store CQL:", statement.getQueryString());
+ long now1 = System.currentTimeMillis();
+ BoundStatement boundStatement = new BoundStatement(statement);
+ BoundStatement bound = boundStatement.bind(duccid, "reservation", true, bb);
+ DbHandle h = dbManager.open();
+
+ h.execute(bound);
+ logger.info(methodName, did, "Time to store reservation", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
+
} catch(Exception e) {
- logger.info(methodName, null, e);
+ logger.info(methodName, did, e);
} finally {
closeStream(in);
closeStream(fis);
@@ -774,11 +913,12 @@ public class DbLoader
return;
} catch ( Exception e ){
logger.info(methodName, null, e);
- }
-
- synchronized(ids) {
- if ( res != null ) {
- ids.add(res.getDuccId().getFriendly());
+ } finally {
+ synchronized(ids) {
+ if ( nbytes > 0 ) {
+ ids.add(duccid);
+ resBytes += nbytes;
+ }
}
}
}
@@ -803,6 +943,7 @@ public class DbLoader
while ( true ) {
Pair<String, Boolean> p = null;
String id = null;
+ int nbytes = 0;
boolean isHistory;
try {
logger.info(methodName, null, "About to take (service id).");
@@ -822,40 +963,41 @@ public class DbLoader
String svc_name = id + ".svc";
String meta_name = id + ".meta";
- svc_in = new FileInputStream(svc_name);
- meta_in = new FileInputStream(meta_name);
+ File svc_file = new File(svc_name);
+ File meta_file = new File(meta_name);
+ nbytes += (svc_file.length() + meta_file.length());
+ svc_in = new FileInputStream(svc_file);
+ meta_in = new FileInputStream(meta_file);
svc_props.load(svc_in);
meta_props.load(meta_in);
- if ( isHistory ) {
- meta_props.setProperty(archive_key, archive_flag);
- }
- String sid = meta_props.getProperty(IStateServices.SvcProps.numeric_id.pname());
+ String sid = meta_props.getProperty(IStateServices.SvcMetaProps.numeric_id.pname());
if ( sid == null ) {
- logger.warn(methodName, null, "Cannot find service id in meta file for", id);
+ logger.error(methodName, null, "Cannot find service id in meta file for", id, "skipping load.");
+ skippedServices.getAndIncrement();
} else {
if ( id.indexOf(sid) < 0 ) {
// must do index of because the 'id' is a full path, not just the numeric id. so we
// are satisfied with making sure the id is in the path.
throw new IllegalStateException("Service id and internal id do not match.");
}
- DuccId did = new DuccId(Long.parseLong(sid));
-
+ DuccId did = new DuccId(Long.parseLong(sid));
+ ssd.storeProperties(did, svc_props, meta_props); // always stores as not history
if ( isHistory ) {
- ssd.storePropertiesUnsafe(did, svc_props, meta_props, DbCategory.History);
- } else {
- ssd.storePropertiesUnsafe(did, svc_props, meta_props, DbCategory.SmReg);
+ ssd.moveToHistory(did, svc_props, meta_props); // updates a single column in each
}
+
synchronized(ids) {
ids.add(did.getFriendly());
+ svcRegBytes += nbytes;
}
}
- counter.getAndDecrement();
} catch(Exception e) {
logger.info(methodName, null, e);
} finally {
closeStream(svc_in);
closeStream(meta_in);
+ counter.getAndDecrement();
}
}