You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/10/28 19:12:54 UTC

svn commit: r1711088 [2/4] - in /uima/sandbox/uima-ducc/trunk: src/main/admin/ src/main/assembly/ src/main/resources/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/per...

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbHandle.java Wed Oct 28 18:12:53 2015
@@ -18,292 +18,51 @@
 */
 package org.apache.uima.ducc.database;
 
-import java.io.IOException;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.uima.ducc.common.DuccNode;
-import org.apache.uima.ducc.common.IIdentity;
-import org.apache.uima.ducc.common.Node;
-import org.apache.uima.ducc.common.NodeIdentity;
-import org.apache.uima.ducc.common.main.DuccService;
 import org.apache.uima.ducc.common.utils.DuccLogger;
-import org.apache.uima.ducc.common.utils.DuccProperties;
-import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.common.utils.id.IDuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
-import org.apache.uima.ducc.database.DbConstants.DbEdge;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
-import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent;
-import org.apache.uima.ducc.transport.cmdline.ICommandLine;
-import org.apache.uima.ducc.transport.event.common.IDuccPerWorkItemStatistics;
-import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
-import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
-import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccStandardInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccUimaAggregateComponent;
-import org.apache.uima.ducc.transport.event.common.IDuccUimaDeployableConfiguration;
-import org.apache.uima.ducc.transport.event.common.IRationale;
-import org.apache.uima.ducc.transport.event.common.ITimeWindow;
-import org.apache.uima.ducc.transport.event.common.JdReservationBean;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.InstanceCreator;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSerializationContext;
-import com.google.gson.JsonSerializer;
-import com.google.gson.TypeAdapter;
-import com.google.gson.TypeAdapterFactory;
-import com.google.gson.reflect.TypeToken;
-import com.google.gson.stream.JsonReader;
-import com.google.gson.stream.JsonToken;
-import com.google.gson.stream.JsonWriter;
-import com.orientechnologies.orient.core.record.impl.ODocument;
-import com.orientechnologies.orient.core.sql.OCommandSQL;
-import com.tinkerpop.blueprints.Direction;
-import com.tinkerpop.blueprints.Edge;
-import com.tinkerpop.blueprints.Vertex;
-import com.tinkerpop.blueprints.impls.orient.OrientBaseGraph;
-import com.tinkerpop.blueprints.impls.orient.OrientEdge;
-import com.tinkerpop.blueprints.impls.orient.OrientVertex;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.SimpleStatement;
 
 public class DbHandle
 {
     private DuccLogger logger = null;
     
     DbManager manager;
-    public OrientBaseGraph graphDb;
 
-    DbHandle(DbManager manager, OrientBaseGraph graphDb)
+    DbHandle(DbManager manager)
     {
-        if ( DuccService.getDuccLogger() == null ) {
-            // not running within a ducc service - just get a regular logger
-            logger = DuccLogger.getLogger(DbHandle.class, "DB");
-        } else {
-            // running within a ducc service - get the component logger
-            logger = DuccService.getDuccLogger(DbHandle.class.getName());
-        }
+        logger = DuccLogger.getLogger(DbHandle.class, "DB");
+//         if ( DuccService.getDuccLogger() == null ) {
+//             // not running within a ducc service - just get a regular logger
+//             logger = DuccLogger.getLogger(DbHandle.class, "DB");
+//         } else {
+//             // running within a ducc service - get the component logger
+//             logger = DuccService.getDuccLogger(DbHandle.class.getName());
+//         }
         this.manager = manager;
-        this.graphDb = graphDb;
-    }
-
-
-    public void close()
-    {
-        if ( graphDb == null ) return;
-        graphDb.shutdown();
-        graphDb = null;
-    }
-    
-    /**
-     * We use id + isHistory + vertex class = primary key
-     * @param id The numeric duccid of the 'thing'
-     * @param type the enum representing the database class of the object 
-     * @param isHistory 'true' if we're searching history, 'false' otherwise
-     */
-    public boolean thingInDatabase(long id, DbVertex type, DbCategory category)
-        throws Exception
-    {
-    	String methodName = "thingInDatabase";
-        Iterable<Vertex> vs =  null;
-        if ( category == DbCategory.Any ) {
-            vs = select("SELECT count(*) FROM " + type.pname() + 
-                                         " WHERE " + DbConstants.DUCCID + "="  + id);
-        } else {
-            vs = select("SELECT count(*) FROM " + type.pname() + 
-                                         " WHERE " + DbConstants.DUCCID + "="  + id + 
-                                         " AND " + DbConstants.DUCC_DBCAT + "='" + category.pname() + "'");
-        }
-
-        try {
-            for ( Vertex v : vs ) {
-                OrientVertex ov = (OrientVertex) v;
-                ODocument doc = ov.getRecord();
-                long did = doc.field("count");
-                return (did > 0);
-            }            
-		} catch (Throwable e) {
-			logger.error(methodName, null, e);			
-		}
-        return false;
-    }
-
-    /**
-     * Find all DUCCIDs associated with the incoming type.
-     *
-     * @param type The type enum for the database class, e.g. Job
-     * @param isHistory 'true' to search history, 'false' otherwise
-     *
-     * @return List of human-reaable (aka "friendly" ducc ids as strings.
-     */
-    public List<Long> listObjectsOfType(DbVertex type, DbCategory dbcat)
-    {
-        Iterable<Vertex> objs = null;
-        if ( dbcat == DbCategory.Any ) {
-            objs = graphDb.getVertices(type.pname(), 
-                                       new String[] {"@class"},
-                                       new Object[]{type.pname()});
-        } else {
-            graphDb.getVertices(type.pname(), 
-                                new String[] {"@class", DbConstants.DUCC_DBCAT}, 
-                                new Object[]{type.pname(), dbcat.pname()});
-        }
-        List<Long> ret = new ArrayList<Long>();
-
-        for ( Vertex v : objs ) {
-            OrientVertex ov = (OrientVertex) v;
-            ret.add((Long)ov.getProperty(DbConstants.DUCCID));
-        }
-
-        return ret;
-    }
-
-    /**
-     * Convert the vertex property set to a DuccPropeties, scrubbing the
-     * ducc metadata.
-     */
-    Properties vertexToProps(OrientVertex v)
-    {
-        Properties ret = new DuccProperties();
-        Set<String> keys = v.getPropertyKeys();
-        for (String k : keys) {
-            if ( k.equals(DbConstants.DUCCID) ) continue;
-            if ( k.equals(DbConstants.DUCC_DBCAT) ) continue;
-            ret.put(k, v.getProperty(k));
-        }
-        return ret;
-    }
-
-    /**
-     * Find the objects of the given type and return their Properties, indexed by duccid.
-     *
-     * NOTE This returns the properties as DuccProperties so you can safely cast and use the
-     *      extended function of that class if you want.
-     * 
-     * @param type The type enum for the object class (e.g. Job)
-     * @param isHistory 'true' to search history, 'false' otherwise
-     */
-    public Map<Long, Properties> getPropertiesForType(DbVertex type, DbCategory dbcat)
-    		throws Exception
-    {
-        String methodName = "getPropertiesForType";
-        
-        Iterable<Vertex> vs = null;
-        if ( dbcat == DbCategory.Any ) {
-            vs = graphDb.getVertices(type.pname(), 
-                                     new String[] {"@class"}, 
-                                     new Object[]{type.pname()});
-            
-        } else {
-            String a = type.pname();
-            String[] b = new String[] {"@class", DbConstants.DUCC_DBCAT};
-            Object[] c = new Object[]{type.pname(), dbcat.pname()};
-            vs = graphDb.getVertices(a, b, c);
-                                     
-                                     
-        }
-
-        Map<Long, Properties> ret = new HashMap<Long, Properties>();
-
-        try {
-            for ( Vertex v : vs ) {
-                OrientVertex ov = (OrientVertex) v;
-                Properties props = vertexToProps(ov);
-                Long did = ov.getProperty(DbConstants.DUCCID);
-                ret.put(did, props);
-            }            
-		} catch (Throwable e) {
-			logger.error(methodName, null, "Database access error: ", e);
-		}
-
-        return ret;
     }
-
-    public Map<Long, Properties> getPropertiesForTypeSel(DbVertex type, DbCategory dbcat)
-    		throws Exception
-    {
-        String methodName = "getPropertiesForType";
         
-        Iterable<Vertex> vs = null;
-        if ( dbcat == DbCategory.Any ) {
-            vs = graphDb.getVertices(type.pname(), 
-                                     new String[] {"@class"}, 
-                                     new Object[]{type.pname()});
-            
-        } else {
-            vs = select("SELECT FROM " + type.pname() + " WHERE " + DbConstants.DUCC_DBCAT + "='" + dbcat.pname()+"'");
-        }
-
-        Map<Long, Properties> ret = new HashMap<Long, Properties>();
-
-        try {
-            for ( Vertex v : vs ) {
-                OrientVertex ov = (OrientVertex) v;
-                Properties props = vertexToProps(ov);
-                Long did = ov.getProperty(DbConstants.DUCCID);
-                ret.put(did, props);
-            }            
-		} catch (Throwable e) {
-			logger.error(methodName, null, "Database access error: ", e);
-		}
-
-        return ret;
-    }
-
-    /**
-     * Use this for selecting, it returns a set of stuff
-     */
-    public Iterable<Vertex> select(String sql)
+    public ResultSet execute(String sql)
         throws Exception
     {
-    	String methodName = "select";
-        long now = System.currentTimeMillis();
-        logger.info(methodName, null, "SQL", sql);
-        logger.info(methodName, null, "Time to select", System.currentTimeMillis() - now);
-
-        return graphDb.command(new OCommandSQL(sql)).execute();
-    }
-
-    /**
-     * Use this for just executing stuff that returns an int rc
-     */
-    public int execute(String sql)
-    {
-    	String methodName = "execute";
+        String methodName = "execute";
         long now = System.currentTimeMillis();
-        logger.info(methodName, null, "SQL", sql);
+        ResultSet ret = manager.execute(sql);
         logger.info(methodName, null, "Time to execute", System.currentTimeMillis() - now);
-
-        return graphDb.command(new OCommandSQL(sql)).execute();
+        
+        return ret;
     }
 
-    public void commit()
+    public ResultSet execute(SimpleStatement s)
     {
-        String methodName = "commit";
-        long now = System.currentTimeMillis();
-        if ( graphDb != null ) graphDb.commit();
-        logger.info(methodName, null, "Time to commit", System.currentTimeMillis() - now);
+        return manager.execute(s);
     }
 
-    public void rollback()
+    public ResultSet execute(BoundStatement s)
     {
-        String methodName = "rollback";
-        logger.warn(methodName, null, "ROLLBACK");
-        if ( graphDb != null ) graphDb.
-        rollback();
+        return manager.execute(s);
     }
 
     /**
@@ -339,24 +98,24 @@ public class DbHandle
      * @param obj The json-ified object to save
      * @param isHistory 'true' if we save to history, 'false' otherwise
      */
-    public OrientVertex saveObject(DbVertex type, Long duccid, String obj, DbCategory dbcat)
-    {
-    	//String methodName = "saveObject";
+//     public OrientVertex saveObject(DbVertex type, Long duccid, String obj, DbCategory dbcat)
+//     {
+//     	//String methodName = "saveObject";
+
+//         //String typename = type.pname();
+
+//         OrientVertex ret = null;
+//         ODocument document = null;
+//         document = new ODocument(type.pname());
+//         ret = new OrientVertex(graphDb, document);
+
+//         document.fromJSON(obj);
+//         document.field(DbConstants.DUCCID, duccid);
+//         document.field(DbConstants.DUCC_DBCAT, dbcat.pname());
+//         graphDb.getRawGraph().save(document);
 
-        //String typename = type.pname();
-
-        OrientVertex ret = null;
-        ODocument document = null;
-        document = new ODocument(type.pname());
-        ret = new OrientVertex(graphDb, document);
-
-        document.fromJSON(obj);
-        document.field(DbConstants.DUCCID, duccid);
-        document.field(DbConstants.DUCC_DBCAT, dbcat.pname());
-        graphDb.getRawGraph().save(document);
-
-        return ret;
-    }
+//         return ret;
+//     }
 
 
     /**
@@ -373,27 +132,27 @@ public class DbHandle
      *        @TODO must research and use it as it will likely be safer and more efficient.
      * 
      */
-    private DbObject traverseObject(OrientVertex v, boolean read_all)
-    {
-        //String methodName = "traverseObject";
-        ODocument doc = v.getRecord();
-        String doc_json = doc.toJSON();
-        String stype = v.getProperty("@class");
-        DbVertex type = manager.vertexType(stype);
-
-        DbObject ret = new DbObject(doc_json, type);
-
-        if ( read_all ) {
-            Iterable<Edge> ed = v.getEdges(Direction.OUT);
-            for ( Edge e : ed ) {
-                OrientEdge oe = (OrientEdge) e;
-                OrientVertex ov = oe.getVertex(Direction.IN);
-                //logger.info(methodName, null, "Found edge connected to vertex of type", ov.getProperty("@class"), "duccid", ov.getProperty(DUCCID));
-                ret.addEmbedded(traverseObject(ov, read_all));
-            }
-        }
-        return ret;
-    }
+//     private DbObject traverseObject(OrientVertex v, boolean read_all)
+//     {
+//         //String methodName = "traverseObject";
+//         ODocument doc = v.getRecord();
+//         String doc_json = doc.toJSON();
+//         String stype = v.getProperty("@class");
+//         DbVertex type = manager.vertexType(stype);
+
+//         DbObject ret = new DbObject(doc_json, type);
+
+//         if ( read_all ) {
+//             Iterable<Edge> ed = v.getEdges(Direction.OUT);
+//             for ( Edge e : ed ) {
+//                 OrientEdge oe = (OrientEdge) e;
+//                 OrientVertex ov = oe.getVertex(Direction.IN);
+//                 //logger.info(methodName, null, "Found edge connected to vertex of type", ov.getProperty("@class"), "duccid", ov.getProperty(DUCCID));
+//                 ret.addEmbedded(traverseObject(ov, read_all));
+//             }
+//         }
+//         return ret;
+//     }
 
     /**
      * Read a database object, optionally chasing edges to get all the various bits.
@@ -402,374 +161,106 @@ public class DbHandle
      * @param duccid The numeric id of the object
      * @param read_all 'true' to recursively chase down the edges, 'false' otherwise.
      */
-    public DbObject readObject(DbVertex type, Long duccid, boolean read_all)
-    {
-    	String methodName = "readObject";
-        Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCCID}, new Object[]{type.pname(), duccid});
-        int count = 0;
-        OrientVertex fv = null;
-        for (Vertex v : s) {
-            fv = (OrientVertex) v;
-            //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
-            count++;
-        }
-        if ( count > 1 ) {
-            logger.error(methodName, null, "Expected unique object, found", count, "objects. Returning only the first object.");
-        }
-
-        //logger.info(methodName, null, "Traversing", fv.getProperty("@class"), "duccid", fv.getProperty(DUCCID));
-        return traverseObject(fv, read_all);
-    }
-
-    /**
-     * Link the 'from' to all the 'to''s' with edges of the right type. Everything is expected
-     * to be in the db already, but  maybe not yet committed, according to the caller.
-     *
-     * @param from the source object 
-     * @param to a list of the things the source is pointing/linking to
-     * @param type The type enum for edge, e.g. 'process'
+//     public DbObject readObject(DbVertex type, Long duccid, boolean read_all)
+//     {
+//     	String methodName = "readObject";
+//         Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCCID}, new Object[]{type.pname(), duccid});
+//         int count = 0;
+//         OrientVertex fv = null;
+//         for (Vertex v : s) {
+//             fv = (OrientVertex) v;
+//             //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
+//             count++;
+//         }
+//         if ( count > 1 ) {
+//             logger.error(methodName, null, "Expected unique object, found", count, "objects. Returning only the first object.");
+//         }
+
+//         //logger.info(methodName, null, "Traversing", fv.getProperty("@class"), "duccid", fv.getProperty(DUCCID));
+//         return traverseObject(fv, read_all);
+//     }
+
+    /**
+     * This views a row as a set of properties.  We update a single column in the table.
+     *
+     * @param table The table name.
+     * @param row   The key to search on for update.  Caller must fully-form it e.g. "name=bob", if the key
+     *                  field is called 'name'.
+     * @param propkey The name of the column.
+     * @param propval The value to replace the existing value.
      */
-    public void addEdges(OrientVertex from, List<OrientVertex> to, DbEdge type)
+    public boolean updateProperty(String table, String row, String propkey, Object propval)
+        throws Exception
     {
-        //String methodName = "addEdges";
-        OrientVertex fv = (OrientVertex) from;
-        String etype = type.pname();
-
-        for ( Object o : to ) {
-            OrientVertex tv = (OrientVertex) o;
-            fv.addEdge(etype, tv);
+    	String methodName = "updateProperty";
+        long now = System.currentTimeMillis();
+        
+        String cql = "UPDATE " + table + " SET " + propkey + "=" + propval + " WHERE " + row;
+        try {
+            manager.execute(cql);
+            return true;
+        } finally {
+            logger.info(methodName, null, "Time to update one property", propkey, System.currentTimeMillis() - now);        
         }
-
     }
-
     
     /**
-     * Link the 'from' to a single 'to' with an e right type. Everything is expected
-     * to be in the db already, but  maybe not yet committed, according to the caller.
-     *
-     * @param from the source object 
-     * @param a single object to point to
-     * @param type The type enum for edge, e.g. 'process'
-     */
-    public void addEdge(OrientVertex from, OrientVertex to, DbEdge type)
-    {
-        //String methodName = "addEdges";
-        OrientVertex fv = (OrientVertex) from;
-        OrientVertex tv = (OrientVertex) to;
-        String etype = type.pname();
-
-        fv.addEdge(etype, tv);
-    }
-
-    // public void addEdges(DbVertex typeFrom, Long fid, DbVertex typeTo, List<Long> tids)
-    // {
-    //     String methodName = "addEdges";
-
-    //     Iterable<Vertex> s = graphDb.getVertices(typeFrom.pname(), new String[] {"@class", DUCCID}, new Object[]{typeFrom.pname(), fid});
-    //     int count = 0;
-    //     OrientVertex fv = null;
-    //     for (Vertex v : s) {
-    //         fv = (OrientVertex) v;
-    //         //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
-    //         count++;
-    //     }
-
-    //     for ( Long tid : tids ) {
-    //         Iterable<Vertex> ss = graphDb.getVertices(typeTo.pname(), new String[] {"@class", DUCCID}, new Object[]{typeTo.pname(), tid});
-    //         count = 0;
-    //         for (Vertex v : ss) {
-    //             OrientVertex tv = (OrientVertex) v;
-    //             // logger.info(methodName, null, "to vertex", count, tv.getIdentity());
-    //             count++;
-                
-    //             fv.addEdge(DbEdge.Edge.pname(), tv);
-    //         }
-    //     }
-
-
-    // }
-
-    // public void addEdge(DbVertex typeFrom, Long fid, DbVertex typeTo, Long tid)
-    // {
-    //     String methodName = "addEdge";
-
-    //     Iterable<Vertex> s = graphDb.getVertices(typeFrom.pname(), new String[] {"@class", DUCCID}, new Object[]{typeFrom.pname(), fid});
-    //     int count = 0;
-    //     OrientVertex fv = null;
-    //     for (Vertex v : s) {
-    //         fv = (OrientVertex) v;
-    //         //logger.info(methodName, null, "from vertex", count, fv.getIdentity());
-    //         count++;
-    //     }
-
-    //     Iterable<Vertex> ss = graphDb.getVertices(typeTo.pname(), new String[] {"@class", DUCCID}, new Object[]{typeTo.pname(), tid});
-    //     count = 0;
-    //     for (Vertex v : ss) {
-    //         OrientVertex tv = (OrientVertex) v;
-    //         // logger.info(methodName, null, "to vertex", count, tv.getIdentity());
-    //         count++;
-
-    //         fv.addEdge(DbEdge.Edge.pname(), tv);
-    //     }
-
-
-    // }
-
-    public void changeCategory(OrientVertex obj, DbCategory to)
-    {
-    	String methodName = "changeCategory";
-        Long id = obj.getProperty(DbConstants.DUCCID);
-        String oldcat = obj.getProperty(DbConstants.DUCC_DBCAT);
-        String type = obj.getProperty("@CLASS");
-        logger.info(methodName, null, "Chnage category obj of type", type, "id", id, "from", oldcat, "to", to.pname());
-        obj.setProperty(DbConstants.DUCC_DBCAT, to.pname());
-    }
-
-    /**
-     * Update a single property.  The object must be unique and must already exist.
+     * This views a row as a set of properties.  We update multiple columns in a single row of the table.
      *
-     * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
-     * @param key Holds the value of the primary key
-     * @param type The vertex type
-     * @param dbcat The category, e.g. history, or checkpoint, or rmstate
-     * @param props The properties object to be placed in the db.
+     * @param table The table name.
+     * @param row   The key to search on for update.  Caller must fully-form it e.g. "name=bob", if the key
+     *                  field is called 'name'.
+     * @param props This is a list of propertes where every even-numbered object is a column name and
+     *                  every odd-numbered object is its value.
+     * 
+     * @throws IllegalArgumentException if the number of 'props' is not even.  Lower layers may throw
+     *                  unchecked exceptions also.
      */
-    public OrientVertex updateProperty(String keyid, Object key, DbVertex type,  DbCategory dbcat, String propkey, Object propval)
+    public void updateProperties(String table, String row, Object... props)
+        throws Exception
     {
-    	String methodName = "updateProperty";
+    	String methodName = "updateProperties";
         long now = System.currentTimeMillis();
 
-        Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
-        logger.info(methodName, null, "Time to search on " + type.pname() + " where category " + dbcat.pname() + " and " + keyid + " propkey "  + key, System.currentTimeMillis() - now);
-        
-        OrientVertex ov = null;
-        int count = 0;                        // some sanity checking, we're not allowed more than one
-        for (Vertex v : s) {
-            ov = (OrientVertex) v;
-            count++;
-        }
-        
-        if ( count > 1 ) {
-            throw new IllegalStateException("Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey "  + propkey + " propval " + propval); 
+        if ( ( props.length % 2 ) != 0 ) {
+            throw new IllegalArgumentException("mkUpdate: need even number of props to form (k,v) list.  Found " + props.length + " props.");
         }
 
-        if ( ov == null ) {
-            throw new IllegalStateException("No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey "  + propkey + " propval " + propval); 
+        String cql = DbUtil.mkUpdate(table, row, props);
+        try {
+            manager.execute(cql);
+        } finally {
+            logger.info(methodName, null, "Total time to update properties", System.currentTimeMillis() - now);
         }
-
-
-        ov.setProperty(propkey, propval);
-        ov.save();            
-        logger.info(methodName, null, "Time to update one property", System.currentTimeMillis() - now);
-
-        return ov;
     }
 
 
-    public OrientVertex updateProperty(Object dbid, String propkey, Object propval)
+    PreparedStatement prepare(String cql)
     {
-    	String methodName = "updateProperty";
-        long now = System.currentTimeMillis();
-                
-        OrientVertex ov = graphDb.getVertex(dbid);
-        logger.info(methodName, null, "Time to search on " + dbid.toString(), System.currentTimeMillis() - now);
-
-        if ( ov == null ) {
-            throw new IllegalStateException("No object in db: Id " + dbid.toString()); 
-        }
-
-        ov.setProperty(propkey, propval);
-        ov.save();            
-        logger.info(methodName, null, "Time to update one property for", dbid.toString(), System.currentTimeMillis() - now);
-
-        return ov;
+        String methodName = "prepare";
+        logger.info(methodName, null, "Preparing", cql);
+        return manager.prepare(cql);
     }
 
-    /**
-     * Update the properties on an object in the db with the incoming properties.  The object must exist.
-     * There is no attempt to synchronize values - the new props either replace, or enrich the object.
-     *
-     * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
-     * @param key Holds the value of the primary key
-     * @param type The vertex type
-     * @param dbcat The category, e.g. history, or checkpoint, or rmstate
-     * @param props A list of (k,v) pairs which will replace their counterparts in the object.
-     */
-    public OrientVertex updateProperties(String keyid, Object key, DbVertex type,  DbCategory dbcat, Object... props)
+    void truncate(String table)
+    	throws Exception
     {
-    	String methodName = "updateProperties";
-        long now = System.currentTimeMillis();
-        Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
-        logger.info(methodName, null, "Time to search on " + type.pname() + " where category " + dbcat.pname() + " and " + keyid + " propkey "  + key, System.currentTimeMillis() - now);
-
-        OrientVertex ov = null;
-        int count = 0;                        // some sanity checking, we're not allowed more than one
-        for (Vertex v : s) {
-            ov = (OrientVertex) v;
-            count++;
-        }
-
-        if ( count > 1 ) {
-            throw new IllegalStateException("updateProperties: Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + keyid + " propkey "  + key);
-        }
-
-        if ( ov == null ) {
-            throw new IllegalStateException("updateProperties: No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + keyid + " propkey "  + key);
-        }
-        logger.info(methodName, null, "Time to find a vertex", System.currentTimeMillis() - now);
-
-        long now2 = System.currentTimeMillis();     
-        ov.setProperties(props);
-        ov.save();            
-        logger.info(methodName, null, "Time to modify a vertex", System.currentTimeMillis() - now2);
-
-        logger.info(methodName, null, "Time to update several properties", System.currentTimeMillis() - now);
-
-        return ov;
+        manager.truncate(table);
     }
 
-    public OrientVertex updateProperties(Object dbid, Object... props)
+
+    void saveObject(PreparedStatement ps, Object ... fields)
+        throws Exception
     {
-    	String methodName = "updateProperties";
+        String methodName = "saveObject";        
         long now = System.currentTimeMillis();
-        
-
-        OrientVertex ov = graphDb.getVertex(dbid);
-
-        if ( ov == null ) {
-            throw new IllegalStateException("updateProperties: No object in db: id " + dbid.toString());
-        }
-        logger.info(methodName, null, "Time to find a vertex", dbid.toString(), System.currentTimeMillis() - now);
-
-        long now2 = System.currentTimeMillis();     
-        ov.setProperties(props);
-        ov.save();            
-        logger.info(methodName, null, "Time to modify vertex", dbid.toString(), System.currentTimeMillis() - now2);
 
-        logger.info(methodName, null, "Total time to update vertex", dbid.toString(), System.currentTimeMillis() - now);
-
-        return ov;
+        BoundStatement boundStatement = new BoundStatement(ps);
+        BoundStatement bound = boundStatement.bind(fields);
+        execute(bound);        
+        logger.info(methodName, null, "Time to execute prepared statement:", ps.getQueryString(), System.currentTimeMillis() - now);
     }
 
-    // /**
-    //  * Synchronize the properties in an object with the incoming properties.  THe object must be unique
-    //  * and already exist.
-    //  *
-    //  * @param keyid DUCCID or DUCC_DBNODE usually. It's the name of the field with the primary key.
-    //  * @param key Holds the value of the primary key
-    //  * @param type The vertex type
-    //  * @param dbcat The category, e.g. history, or checkpoint, or rmstate
-    //  * @param props The properties object to be placed in the db.
-    //  */
-    // public OrientVertex synchronizeProperties(String keyid, Object key, DbVertex type,  DbCategory dbcat, Properties props)
-    // {
-    //     String methodName = "synchronizeProperties";
-    //     long now = System.currentTimeMillis();
-
-    //     Iterable<Vertex> s = graphDb.getVertices(type.pname(), new String[] {"@class", DbConstants.DUCC_DBCAT, keyid}, new Object[]{type.pname(), dbcat.pname(), key});
-    //     logger.info(methodName, null, "Time to search " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now);
-
-
-    //     long now2 = System.currentTimeMillis();
- 
-    //     OrientVertex ov = null;
-    //     int count = 0;                        // some sanity checking, we're not allowed more than one
-    //     for (Vertex v : s) {
-    //         ov = (OrientVertex) v;
-    //         count++;
-    //     }
-
-    //     if ( count > 1 ) {
-    //         throw new IllegalStateException("updateProperties: Duplocate object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey "  + key);
-    //     }
-
-    //     if ( ov == null ) {
-    //         throw new IllegalStateException("updateProperties: No object in db: Type " + type.pname() + " category " + dbcat.pname() + " key " + key + " propkey "  + key);
-    //     }
-    //     logger.info(methodName, null, "Time to itarate " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now2);
-    //     now2 = System.currentTimeMillis();
-                                                  
-    //     logger.info(methodName, null, "Update db record of type", type.pname(), "category", dbcat.pname(), "key", key);
-    //     Set<String> keys = ov.getPropertyKeys();
-    //     for (String k : keys) {
-    //         if ( k.equals(DbConstants.DUCCID) )      continue;      // must bypass schema things
-    //         if ( k.equals(DbConstants.DUCC_DBCAT) )  continue;
-    //         if ( k.equals(DbConstants.DUCC_DBNODE) ) continue;
-     
-    //         Object val1 = ov.getProperty(k);
-    //         Object val2 = props.get(k);
-     
-    //         if ( val2 == null ) {                                  // the property is removed
-    //             logger.info(methodName, null, "Removed property", k);
-    //             ov.removeProperty(k);
-    //             continue;
-    //         }
-     
-    //         if ( !val2.equals(val1) ) {                            // replace/add a property. val2 is known not null
-    //             logger.info(methodName, null, "Replaced/added property", k, "value", val1, "with", val2);
-    //             ov.setProperty(k, val2);
-    //             continue;
-    //         }
-    //     }
- 
-    //     ov.setProperty(keyid, key);
-    //     ov.save();            
-    //     logger.info(methodName, null, "Time to update " + type.pname() + " category " + dbcat.pname() + " key " + key, System.currentTimeMillis() - now2);
-
-    //     logger.info(methodName, null, "Time to synchronize properties", System.currentTimeMillis() - now);
-
-    //     return ov;
-    // }
-
-    // public OrientVertex synchronizeProperties(Object dbid, Properties props)
-    // {
-    //     String methodName = "synchronizeProperties";
-    //     long now = System.currentTimeMillis();
-
-
-
-    //     long now2 = System.currentTimeMillis();
- 
-    //     OrientVertex ov = graphDb.getVertex(dbid);
-    //     if ( ov == null ) {
-    //         throw new IllegalStateException("updateProperties: No object in db. Id " + dbid);
-    //     }
-    //     logger.info(methodName, null, "Time to search for ", dbid, System.currentTimeMillis() - now);
-
-    //     now2 = System.currentTimeMillis();                                                  
-    //     logger.info(methodName, null, "Update db record ", dbid);
-    //     Set<String> keys = ov.getPropertyKeys();
-    //     for (String k : keys) {
-    //         if ( k.equals(DbConstants.DUCCID) )      continue;      // must bypass schema things
-    //         if ( k.equals(DbConstants.DUCC_DBCAT) )  continue;
-    //         if ( k.equals(DbConstants.DUCC_DBNODE) ) continue;
-
-    //         if ( k.equals("svc_dbid") ) continue;
-    //         if ( k.equals("meta_dbid") ) continue;
-
-    //         Object val1 = ov.getProperty(k);
-    //         Object val2 = props.get(k);
-     
-    //         if ( val2 == null ) {                                  // the property is removed
-    //             logger.info(methodName, null, "Removed property", k);
-    //             ov.removeProperty(k);
-    //             continue;
-    //         }
-     
-    //         if ( !val2.equals(val1) ) {                            // replace/add a property. val2 is known not null
-    //             logger.info(methodName, null, "Replaced/added property", k, "value", val1, "with", val2);
-    //             ov.setProperty(k, val2);
-    //             continue;
-    //         }
-    //     }
- 
-    //     ov.save();            
-    //     logger.info(methodName, null, "Time to update record", dbid, System.currentTimeMillis() - now2);
-    //     logger.info(methodName, null, "Total time to synchronize record", dbid, System.currentTimeMillis() - now);
-
-    //     return ov;
-    //    }
-
     /**
      * Create an object in the db from a properties object.  The caller must do the the checking to insure
      * the object already exists (or not, e.g. for a db loader).
@@ -779,222 +270,223 @@ public class DbHandle
      * @param duccid The numeric id of the object
      * @param isHistory 'True' if it is to be placed in history, 'false' otherwise.
      */
-    public OrientVertex createProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Properties props)
-    {
-    	String methodName = "createPropertiesObject";
-        String typeName = type.pname();
-        OrientVertex ov = null;
-
-        logger.info(methodName, null, "Create new properties object of type", type.pname(), "category", dbcat.pname(), "key", key); 
-        ov = graphDb.addVertex("class:" + typeName, keyid, key, DbConstants.DUCC_DBCAT, dbcat.pname());
-        ov.setProperties(props);
-        return ov;
-    }
-
-    static Gson gson = null;
-    static Gson mkGsonForJob()
-    {
-        synchronized(DbHandle.class) {
-            if ( gson != null ) return gson;
+//     public OrientVertex createProperties(String keyid, Object key, DbVertex type, DbCategory dbcat, Properties props)
+//     {
+//     	String methodName = "createPropertiesObject";
+//         String typeName = type.pname();
+//         OrientVertex ov = null;
+
+//         logger.info(methodName, null, "Create new properties object of type", type.pname(), "category", dbcat.pname(), "key", key); 
+//         ov = graphDb.addVertex("class:" + typeName, keyid, key, DbConstants.DUCC_DBCAT, dbcat.pname());
+//         ov.setProperties(props);
+//         return ov;
+//     }
+
+//     static Gson gson = null;
+//     static Gson mkGsonForJob()
+//     {
+//         synchronized(DbHandle.class) {
+//             if ( gson != null ) return gson;
 
-            // We need to define Instance creators and such so we do it in a common place
-            GsonBuilder gb = new GsonBuilder();
+//             // We need to define Instance creators and such so we do it in a common place
+//             GsonBuilder gb = new GsonBuilder();
             
-            GenericInterfaceAdapter customAdapter = new GenericInterfaceAdapter();
-            gb.serializeSpecialFloatingPointValues().setPrettyPrinting();
-            gb.enableComplexMapKeySerialization();
+//             GenericInterfaceAdapter customAdapter = new GenericInterfaceAdapter();
+//             gb.serializeSpecialFloatingPointValues().setPrettyPrinting();
+//             gb.enableComplexMapKeySerialization();
             
-            gb.registerTypeAdapter(Node.class, new NodeInstanceCreator());
-            gb.registerTypeAdapter(NodeIdentity.class, new NodeIdentityCreator());
+//             gb.registerTypeAdapter(Node.class, new NodeInstanceCreator());
+//             gb.registerTypeAdapter(NodeIdentity.class, new NodeIdentityCreator());
             
-            //gb.registerTypeAdapter(IIdentity.class, new IdentityInstanceCreator());
-            gb.registerTypeAdapter(IIdentity.class, customAdapter);
+//             //gb.registerTypeAdapter(IIdentity.class, new IdentityInstanceCreator());
+//             gb.registerTypeAdapter(IIdentity.class, customAdapter);
             
-            gb.registerTypeAdapter(IDuccId.class, customAdapter);
-            gb.registerTypeAdapter(ICommandLine.class, customAdapter);
-            gb.registerTypeAdapter(ITimeWindow.class, customAdapter);
-            gb.registerTypeAdapter(IDuccProcessWorkItems.class, customAdapter);
-            gb.registerTypeAdapter(IDuccUimaAggregateComponent.class, customAdapter);
-            gb.registerTypeAdapter(IUimaPipelineAEComponent.class, customAdapter);
-            gb.registerTypeAdapter(IRationale.class, customAdapter);
-            gb.registerTypeAdapter(IDuccUimaDeployableConfiguration.class, customAdapter);
-            gb.registerTypeAdapter(IDuccStandardInfo.class, customAdapter);
-            gb.registerTypeAdapter(IDuccSchedulingInfo.class, customAdapter);
-            gb.registerTypeAdapter(IDuccPerWorkItemStatistics.class, customAdapter);
-            gb.registerTypeAdapter(IDuccReservationMap.class, customAdapter);
-            gb.registerTypeAdapter(JdReservationBean.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccId.class, customAdapter);
+//             gb.registerTypeAdapter(ICommandLine.class, customAdapter);
+//             gb.registerTypeAdapter(ITimeWindow.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccProcessWorkItems.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccUimaAggregateComponent.class, customAdapter);
+//             gb.registerTypeAdapter(IUimaPipelineAEComponent.class, customAdapter);
+//             gb.registerTypeAdapter(IRationale.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccUimaDeployableConfiguration.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccStandardInfo.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccSchedulingInfo.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccPerWorkItemStatistics.class, customAdapter);
+//             gb.registerTypeAdapter(IDuccReservationMap.class, customAdapter);
+//             gb.registerTypeAdapter(JdReservationBean.class, customAdapter);
             
-            //ConcurrentHashMap<DuccId, Long> x = new ConcurrentHashMap<DuccId, Long>();
-            //gb.registerTypeAdapter(x.getClass(), new MapAdaptor());
+//             //ConcurrentHashMap<DuccId, Long> x = new ConcurrentHashMap<DuccId, Long>();
+//             //gb.registerTypeAdapter(x.getClass(), new MapAdaptor());
             
-            //gb.registerTypeAdapterFactory(new DuccTypeFactory());
-            //Object obj = new ArrayList<IJdReservation>();
-            //gb.registerTypeAdapter(obj.getClass(), customAdapter);
-            Gson g = gb.create();
-            return g;
-        }
-    }
-
-    // ----------------------------------------------------------------------------------------------------
-    // Instance creators and adaptors for GSON
-    // ----------------------------------------------------------------------------------------------------
-
-    // We need these for the DuccNode and NodeIdentity because they don't have no-arg
-    // Constructors.  
-    //
-    // @TODO after merge, consult with Jerry about adding in those constructors
-    private static class NodeInstanceCreator implements InstanceCreator<Node> {
-        public Node createInstance(Type type) {
-            //            System.out.println("DuccNode");
-            return new DuccNode(null, null, false);
-        }
-    }
-
-    private static class NodeIdentityCreator implements InstanceCreator<NodeIdentity> {
-        public NodeIdentity createInstance(Type type) {
-            //            System.out.println("DuccNodeIdentity");
-            try { return new NodeIdentity(null, null); } catch ( Exception e ) {}
-            return null;
-        }
-    }
+//             //gb.registerTypeAdapterFactory(new DuccTypeFactory());
+//             //Object obj = new ArrayList<IJdReservation>();
+//             //gb.registerTypeAdapter(obj.getClass(), customAdapter);
+//             Gson g = gb.create();
+//             return g;
+//         }
+//     }
+
+//     // ----------------------------------------------------------------------------------------------------
+//     // Instance creators and adaptors for GSON
+//     // ----------------------------------------------------------------------------------------------------
+
+//     // We need these for the DuccNode and NodeIdentity because they don't have no-arg
+//     // Constructors.  
+//     //
+//     // @TODO after merge, consult with Jerry about adding in those constructors
+//     private static class NodeInstanceCreator implements InstanceCreator<Node> {
+//         public Node createInstance(Type type) {
+//             //            System.out.println("DuccNode");
+//             return new DuccNode(null, null, false);
+//         }
+//     }
+
+//     private static class NodeIdentityCreator implements InstanceCreator<NodeIdentity> {
+//         public NodeIdentity createInstance(Type type) {
+//             //            System.out.println("DuccNodeIdentity");
+//             try { return new NodeIdentity(null, null); } catch ( Exception e ) {}
+//             return null;
+//         }
+//     }
+
+//     /**
+//      * JSON helper for our complex objects.  Gson doesn't save type information in the json so
+//      * it doesn't know how to construct things declared as interfaces.
+//      *
+//      * This class is a Gson adapter that saves the actual object type in the json on serialization,
+//      * and uses that information on deserialization to construct the right thing.
+//      */
+//     private static class GenericInterfaceAdapter
+//         implements
+//             JsonSerializer<Object>, 
+//             JsonDeserializer<Object> 
+//     {
 
-    /**
-     * JSON helper for our complex objects.  Gson doesn't save type information in the json so
-     * it doesn't know how to construct things declared as interfaces.
-     *
-     * This class is a Gson adapter that saves the actual object type in the json on serialization,
-     * and uses that information on deserialization to construct the right thing.
-     */
-    private static class GenericInterfaceAdapter
-        implements
-            JsonSerializer<Object>, 
-            JsonDeserializer<Object> 
-    {
-
-        private static final String DUCC_META_CLASS = "DUCC_META_CLASS";
+//         private static final String DUCC_META_CLASS = "DUCC_META_CLASS";
         
-        @Override
-        public Object deserialize(JsonElement jsonElement, 
-                                  Type type,
-                                  JsonDeserializationContext jsonDeserializationContext)
-        throws JsonParseException 
-        {
-            // Reconstitute the "right" class based on the actual class it came from as
-            // found in metadata
-            JsonObject  obj    = jsonElement.getAsJsonObject();
-            JsonElement clElem= obj.get(DUCC_META_CLASS);
-
-            if ( clElem== null ) {
-                throw new IllegalStateException("Cannot determine concrete class for " + type + ". Must register explicit type adapter for it.");
-            }
-            String clName = clElem.getAsString();
-
-            //System.out.println("----- elem: " + clName + " clElem: " + obj);
-            try {
-                Class<?> clz = Class.forName(clName);
-                return jsonDeserializationContext.deserialize(jsonElement, clz);
-            } catch (ClassNotFoundException e) {
-                throw new JsonParseException(e);
-            }
-        }
+//         @Override
+//         public Object deserialize(JsonElement jsonElement, 
+//                                   Type type,
+//                                   JsonDeserializationContext jsonDeserializationContext)
+//         throws JsonParseException 
+//         {
+//             // Reconstitute the "right" class based on the actual class it came from as
+//             // found in metadata
+//             JsonObject  obj    = jsonElement.getAsJsonObject();
+//             JsonElement clElem= obj.get(DUCC_META_CLASS);
+
+//             if ( clElem== null ) {
+//                 throw new IllegalStateException("Cannot determine concrete class for " + type + ". Must register explicit type adapter for it.");
+//             }
+//             String clName = clElem.getAsString();
+
+//             //System.out.println("----- elem: " + clName + " clElem: " + obj);
+//             try {
+//                 Class<?> clz = Class.forName(clName);
+//                 return jsonDeserializationContext.deserialize(jsonElement, clz);
+//             } catch (ClassNotFoundException e) {
+//                 throw new JsonParseException(e);
+//             }
+//         }
         
-        @Override
-        public JsonElement serialize(Object object, 
-                                     Type type,
-                                     JsonSerializationContext jsonSerializationContext) 
-        {
-            // Add the mete element indicating what kind of concrete class is this came from
-            //String n = object.getClass().getCanonicalName();
-            //System.out.println("**** Serialize object A " + n + " of type " + type);
-            //if ( n.contains("Concurrent") ) {
-             //   int stop = 1;
-               // stop++;
-            //}
-
-            JsonElement ele = jsonSerializationContext.serialize(object, object.getClass());
-            //System.out.println("**** Serialize object B " + object.getClass().getCanonicalName() + " of type " + type + " : ele " + ele);
-            ele.getAsJsonObject().addProperty(DUCC_META_CLASS, object.getClass().getCanonicalName());
-            return ele;
-        }
-    }
-
-    @SuppressWarnings("unused")
-	private class DuccTypeFactory 
-        implements TypeAdapterFactory
-    {
+//         @Override
+//         public JsonElement serialize(Object object, 
+//                                      Type type,
+//                                      JsonSerializationContext jsonSerializationContext) 
+//         {
+//             // Add the mete element indicating what kind of concrete class is this came from
+//             //String n = object.getClass().getCanonicalName();
+//             //System.out.println("**** Serialize object A " + n + " of type " + type);
+//             //if ( n.contains("Concurrent") ) {
+//              //   int stop = 1;
+//                // stop++;
+//             //}
+
+//             JsonElement ele = jsonSerializationContext.serialize(object, object.getClass());
+//             //System.out.println("**** Serialize object B " + object.getClass().getCanonicalName() + " of type " + type + " : ele " + ele);
+//             ele.getAsJsonObject().addProperty(DUCC_META_CLASS, object.getClass().getCanonicalName());
+//             return ele;
+//         }
+//     }
+
+//     @SuppressWarnings("unused")
+// 	private class DuccTypeFactory 
+//         implements TypeAdapterFactory
+//     {
+
+//         public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) 
+//         {
+//             //System.out.println("TYPETOKEN: " + typeToken + " raw type: " + typeToken.getRawType().getName());
+//             Class<?> cl = typeToken.getRawType();
+//             //System.out.println("         Canonical name: " + cl.getCanonicalName());
+//             Type type = typeToken.getType();
+//             if ( typeToken.getRawType() != ConcurrentHashMap.class ) {
+//                 //System.out.println("Skipping type " + typeToken);
+//                 return null;
+//             }
 
-        public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> typeToken) 
-        {
-            //System.out.println("TYPETOKEN: " + typeToken + " raw type: " + typeToken.getRawType().getName());
-            Class<?> cl = typeToken.getRawType();
-            //System.out.println("         Canonical name: " + cl.getCanonicalName());
-            Type type = typeToken.getType();
-            if ( typeToken.getRawType() != ConcurrentHashMap.class ) {
-                //System.out.println("Skipping type " + typeToken);
-                return null;
-            }
-
-            if ( type instanceof ParameterizedType ) {
+//             if ( type instanceof ParameterizedType ) {
                 
-                ParameterizedType pt = (ParameterizedType) type;
-                Type[] types = pt.getActualTypeArguments();
-                //for ( Type tt : types ) {
-                    // System.out.println("     TYPE ARGUMENTS: " + tt);
-                //}
-                Type tt = types[0];
-                Class<?> cll = (Class<?>) tt;
+//                 ParameterizedType pt = (ParameterizedType) type;
+//                 Type[] types = pt.getActualTypeArguments();
+//                 //for ( Type tt : types ) {
+//                     // System.out.println("     TYPE ARGUMENTS: " + tt);
+//                 //}
+//                 Type tt = types[0];
+//                 Class<?> cll = (Class<?>) tt;
                 
-            }
-            return null;
-        }
-
-    }
-
-    @SuppressWarnings("unused")
-	private class MapAdaptor
-        extends TypeAdapter<ConcurrentHashMap<DuccId, Long>>
-    {
+//             }
+//             return null;
+//         }
+
+//     }
+
+//     @SuppressWarnings("unused")
+// 	private class MapAdaptor
+//         extends TypeAdapter<ConcurrentHashMap<DuccId, Long>>
+//     {
             
-        public void write(JsonWriter out, ConcurrentHashMap<DuccId, Long> map) throws IOException {
-            System.out.println("***************** Writing");
-            if (map == null) {
-                out.nullValue();
-                return;
-            }
-
-            out.beginArray();
-            for (DuccId k : map.keySet() ) {
-                out.beginObject();
-                out.value(k.getFriendly());
-                out.value(k.getUnique());
-                out.value(map.get(k));
-                out.endObject();
-            }
-            out.endArray();
-        }
+//         public void write(JsonWriter out, ConcurrentHashMap<DuccId, Long> map) throws IOException {
+//             System.out.println("***************** Writing");
+//             if (map == null) {
+//                 out.nullValue();
+//                 return;
+//             }
+
+//             out.beginArray();
+//             for (DuccId k : map.keySet() ) {
+//                 out.beginObject();
+//                 out.value(k.getFriendly());
+//                 out.value(k.getUnique());
+//                 out.value(map.get(k));
+//                 out.endObject();
+//             }
+//             out.endArray();
+//         }
                 
-        public ConcurrentHashMap<DuccId, Long> read(JsonReader in) throws IOException {
-            System.out.println("***************** reading");
-            if (in.peek() == JsonToken.NULL) {
-                in.nextNull();
-                return null;
-            }
+//         public ConcurrentHashMap<DuccId, Long> read(JsonReader in) throws IOException {
+//             System.out.println("***************** reading");
+//             if (in.peek() == JsonToken.NULL) {
+//                 in.nextNull();
+//                 return null;
+//             }
                     
-            ConcurrentHashMap<DuccId, Long> ret = new ConcurrentHashMap<DuccId, Long>();
-            in.beginArray();
-            while (in.hasNext()) {
-                in.beginObject();
-                Long friendly = in.nextLong();
-                String unique = in.nextString();
-
-                Long val = in.nextLong();
-                in.endObject();
-                DuccId id = new DuccId(friendly);
-                id.setUUID(UUID.fromString(unique));
-                ret.put(id, val);
-            }
-            in.endArray();
-            return ret;
-        }
-    }
+//             ConcurrentHashMap<DuccId, Long> ret = new ConcurrentHashMap<DuccId, Long>();
+//             in.beginArray();
+//             while (in.hasNext()) {
+//                 in.beginObject();
+//                 Long friendly = in.nextLong();
+//                 String unique = in.nextString();
+
+//                 Long val = in.nextLong();
+//                 in.endObject();
+//                 DuccId id = new DuccId(friendly);
+//                 id.setUUID(UUID.fromString(unique));
+//                 ret.put(id, val);
+//             }
+//             in.endArray();
+//             return ret;
+//         }
+//     }
 }
+

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbLoader.java Wed Oct 28 18:12:53 2015
@@ -23,9 +23,11 @@ import java.io.FileInputStream;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
+import java.util.UUID;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -33,15 +35,17 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.uima.ducc.common.Pair;
 import org.apache.uima.ducc.common.persistence.services.IStateServices;
+import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcMetaProps;
+import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcRegProps;
 import org.apache.uima.ducc.common.utils.DuccLogger;
 import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
 import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
-import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
 
-import com.orientechnologies.orient.core.config.OGlobalConfiguration;
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.PreparedStatement;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
 
 /**
  * Toy orientdb loader to load a historydb from ducc history
@@ -51,7 +55,10 @@ public class DbLoader
 {
     DuccLogger logger = DuccLogger.getLogger(DbLoader.class, "DBLOAD");
     String DUCC_HOME;
+    String SVC_HISTORY_KEY = SvcRegProps.is_archived.columnName();
+    String META_HISTORY_KEY = SvcMetaProps.is_archived.columnName();
 
+    DbManager dbManager = null;
     boolean archive = true;        // for debug and test, bypass archive-rename
     HistoryManagerDb hmd = null;
     StateServicesDb  ssd = null;
@@ -76,32 +83,37 @@ public class DbLoader
     String serviceRegistry = "/state/services";
 
     String checkpointFile = "/state/orchestrator.ckpt";
-    String archive_key  = IStateServices.archive_key;
-    String archive_flag = IStateServices.archive_flag;
 
     int nthreads = 20;
     AtomicInteger counter = new AtomicInteger(0);
 
-    //int joblimit         = 10000;
-    //int reservationlimit = 10000;
-    //int servicelimit     = 10000;
-    //int registrylimit    = 10000;
+    //int joblimit         = 10;
+    //int reservationlimit = 10;
+    //int servicelimit     = 10;
+    //int registrylimit    = 1;
 
     int joblimit = Integer.MAX_VALUE;
     int reservationlimit = Integer.MAX_VALUE;
     int servicelimit = Integer.MAX_VALUE;
     int registrylimit = Integer.MAX_VALUE;
 
-    boolean dojobs         = true;
-    boolean doreservations = true;
-    boolean doservices     = true;
+    boolean dojobs         = false;
+    boolean doreservations = false;
+    boolean doservices     = false;
     boolean doregistry     = true;
-    boolean docheckpoint   = true;
+    boolean docheckpoint   = false;
+
+    long jobBytes = 0;
+    long resBytes = 0;
+    long svcBytes = 0;
+    long svcRegBytes= 0;
+
+    AtomicInteger skippedServices = new AtomicInteger(0);
 
     public DbLoader(String from, String to)
         throws Exception
     {
-    	String methodName = "<ctr>";
+    	//String methodName = "<ctr>";
         DUCC_HOME = System.getProperty("DUCC_HOME");        
         if ( DUCC_HOME == null ) {
             System.out.println("System proprety -DDUCC_HOME must be set.");
@@ -130,10 +142,10 @@ public class DbLoader
         }
 
         String databasedir =  to + "/database/databases";
-        String databasename = databasedir + "/DuccState";
+
         // We always use a non-networked version for loading
         //state_url = "plocal:" + databasedir + "/DuccState";
-        state_url = "remote:bluej538/DuccState";
+        state_url = "bluej538";
         System.setProperty("ducc.state.database.url", state_url);
 
         if ( state_url.startsWith("plocal") ) {
@@ -160,6 +172,7 @@ public class DbLoader
     }
 
     public void loadJobs()
+    	throws Exception
     {
         String methodName = "loadJobs";
 
@@ -234,6 +247,7 @@ public class DbLoader
     }
 
     public void loadReservations()
+        throws Exception
     {
         String methodName = "loadReservations";
 
@@ -309,6 +323,7 @@ public class DbLoader
 
 
     public void loadServices()
+    	throws Exception
     {
         String methodName = "loadServices";
 
@@ -390,7 +405,13 @@ public class DbLoader
 
         int c = 0;
         File dir = new File(registry);
+        if ( ! dir.isDirectory() ) {
+            logger.error(methodName, null, registry, "is not a directory and cannot be loaded.");
+            return;
+        }
+
         File[] files = dir.listFiles();
+        
         if ( files.length == 0 ) {
             if ( isHistory ) {
                 logger.info(methodName, null, "Nothing in service registry history to move to database");
@@ -504,84 +525,120 @@ public class DbLoader
         }
     }    
 
+    void test()
+    	throws Exception
+    {
+    	String methodName = "foo";
+        DbHandle h = dbManager.open();
+        SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDb.JOB_TABLE + " limit 5000");
+        //SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDb.RES_TABLE + " limit 5000");
+        //SimpleStatement s = new SimpleStatement("SELECT * from " + HistoryManagerDbB.SVC_TABLE + " limit 5000");
+        logger.info(methodName, null, "Fetch size", s.getFetchSize());
+        s.setFetchSize(100);
+        long now = System.currentTimeMillis();
+
+        try {
+            int counter = 0;
+            int nbytes = 0;
+            ResultSet rs = h.execute(s);
+            for ( Row r : rs ) {
+                counter++;
+                ByteBuffer b = r.getBytes("work");
+                nbytes += b.array().length;
+                logger.info(methodName, null, "found", r.getLong("ducc_dbid"), "of type", r.getString("type"));
+            }
+            
+            logger.info(methodName, null, "Found", counter, "results. Total bytes", nbytes);
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+        logger.info(methodName, null, "Time to select:", System.currentTimeMillis() - now);
+    }
+
+
     void run()
     	throws Exception
     {
     	String methodName = "run";
         long now = System.currentTimeMillis();
 
-        DbManager dbm = new DbManager(state_url, logger);
-        if ( dbm.checkForDatabase() ) {
-            dbm.init();
-            dbm.drop();
-            dbm.shutdown();
-        }
+        if ( false ) {
+            try {
+                dbManager = new DbManager(state_url, logger);
+                dbManager.init();
+                
+                test();
+                if ( true ) return;
+            } finally {
+                dbManager.shutdown();
+            }
+            return;
 
-        DbCreate cr = new DbCreate(state_url, logger);
-        if ( state_url.startsWith("plocal") ) {
-            cr.createPlocalDatabase();
-        } else {
-            cr.createDatabase();
         }
 
-        logger.info(methodName, null, "storage.useWAL", System.getProperty("storage.useWAL"));
-        logger.info(methodName, null, "tx.useLog", System.getProperty("tx.useLog"));
-        if ( true ) {
-            try {
+        dbManager = new DbManager(state_url, logger);
+        dbManager.init();
 
-                OGlobalConfiguration.USE_WAL.setValue(false);
-                // OGlobalConfiguration.USE_LOG.setValue(false);
+//        DbCreate cr = new DbCreate(state_url, logger);
+//        if ( state_url.startsWith("plocal") ) {
+//            cr.createPlocalDatabase();
+//        } else {
+//            cr.createDatabase();
+//        }
+        
 
-                OGlobalConfiguration.dumpConfiguration(System.out);
 
-                hmd = new HistoryManagerDb(logger);
+        if ( true ) {
+            try {
 
+                hmd = new HistoryManagerDb();
+                hmd.init(logger, dbManager);
+                
                 long nowt = System.currentTimeMillis();
                 if ( docheckpoint ) loadCheckpoint();
                 logger.info(methodName, null, "***** Time to load checkpoint A ****", System.currentTimeMillis() - nowt);
 
-                OGlobalConfiguration.dumpConfiguration(System.out);
-
-
                 // ---------- Load job history
                 nowt = System.currentTimeMillis();
                 if ( dojobs ) loadJobs();            
-                logger.info(methodName, null, "**** Time to load jobs**** ", System.currentTimeMillis() - nowt);
+                logger.info(methodName, null, "**** Time to load jobs**** ", System.currentTimeMillis() - nowt, "Total bytes loaded:", jobBytes);
 
                 // ---------- Load reservation history
                 nowt = System.currentTimeMillis();
                 if ( doreservations ) loadReservations();                         
-                logger.info(methodName, null, "**** Time to load reservations ****", System.currentTimeMillis() - nowt);
+                logger.info(methodName, null, "**** Time to load reservations ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", resBytes);
 
 
                 // ---------- Load service isntance and AP history
                 nowt = System.currentTimeMillis();
                 if ( doservices ) loadServices();
-                logger.info(methodName, null, "**** Time to load service instances ****", System.currentTimeMillis() - nowt);
+                logger.info(methodName, null, "**** Time to load services instances ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcBytes);
+
 
                 // ---------- Load service registry
+                long totalSvcBytes = 0;
                 if ( doregistry ) {
                     nowt = System.currentTimeMillis();
                     ssd = new StateServicesDb();
-                    ssd.init(logger);
+                    ssd.init(logger,dbManager);
                     loadServiceRegistry(serviceRegistry, false);
-                    try {
-                        ssd.shutdown(); 
-                    } catch ( Exception e ) {
-                        e.printStackTrace();
-                    }
-                    logger.info(methodName, null, "**** Time to load Service registry ****", System.currentTimeMillis() - nowt);
+                    logger.info(methodName, null, "**** Time to load Service registry ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcRegBytes);
+                    totalSvcBytes = svcRegBytes;
                     
                     // ---------- Load service registry history
+                    svcRegBytes = 0;
                     nowt = System.currentTimeMillis();
-                    ssd = new StateServicesDb();
-                    ssd.init(logger);
                     loadServiceRegistry(serviceRegistryHistory, true);                          
-                    logger.info(methodName, null, "**** Time to load Service history ****", System.currentTimeMillis() - nowt);
+                    logger.info(methodName, null, "**** Time to load Service history ****", System.currentTimeMillis() - nowt, "Total bytes loaded:", svcRegBytes);
+                    totalSvcBytes = svcRegBytes;
+
+                    logger.info(methodName, null, "**** Skipped services:", skippedServices);
+                    // don't shutdown the ssm.  we'll close the db in the 'finally' below
                 }
 
                 nowt = System.currentTimeMillis();
-                logger.info(methodName, null, "**** Total load time ****", System.currentTimeMillis() - now);
+                logger.info(methodName, null, "**** Total load time ****", System.currentTimeMillis() - now, "Total bytes loaded:", (jobBytes + resBytes + svcBytes + totalSvcBytes));
 
                 if ( docheckpoint ) loadCheckpoint();
                 logger.info(methodName, null, "**** Time to reload checkpoint B ****", System.currentTimeMillis() - nowt);
@@ -590,8 +647,7 @@ public class DbLoader
             	logger.error(methodName, null, e);
 
             } finally {
-                if ( hmd != null ) hmd.shutdown();
-                if ( ssd != null ) ssd.shutdown();
+                if ( dbManager!= null ) dbManager.shutdown();
             }
         }
 
@@ -627,12 +683,17 @@ public class DbLoader
     class JobLoader
         implements Runnable
     {
+        PreparedStatement statement = null;
         BlockingQueue<File> queue;
         List<Long> ids;
         JobLoader(BlockingQueue<File> queue, List<Long> ids)
+                throws Exception
         {
             this.queue = queue;
             this.ids = ids;
+
+            DbHandle h = dbManager.open();
+            statement = h.prepare("INSERT INTO " + HistoryManagerDb.JOB_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");            
         }
 
         public void run()
@@ -640,22 +701,44 @@ public class DbLoader
             String methodName = "JobLoader.run";
             while ( true ) {
             	File  f = null;
-                IDuccWorkJob job = null;
+                long nbytes = 0;
+                long duccid = 0;
+                DuccId did = null;
                 try {
                     f = queue.take();
-
                     FileInputStream fis = null;
                     ObjectInputStream in = null;
 
                     try {
                         long now = System.currentTimeMillis();
+                        
+                        String s = f.getName();
+                        int ndx = s.indexOf(".");
+                        duccid = Long.parseLong(s.substring(0, ndx));
+                        did = new DuccId(duccid);
+                        nbytes = f.length();
+                        if ( nbytes > 16*1024*1024) {
+                            logger.warn(methodName, did, "Skipping outsized job", duccid, "length=", nbytes);
+                            nbytes = 0;
+                            continue;
+                        }
+
+                        byte[] buf = new byte[(int)nbytes];
                         fis = new FileInputStream(f);
-                        in = new ObjectInputStream(fis);
-                        job =  (IDuccWorkJob) in.readObject();
-                        logger.info(methodName, job.getDuccId(), "Time to read job:", System.currentTimeMillis() - now);
-                        hmd.saveJobUnsafe(job);
+                        fis.read(buf);
+
+                        ByteBuffer bb = ByteBuffer.wrap(buf);
+                        logger.info(methodName, did, "Time to read job:", System.currentTimeMillis() - now+" MS", "bytes:", nbytes);
+                        logger.info(methodName, did, "Job", duccid, "Store CQL:", statement.getQueryString());
+                        long now1 = System.currentTimeMillis();
+                        BoundStatement boundStatement = new BoundStatement(statement);
+                        BoundStatement bound = boundStatement.bind(duccid, "job", true, bb);
+                        DbHandle h = dbManager.open();
+
+                        h.execute(bound);
+                        logger.info(methodName, did, "Time to store job", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
                     } catch(Exception e) {
-                        logger.info(methodName, null, e);
+                        logger.info(methodName, did, e);
                     } finally {
                         closeStream(in);
                         closeStream(fis);
@@ -666,11 +749,13 @@ public class DbLoader
                     return;
                 } catch(Exception e) {
                     logger.info(methodName, null, e);
-                } 
+                } finally {
 
-                synchronized(ids) {
-                    if ( job != null ) {
-                        ids.add(job.getDuccId().getFriendly());
+                    synchronized(ids) {
+                        if ( nbytes > 0 ) {
+                            ids.add(duccid);
+                            jobBytes += nbytes;
+                        }
                     }
                 }
             }
@@ -681,20 +766,26 @@ public class DbLoader
     class ServiceLoader
         implements Runnable
     {
+        PreparedStatement statement = null;
         BlockingQueue<File> queue;
         List<Long> ids;
         ServiceLoader(BlockingQueue<File> queue, List<Long> ids)
+        	throws Exception
         {
             this.queue = queue;
             this.ids = ids;
+            DbHandle h = dbManager.open();
+            statement = h.prepare("INSERT INTO " + HistoryManagerDb.SVC_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");            
         }
 
         public void run()
         {
             String methodName = "ServiceLoader.run";
             while ( true ) {
-            	IDuccWorkService svc = null;
                 File f = null;
+                long nbytes = 0;
+                long duccid = 0;
+                DuccId did = null;
                 try {
                     f = queue.take();
                     FileInputStream fis = null;
@@ -702,30 +793,50 @@ public class DbLoader
                     
                     try {
                         long now = System.currentTimeMillis();
+                        
+                        String s = f.getName();
+                        int ndx = s.indexOf(".");
+                        duccid = Long.parseLong(s.substring(0, ndx));
+                        did = new DuccId(duccid);
+                        nbytes = f.length();
+                        if ( nbytes > 16*1024*1024) {
+                            logger.warn(methodName, did, "Skipping outsized service", duccid, "length=", nbytes);
+                            nbytes = 0;
+                            continue;
+                        }
+
+                        byte[] buf = new byte[(int)nbytes];
                         fis = new FileInputStream(f);
-                        in = new ObjectInputStream(fis);
-                        svc =  (IDuccWorkService) in.readObject();
-                        logger.info(methodName, svc.getDuccId(), "Time to read service:", System.currentTimeMillis() - now);                        
-                        hmd.saveServiceUnsafe(svc);
+                        fis.read(buf);
+
+                        ByteBuffer bb = ByteBuffer.wrap(buf);
+                        logger.info(methodName, did, "Time to read service", duccid, ":", System.currentTimeMillis() - now + " MS", "bytes:", nbytes);
+                        logger.info(methodName, did, "Service", duccid, "Store CQL:", statement.getQueryString());
+                        long now1 = System.currentTimeMillis();
+                        BoundStatement boundStatement = new BoundStatement(statement);
+                        BoundStatement bound = boundStatement.bind(duccid, "service", true, bb);
+                        DbHandle h = dbManager.open();
+
+                        h.execute(bound);
+                        logger.info(methodName, did, "Time to store service", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
                     } catch(Exception e) {
-                        logger.info(methodName, null, "Error reading or saving service:", f);
-                        logger.info(methodName, null, e);
+                        logger.info(methodName, did, e);
                     } finally {
                         closeStream(in);
                         closeStream(fis);
-                        counter.getAndDecrement(); 
+                        counter.getAndDecrement();
                     }
-
                 } catch ( InterruptedException e ) {
                     return;
-                } catch ( Exception e ){
-                    logger.info(methodName, null, "Error reading or saving service:", f);
-                    logger.info(methodName, null, e);
-                }
+                } catch(Exception e) {
+                    logger.info(methodName, did, e);
+                } finally {
 
-                synchronized(ids) {
-                    if ( svc != null ) {
-                        ids.add(svc.getDuccId().getFriendly());
+                    synchronized(ids) {
+                        if ( nbytes > 0 ) {
+                            ids.add(duccid);
+                            svcBytes += nbytes;
+                        }
                     }
                 }
             }
@@ -735,20 +846,26 @@ public class DbLoader
     class ReservationLoader
         implements Runnable
     {
+        PreparedStatement statement = null;
         BlockingQueue<File> queue;
         List<Long> ids;
         ReservationLoader(BlockingQueue<File> queue, List<Long> ids)
+        	throws Exception
         {
             this.queue = queue;
-            this.ids = ids;
+            this.ids = ids;            
+            DbHandle h = dbManager.open();
+            statement = h.prepare("INSERT INTO " + HistoryManagerDb.RES_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?);");
         }
 
         public void run()
         {
             String methodName = "ReservationLoader.run";
             while ( true ) {
-            	IDuccWorkReservation res = null;
                 File f = null;
+                long nbytes = 0;
+                long duccid = 0;
+                DuccId did = null;
                 try {
                     f = queue.take();
                     FileInputStream fis = null;
@@ -756,13 +873,35 @@ public class DbLoader
 
                     try {
                         long now = System.currentTimeMillis();
+
+                        String s = f.getName();
+                        int ndx = s.indexOf(".");
+                        duccid = Long.parseLong(s.substring(0, ndx));
+                        did = new DuccId(duccid);
+                        nbytes = f.length();
+                        if ( nbytes > 16*1024*1024) {
+                            logger.warn(methodName, did, "Skipping outsized reservation", duccid, "length=", nbytes);
+                            nbytes = 0;
+                            continue;
+                        }
+
+                        byte[] buf = new byte[(int)nbytes];
                         fis = new FileInputStream(f);
-                        in = new ObjectInputStream(fis);
-                        res =  (IDuccWorkReservation) in.readObject();
-                        logger.info(methodName, res.getDuccId(), "Time to read reservation:", System.currentTimeMillis() - now);
-                        hmd.saveReservationUnsafe(res);
+                        fis.read(buf);
+
+                        ByteBuffer bb = ByteBuffer.wrap(buf);
+                        logger.info(methodName, did, "Time to read reservation", duccid, ":", System.currentTimeMillis() - now+" MS", "bytes:", nbytes);
+                        logger.info(methodName, did, "Reservation", duccid, "Store CQL:", statement.getQueryString());
+                        long now1 = System.currentTimeMillis();
+                        BoundStatement boundStatement = new BoundStatement(statement);
+                        BoundStatement bound = boundStatement.bind(duccid, "reservation", true, bb);
+                        DbHandle h = dbManager.open();
+
+                        h.execute(bound);
+                        logger.info(methodName, did, "Time to store reservation", duccid, "- Database update:", (System.currentTimeMillis() - now1) + " MS", "Total save time:", (System.currentTimeMillis() - now) + " MS");
+
                     } catch(Exception e) {
-                        logger.info(methodName, null, e);
+                        logger.info(methodName, did, e);
                     } finally {
                         closeStream(in);
                         closeStream(fis);
@@ -774,11 +913,12 @@ public class DbLoader
                     return;
                 } catch ( Exception e ){
                     logger.info(methodName, null, e);
-                }
-
-                synchronized(ids) {
-                    if ( res != null ) {
-                        ids.add(res.getDuccId().getFriendly());
+                } finally {
+                    synchronized(ids) {
+                        if ( nbytes > 0 ) {
+                            ids.add(duccid);
+                            resBytes += nbytes;
+                        }
                     }
                 }
             }
@@ -803,6 +943,7 @@ public class DbLoader
             while ( true ) {
                 Pair<String, Boolean> p = null;
                 String id = null;
+                int nbytes = 0;
                 boolean isHistory;
                 try {
                     logger.info(methodName, null, "About to take (service id).");
@@ -822,40 +963,41 @@ public class DbLoader
                     String svc_name = id + ".svc";
                     String meta_name = id + ".meta";
 
-                    svc_in = new FileInputStream(svc_name);
-                    meta_in = new FileInputStream(meta_name);
+                    File svc_file = new File(svc_name);
+                    File meta_file = new File(meta_name);
+                    nbytes += (svc_file.length() + meta_file.length());
+                    svc_in = new FileInputStream(svc_file);
+                    meta_in = new FileInputStream(meta_file);
                     svc_props.load(svc_in);
                     meta_props.load(meta_in);
-                    if ( isHistory ) {
-                        meta_props.setProperty(archive_key, archive_flag);
-                    }
 
-                    String sid = meta_props.getProperty(IStateServices.SvcProps.numeric_id.pname());
+                    String sid = meta_props.getProperty(IStateServices.SvcMetaProps.numeric_id.pname());
                     if ( sid == null ) {
-                        logger.warn(methodName, null, "Cannot find service id in meta file for", id);
+                        logger.error(methodName, null, "Cannot find service id in meta file for", id, "skipping load.");
+                        skippedServices.getAndIncrement();
                     } else {
                         if ( id.indexOf(sid) < 0 ) {
                             // must do index of because the 'id' is a full path, not just the numeric id.  so we
                             // are satisfied with making sure the id is in the path.
                             throw new IllegalStateException("Service id and internal id do not match.");
                         }
-                        DuccId did = new DuccId(Long.parseLong(sid));
-                        
+                        DuccId did = new DuccId(Long.parseLong(sid));                        
+                        ssd.storeProperties(did, svc_props, meta_props);     // always stores as not history
                         if ( isHistory ) {
-                            ssd.storePropertiesUnsafe(did, svc_props, meta_props, DbCategory.History);
-                        } else {
-                            ssd.storePropertiesUnsafe(did, svc_props, meta_props, DbCategory.SmReg);
+                            ssd.moveToHistory(did, svc_props, meta_props);       // updates a single column in each
                         }
+
                         synchronized(ids) {
                             ids.add(did.getFriendly());
+                            svcRegBytes += nbytes;
                         }
                     }
-                    counter.getAndDecrement();
                 } catch(Exception e) {
                     logger.info(methodName, null, e);
                 } finally {
                     closeStream(svc_in);
                     closeStream(meta_in);
+                    counter.getAndDecrement();
                 }
 
             }