You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/10/28 19:12:54 UTC
svn commit: r1711088 [4/4] - in /uima/sandbox/uima-ducc/trunk:
src/main/admin/ src/main/assembly/ src/main/resources/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/persistence/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/per...
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java Wed Oct 28 18:12:53 2015
@@ -19,20 +19,23 @@
package org.apache.uima.ducc.database;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.UUID;
+import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.persistence.services.IStateServices;
import org.apache.uima.ducc.common.persistence.services.StateServicesDirectory;
import org.apache.uima.ducc.common.persistence.services.StateServicesSet;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.database.DbConstants.DbCategory;
-import org.apache.uima.ducc.database.DbConstants.DbVertex;
-import com.tinkerpop.blueprints.impls.orient.OrientVertex;
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.SimpleStatement;
public class StateServicesDb
implements IStateServices
@@ -40,25 +43,45 @@ public class StateServicesDb
private DuccLogger logger = null;
private DbManager dbManager;
+ private static final String SVC_TABLE = "ducc." + SvcRegProps.TABLE_NAME.pname();
+ private static final String META_TABLE = "ducc." + SvcMetaProps.TABLE_NAME.pname();
+ // (My) db conventions are that everything must follow the conventions of IDbProperty. SM
+ // uses properties directly. Maybe we'll change this some time. For now, we need to efficiently
+ // convert a Properties object into a Map keyed on IDbProperty; hence these two convenient maps
+ // from string to property.
+ Map<String, SvcRegProps> s2regProps = new HashMap<String, SvcRegProps>();
+ Map<String, SvcMetaProps> s2metaProps = new HashMap<String, SvcMetaProps>();
public StateServicesDb()
{
+ for ( SvcRegProps p : SvcRegProps.values() ) {
+ s2regProps.put(p.pname(), p);
+ }
+ for ( SvcMetaProps p : SvcMetaProps.values() ) {
+ s2metaProps.put(p.pname(), p);
+ }
}
- private boolean init(String dburl)
+ private boolean init(String dburl, DbManager dbm)
throws Exception
{
+ String methodName = "init";
// log4j issue - the caller must pass in a logger configured correctly for the component
// to which it belongs or it won't get sent to the right appender.
- boolean ret = false;
- try {
- dbManager = new DbManager(dburl, logger);
- dbManager.init();
- ret = true;
- } catch ( Exception e ) {
- // logger.error("StateServicesDb", null, "Cannot open the service database:", e);
- throw e;
+ boolean ret = true;
+ if ( dbm != null ) {
+ this.dbManager = dbm;
+ } else {
+ try {
+ dbManager = new DbManager(dburl, logger);
+ dbManager.init();
+ ret = true;
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Cannot open the state database:", e);
+ throw e;
+ }
}
+
return ret;
}
@@ -67,60 +90,59 @@ public class StateServicesDb
{
this.logger = logger;
String stateUrl = System.getProperty("ducc.state.database.url");
- return init(stateUrl);
+ return init(stateUrl, null);
}
- /**
- * Helper for restoring service registrations. This looks in the non-history part of the DB.
- */
- private List<Long> getListOfType(DbVertex type)
+ // package only, for the loader
+ boolean init(DuccLogger logger, DbManager dbManager)
+ throws Exception
{
- String methodName = "getSvcList";
-
- if ( dbManager == null ) {
- logger.error(methodName, null, "Service database is not initialized.");
- return new ArrayList<Long>(); // avoid NPE in caller
- }
-
- List<Long> ret = null;
- DbHandle h = null;
- try {
- h = dbManager.open();
- ret = h.listObjectsOfType(type, DbCategory.SmReg);
- } catch (Throwable e) {
- logger.error(methodName, null, e);
- } finally {
- if ( h != null ) h.close();
- }
-
- return ret;
+ this.logger = logger;
+ String stateUrl = System.getProperty("ducc.state.database.url");
+ return init(stateUrl, dbManager);
}
- /**
- * Return a list of service property file names. Must query the db every time.
- */
- public List<Long> getSvcList()
- {
- return getListOfType(DbVertex.ServiceReg);
- }
-
- /**
- * Return a list of sersvice meta file names. Must query the db every time.
- */
- public List<Long> getMetaList()
+ private Map<Long, DuccProperties> getProperties(String tableid, IDbProperty[] props, boolean active)
+ throws Exception
{
- return getListOfType(DbVertex.ServiceMeta);
+ String methodName = "getProperties";
+ Map<Long, DuccProperties> ret = new HashMap<Long, DuccProperties>();
+
+ SimpleStatement s = new SimpleStatement( "SELECT * FROM " + tableid + " WHERE is_archived=" + active);
+ s.setFetchSize(100);
+ DbHandle h = dbManager.open();
+ ResultSet rs = h.execute(s);
+ for ( Row r: rs ) {
+ long id = Long.parseLong(r.getString("numeric_id"));
+ DuccId did = new DuccId(id);
+ logger.debug(methodName, did, "Found properties in table", tableid);
+ DuccProperties dp = new DuccProperties();
+ for ( IDbProperty p : props ) {
+ String val = null;
+ if ( logger.isTrace() ) {
+ logger.info(methodName, null, "Fetching", p.pname(), "from", tableid);
+ }
+ if ( !(p.isPrivate() || p.isMeta()) ) { // collect non-private / non-meta properties into dp
+ val = r.getString(p.columnName());
+ if ( val != null ) {
+ dp.put(p.pname(), val);
+ }
+ }
+ if ( p.isPrimaryKey() ) { // once we find the key we set the dp int return val
+ String k = null;
+ if ( val == null ) {
+ k = r.getString(p.columnName());
+ } else {
+ k = val;
+ }
+ ret.put(Long.parseLong(k), dp);
+ }
+ }
+ }
+ return ret;
}
- /**
- * This is adapted from the file-based version and as such, perhaps should be named better.
- *
- * This reads the entire (live, non-history) service registration into an object called
- * StateServicesDirectory. This in turn contains a map of StateServicesSet objects. Each
- * StateServiceSet contains two properties files, one for the submitted properties, and one
- * for the service meta properties (SM state).
- */
- public StateServicesDirectory getStateServicesDirectory()
+ StateServicesDirectory fetchServices(boolean isArchived) // pkg protection, for db utilities to use
throws Exception
{
String methodName = "getStateServicesDirectory";
@@ -133,30 +155,89 @@ public class StateServicesDb
return ret; // avoid NPE in caller
}
- DbHandle h = dbManager.openNoTx();
+ DbHandle h = null;
try {
- Map<Long, Properties> svcset = h.getPropertiesForTypeSel(DbVertex.ServiceReg , DbCategory.SmReg);
- Map<Long, Properties> metaset = h.getPropertiesForTypeSel(DbVertex.ServiceMeta, DbCategory.SmReg);
-
+ h = dbManager.open();
+ Map<Long, DuccProperties> svcset = getProperties(SVC_TABLE, IStateServices.SvcRegProps.values(), isArchived);
+ Map<Long, DuccProperties> metaset = getProperties(META_TABLE, IStateServices.SvcMetaProps.values(), isArchived);
+
for ( Long k : svcset.keySet() ) {
logger.trace(methodName, null, "Handling key", k);
- DuccProperties sp = new DuccProperties(svcset.get(k));
- DuccProperties mp = new DuccProperties(metaset.get(k));
+ DuccProperties sp = svcset.get(k);
+ DuccProperties mp = metaset.get(k);
StateServicesSet sss = new StateServicesSet();
sss.put(svc, sp);
sss.put(meta, mp);
ret.put(k, sss);
}
- } finally {
- if ( h != null ) h.close();
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Cannot read service directory:", e);
}
-
+
logger.info(methodName, null, "Time to read service registy", System.currentTimeMillis() - now);
return ret;
}
/**
+ * This is adapted from the file-based version and as such, perhaps should be named better.
+ *
+ * This reads the entire (live, non-history) service registration into an object called
+ * StateServicesDirectory. This in turn contains a map of StateServicesSet objects. Each
+ * StateServiceSet contains two properties files, one for the submitted properties, and one
+ * for the service meta properties (SM state).
+ */
+ public StateServicesDirectory getStateServicesDirectory()
+ throws Exception
+ {
+ return fetchServices(false); // get the non-archived stsuff
+ }
+
+ Map<IDbProperty, Object> mkMap(DuccId did, String table, Map<String, ? extends IDbProperty> converter, Properties props)
+ {
+ String methodName = "mkMap";
+
+ // hmmm - some old registrations didn't seem to get converted - we'll do it now
+ String kk = "process_failures_limit";
+ if ( props.containsKey(kk) ) {
+ Object v = props.remove(kk);
+ kk = SvcRegProps.instance_failures_limit.columnName();
+ props.put(kk, v);
+ }
+ kk = "process_DD";
+ if ( props.containsKey(kk) ) {
+ Object v = props.remove(kk);
+ kk = SvcRegProps.process_dd.columnName();
+ props.put(kk, v);
+ }
+ kk = "process_classpath";
+ if ( props.containsKey(kk) ) {
+ Object v = props.remove(kk);
+ kk = SvcRegProps.classpath.columnName();
+ props.put(kk, v);
+ }
+ kk = "jvm_args";
+ if ( props.containsKey(kk) ) {
+ Object v = props.remove(kk);
+ kk = SvcRegProps.process_jvm_args.columnName();
+ props.put(kk, v);
+ }
+
+ Map<IDbProperty, Object> ret = new HashMap<IDbProperty, Object>();
+ for ( Object k : props.keySet() ) {
+ IDbProperty p = converter.get((String) k);
+ if (p == null ) {
+ logger.error(methodName, did, "Unrecognized property", k, "for table", table);
+ continue;
+ }
+ String val = (String) props.get(k);
+ val = val.replace("'", "''"); // must escape single quotes - this is how CQL and SQL do it
+ ret.put(p, val);
+ }
+ return ret;
+ }
+
+ /**
* Save the src and meta propeties into the non-history part of the DB.
*
* @param serviceID The SM-assigned duccid for the service registration.
@@ -169,146 +250,203 @@ public class StateServicesDb
* equivalent in the Java interface? If so, we should modify this to use it
* and can then eliminate the 'safe' flag.
*/
- boolean storePropertiesInternal (DuccId serviceId, Properties svc_props, Properties meta_props, boolean safe, DbCategory category)
+ public boolean storeProperties (DuccId serviceId, Properties svc_props, Properties meta_props)
{
String methodName = "storePropertiesInternal";
DbHandle h = null;
+ boolean ret = false;
+ long now = System.currentTimeMillis();
try {
- if ( safe ) {
- h = dbManager.open();
- } else {
- h = dbManager.openNoTx();
- }
-
- if ( meta_props.containsKey("meta_dbid")) return false; // if it's assigned, it came from the db so we know it's already there
-
- Long id = serviceId.getFriendly();
- OrientVertex ov_svc = h.createProperties(DbConstants.DUCCID, id, DbVertex.ServiceReg , category, svc_props);
- OrientVertex ov_meta = h.createProperties(DbConstants.DUCCID, id, DbVertex.ServiceMeta, category, meta_props);
-
- Object dbid = ov_svc.getId();
- meta_props.put("svc_dbid", dbid);
- ov_meta.setProperty("svc_dbid", dbid);
-
- dbid = ov_meta.getId();
- meta_props.put("meta_dbid", dbid);
- ov_meta.setProperty("meta_dbid", dbid);
-
- h.commit();
- return true;
+ h = dbManager.open();
+ long numeric = serviceId.getFriendly();
+ UUID uuid = serviceId.getUUID();
+ // the utils want Maps of IDbProperty which give hints how to form the cql; properties don't
+ Map<IDbProperty, Object> svc_map = mkMap(serviceId, SVC_TABLE, s2regProps, svc_props);
+ Map<IDbProperty, Object> meta_map = mkMap(serviceId, META_TABLE, s2metaProps, meta_props);
+ svc_map.put(SvcRegProps.numeric_id, numeric);
+ svc_map.put(SvcRegProps.uuid, uuid.toString());
+
+ svc_map.put(SvcRegProps.is_archived, "false"); // never archived when first put into db
+ meta_map.put(SvcMetaProps.is_archived, "false");
+
+ // in Cassandra, we use "batch" mode to get multiple inserts managed as a single transaction
+ String cql = "";
+ StringBuffer buf = new StringBuffer("BEGIN BATCH ");
+ cql = DbUtil.mkInsert(SVC_TABLE, svc_map);
+ buf.append(cql);
+ buf.append("; ");
+ cql = DbUtil.mkInsert(META_TABLE, meta_map);
+ buf.append(cql);
+ buf.append("; ");
+ buf.append("APPLY BATCH");
+ h.execute(buf.toString());
+
} catch ( Exception e ) {
- logger.error(methodName, serviceId, "ROLLBACK: ", e);
- if ( h != null ) h.rollback();
- return false;
+ logger.error(methodName, null, "Error storing props for new registration:", e);
+ ret = false;
} finally {
- if ( h != null ) h.close();
+ logger.info(methodName, serviceId, "Time to create (2) proeprties files:", System.currentTimeMillis() - now);
}
+ return ret;
}
-
- /**
- * Save the props into the database, don't check to see if they're there already. Used by the
- * loader for converting old registries to the db.
- */
- public boolean storePropertiesUnsafe(DuccId serviceId, Properties svc_props, Properties meta_props, DbCategory category)
- {
- return storePropertiesInternal(serviceId, svc_props, meta_props, false, category);
- }
-
- /**
- * Save the props into the db. If the object exists don't overwrite it, and return an error.
- * The only non-error return is if the object doesn't already exist, and it is safely committed.
- *
- * This is used by the SM on initial service registration only.
- */
- public boolean storeProperties(DuccId serviceId, Properties svc_props, Properties meta_props)
- {
- return storePropertiesInternal(serviceId, svc_props, meta_props, true, DbCategory.SmReg);
- }
-
+
/**
* The registration is removed, move it to the history part of the DB.
*/
- public void moveToHistory(DuccId serviceId, Properties job_props, Properties meta_props)
+ public boolean moveToHistory(DuccId serviceId, Properties job_props, Properties meta_props)
{
- // All we need to do is re-sync the final properties, and be sure to set DUCC_HISTORY to false
+ // All we need to do is re-sync the final properties, and be sure to set DUCC_HISTORY to false
String methodName = "moveToHistory";
DbHandle h = null;
try {
-
+
h = dbManager.open(); // get new connection from the pool
- Object svc_dbid = meta_props.get("svc_dbid");
- Object meta_dbid = meta_props.get("meta_dbid");
- OrientVertex obj_reg = h.updateProperties(svc_dbid, job_props);
- OrientVertex obj_meta = h.updateProperties(meta_dbid, meta_props);
-
- h.changeCategory(obj_reg, DbCategory.History);
- h.changeCategory(obj_meta, DbCategory.History);
- h.commit();
- // h.syncProperties(job_props, DbVertex.ServiceReg, id, DbCategory.History);
- // h.syncProperties(meta_props, DbVertex.ServiceMeta, id, DbCategory.History);
+ job_props.put(SvcRegProps.is_archived.pname(), "true");
+ meta_props.put(SvcRegProps.is_archived.pname(), "true");
+ StringBuffer buf = new StringBuffer("BEGIN BATCH ");
+ buf.append("UPDATE ");
+ buf.append(SVC_TABLE);
+ buf.append(" SET ");
+ buf.append(SvcRegProps.is_archived.columnName());
+ buf.append("=true WHERE numeric_id='");
+ buf.append(Long.toString(serviceId.getFriendly()));
+ buf.append("';");
+
+ buf.append("UPDATE ");
+ buf.append(META_TABLE);
+ buf.append(" SET ");
+ buf.append(SvcMetaProps.is_archived.columnName());
+ buf.append("=true WHERE numeric_id='");
+ buf.append(Long.toString(serviceId.getFriendly()));
+ buf.append("';");
+
+ buf.append("APPLY BATCH");
+ h.execute(buf.toString());
+
+ return true;
} catch ( Exception e ) {
- logger.error(methodName, serviceId, "ROLLBACK: ", e);
- if ( h != null ) h.rollback();
- } finally {
- if ( h != null ) h.close();
- }
+ logger.error(methodName, serviceId, "Error moving registration to history:", e);
+ return false;
+ }
}
/**
* Helper method to Update the indicated properties file, in the non-history part of the db.
* This is most often called by SM to update the service meta after state changes.
*
- * @param serviceId The SM-assigned DUCC ID for the service registration.
+ * @param keyField - the IDbProperty identifying the row key
+ * @parem table - the String name of the table
+ * @param key - the String value of the key identifying the row
+ * @param converter - either s2regProps or s2metaProps, identifying the String - to - IDbProperty converter
* @param props The properties file to save. Usually it's just the meta but if
- * the service is being modified, it could also be the registration.
- * @param type The type enum, ususally Service or ServiceMeta.
+ * the service is being modified, it could also be the registration being updated.
*/
- private boolean updateProperties(Object dbid, DuccId serviceId, Properties props, DbVertex type)
+ private boolean updateProperties(DuccId serviceId, String table, String key, Map<String, ? extends IDbProperty> converter, Properties props)
{
String methodName = "updatePropeties";
DbHandle h = null;
try {
h = dbManager.open();
- h.updateProperties(dbid, props);
- // h.synchronizeProperties(DbConstants.DUCCID, serviceId.getFriendly(), type, DbCategory.SmReg, props);
- // h.synchronizeProperties(dbid, props);
-
- h.commit();
- // h.syncProperties(props, type, serviceId.getFriendly(), DbCategory.SmReg);
+ Map<IDbProperty, Object> map = mkMap(serviceId, table, converter, props);
+ Object[] updates = new Object[props.size()*2];
+ int i = 0;
+ for ( IDbProperty k : map.keySet() ) {
+ logger.info(methodName, null, "Updating", k.columnName(), "with", map.get(k));
+ updates[i++] = k;
+ updates[i++] = map.get(k);
+ }
+
+ h.updateProperties(table, key, updates);
return true;
} catch ( Exception e ) {
- logger.error(methodName, serviceId, "ROLLBACK:", e);
- if ( h != null ) h.rollback();
+ logger.error(methodName, null, "Unable to update properties for service", key, "table", table, ":", e);
return false;
- } finally {
- if ( h != null ) h.close();
- }
+ }
}
/**
* Update the service registration.
*/
- public boolean updateJobProperties(Object dbid, DuccId serviceId, Properties props)
+ public boolean updateJobProperties(DuccId serviceId, Properties props)
{
- return updateProperties(dbid, serviceId, props, DbVertex.ServiceReg);
+ return updateProperties(serviceId, SVC_TABLE, "numeric_id='" + Long.toString(serviceId.getFriendly()) + "'", s2regProps, props);
}
/**
* Update the service meta data.
*/
- public boolean updateMetaProperties(Object dbid, DuccId serviceId, Properties props)
+ public boolean updateMetaProperties(DuccId serviceId, Properties props)
{
- return updateProperties(dbid, serviceId, props, DbVertex.ServiceMeta);
+ return updateProperties(serviceId, META_TABLE, "numeric_id='" + Long.toString(serviceId.getFriendly()) + "'", s2metaProps, props);
}
- /**
- * Close and discard the database connection pool.
- */
- public void shutdown()
+ public void shutdown()
{
dbManager.shutdown();
- }
+ }
+
+ static List<SimpleStatement> mkSchema()
+ throws Exception
+ {
+ List<SimpleStatement> ret = new ArrayList<SimpleStatement>();
+
+
+ StringBuffer buf = new StringBuffer("CREATE TABLE IF NOT EXISTS " + SVC_TABLE + " (");
+ buf.append(DbUtil.mkSchema(SvcRegProps.values()));
+ buf.append(")");
+ ret.add(new SimpleStatement(buf.toString()));
+ ret.add(new SimpleStatement("CREATE INDEX IF NOT EXISTS ON " + SVC_TABLE + "(" + SvcRegProps.is_archived.columnName() + ")"));
+
+ buf = new StringBuffer("CREATE TABLE IF NOT EXISTS " + META_TABLE + " (");
+ buf.append(DbUtil.mkSchema(SvcMetaProps.values()));
+ buf.append(")");
+ ret.add(new SimpleStatement(buf.toString()));
+ ret.add(new SimpleStatement("CREATE INDEX IF NOT EXISTS ON " + META_TABLE + "(" + SvcMetaProps.is_archived.columnName() +")"));
+
+ return ret;
+ }
+
+ // static String[] mkSchemaForReg()
+ // {
+
+ // int size = SvcRegProps.values().length;
+ // String[] ret = new String[size];
+
+ // int ndx = 0;
+
+ // for ( SvcRegProps p: SvcRegProps.values() ) {
+ // String s = p.pname();
+ // s = s + " " + DbUtil.typeToString(p.type());
+ // if ( p.isPrimaryKey() ) {
+ // s = s + " PRIMARY KEY";
+ // }
+ // ret[ndx++] = s;
+ // }
+ // return ret;
+
+ // }
+
+ // static String[] mkSchemaForMeta()
+ // {
+ // int size = SvcMetaProps.values().length;
+ // String[] ret = new String[size];
+
+ // int ndx = 0;
+
+ // for ( SvcMetaProps p: SvcMetaProps.values() ) {
+ // String s = p.pname();
+ // s = s + " " + DbUtil.typeToString(p.type());
+ // if ( p.isPrimaryKey() ) {
+ // s = s + " PRIMARY KEY";
+ // }
+ // ret[ndx++] = s;
+ // }
+
+ // return ret;
+
+ // }
+
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/OrchestratorCheckpoint.java Wed Oct 28 18:12:53 2015
@@ -24,9 +24,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
@@ -249,12 +251,16 @@ public class OrchestratorCheckpoint {
try
{
logger.info(methodName, null, messages.fetchLabel("restoring from")+fileName);
- DuccWorkMap work = new DuccWorkMap();
- ConcurrentHashMap<DuccId, DuccId> processToJob = new ConcurrentHashMap<DuccId, DuccId>();
- Checkpointable checkpointable = new Checkpointable(work, processToJob);
- retVal = saver.restore(work, processToJob);
- orchestratorCommonArea.setCheckpointable(checkpointable);
- logger.info(methodName, null, messages.fetch("restored"));
+ Pair<DuccWorkMap, Map<DuccId, DuccId>> ret = saver.restore();
+ if ( ret.first() != null ) {
+ //Checkpointable checkpointable = new Checkpointable(ret.first(), (ConcurrentHashMap<DuccId, DuccId>) ret.second());
+ Checkpointable checkpointable = new Checkpointable(ret.first(), (ConcurrentHashMap<DuccId, DuccId>) ret.second());
+
+ orchestratorCommonArea.setCheckpointable(checkpointable);
+ logger.info(methodName, null, messages.fetch("restored"));
+ } else {
+ logger.info(methodName, null, "No checkpoint found.");
+ }
}
catch(ClassNotFoundException e)
{
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-parent/pom.xml Wed Oct 28 18:12:53 2015
@@ -134,7 +134,7 @@
<org.apache.camel.version>2.10.1</org.apache.camel.version>
<!-- org.apache.uima.version>2.6.0</org.apache.uima.version Same as uima-as-->
<org.apache.uima.as.version>2.6.0</org.apache.uima.as.version>
- <com.google.guava.version>r09</com.google.guava.version>
+ <com.google.guava.version>14.0.1</com.google.guava.version> <!-- updated from r09 for cassandra -->
<commons.cli.version>1.2</commons.cli.version>
<joda.time.version>2.1</joda.time.version>
<jsch.version>0.1.29</jsch.version>
@@ -146,7 +146,7 @@
<commons.lang.version>2.6</commons.lang.version>
<commons.math.version>2.0</commons.math.version>
<log4j.version>1.2.16</log4j.version>
- <slf4j.version>1.6.4</slf4j.version>
+ <slf4j.version>1.7.6</slf4j.version> <!-- updated from 1.6.4 for cassandra -->
<xmlbeans.version>2.5.0</xmlbeans.version>
<junit.version>4.8.2</junit.version>
<servlet.api.version>2.5</servlet.api.version>
@@ -165,8 +165,16 @@
<jetty.version>7.4.4.v20110707</jetty.version>
<orbit-org-apache-jasper.version>2.1.0.v201110031002</orbit-org-apache-jasper.version>
<servlet-api.version>2.5</servlet-api.version>
- <orientdb.version>2.1.2</orientdb.version>
- <orientdb.studio.version>2.0-M3</orientdb.studio.version>
+
+ <!-- These are all cassandra client -->
+ <cassandra.driver.version>2.1.8</cassandra.driver.version>
+ <netty.version>4.0.27.Final</netty.version>
+ <hdr.version>2.1.4</hdr.version>
+ <metrics.version>3.0.2</metrics.version>
+ <snappy.version>1.0.5</snappy.version>
+ <lz4.version>1.2.0</lz4.version>
+ <hdr.version>2.1.4</hdr.version>
+ <!-- end of cassandra client -->
<http.commons.client.version>4.3.5</http.commons.client.version>
<http.commons.client-cache.version>4.3.5</http.commons.client-cache.version>
@@ -391,6 +399,39 @@ ${uimaDUCCNoticeText}
<version>${joda.time.version}</version>
</dependency>
+ <!-- cassandra RT -->
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ <version>${netty.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>${snappy.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.jpountz.lz4</groupId>
+ <artifactId>lz4</artifactId>
+ <version>${lz4.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hdrhistogram</groupId>
+ <artifactId>HdrHistogram</artifactId>
+ <version>${hdr.version}</version>
+ </dependency>
+
+ <!-- end of cassandra RT -->
+
<dependency>
<groupId>org.apache.uima</groupId>
<artifactId>uimaj-core</artifactId>
@@ -695,7 +736,7 @@ ${uimaDUCCNoticeText}
<!-- Get SVN revision number and include in the manifest.
Disable the check for local changes and the svn update -->
- <plugin>
+ <!-- plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.3</version>
@@ -711,7 +752,7 @@ ${uimaDUCCNoticeText}
<doCheck>false</doCheck>
<doUpdate>false</doUpdate>
</configuration>
- </plugin>
+ </plugin -->
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java Wed Oct 28 18:12:53 2015
@@ -26,7 +26,7 @@ import org.apache.uima.ducc.common.NodeI
import org.apache.uima.ducc.common.admin.event.RmQueriedMachine;
import org.apache.uima.ducc.common.admin.event.RmQueriedShare;
import org.apache.uima.ducc.common.persistence.rm.IRmPersistence;
-import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmPropName;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmProperty;
import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -76,7 +76,6 @@ public class Machine
private HashMap<Share, Share> activeShares = new HashMap<Share, Share>();
private IRmPersistence persistence = null;
- Object dbid = null;
public Machine(Node node)
{
@@ -175,7 +174,7 @@ public class Machine
if ( heartbeats == 0 ) return; // no need to rereset it
try {
logger.info(methodName, null, "Reset heartbeat to 0");
- persistence.setProperty(dbid, id, RmPropName.Heartbeats, 0);
+ persistence.setProperty(id, RmProperty.Heartbeats, 0);
logger.info(methodName, null, "Time to reset heartbeat", System.currentTimeMillis() - now);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot update heartbeat count in database:", e);
@@ -191,23 +190,13 @@ public class Machine
try {
heartbeats = c;
logger.info(methodName, null, "Missed heartbeat count", c);
- persistence.setProperty(dbid, id, RmPropName.Heartbeats, c);
+ persistence.setProperty(id, RmProperty.Heartbeats, c);
logger.info(methodName, null, "Time to record misssed heartbeat", System.currentTimeMillis() - now);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot update heartbeat count in database:", e);
}
}
- Object getDbId()
- {
- return this.dbid;
- }
-
- void setDbId(Object dbid)
- {
- this.dbid = dbid;
- }
-
public NodeIdentity getNodeIdentity()
{
return node.getNodeIdentity();
@@ -328,7 +317,7 @@ public class Machine
activeShares.put(s, s);
shares_left -= s.getShareOrder();
try {
- persistence.setProperties(dbid, id, RmPropName.Assignments.pname(), activeShares.size(), RmPropName.SharesLeft.pname(), shares_left);
+ persistence.setProperties(id, RmProperty.Assignments, activeShares.size(), RmProperty.SharesLeft, shares_left);
logger.info(methodName, null, "Time to assign share in db", System.currentTimeMillis() - now);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot save state; shares_left", shares_left);
@@ -345,7 +334,7 @@ public class Machine
nodepool.removeShare(s);
shares_left += s.getShareOrder();
try {
- persistence.setProperties(dbid, id, RmPropName.Assignments.pname(), activeShares.size(), RmPropName.SharesLeft.pname(), shares_left);
+ persistence.setProperties(id, RmProperty.Assignments, activeShares.size(), RmProperty.SharesLeft, shares_left);
logger.info(methodName, null, "Time to remove share in db", System.currentTimeMillis() - now);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot save state; shares_left", shares_left);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Wed Oct 28 18:12:53 2015
@@ -27,12 +27,11 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.persistence.rm.IRmPersistence;
-import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmPropName;
+import org.apache.uima.ducc.common.persistence.rm.IRmPersistence.RmProperty;
import org.apache.uima.ducc.common.persistence.rm.RmPersistenceFactory;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
@@ -995,35 +994,35 @@ class NodePool
}
- void signalDb(Machine m, RmPropName key, Object value)
+ void signalDb(Machine m, RmProperty key, Object value)
{
String methodName = "signalDb";
try {
- persistence.setProperty(m.getDbId(), m.getNode().getNodeIdentity().getName(), key, value);
+ persistence.setProperty(m.getNode().getNodeIdentity().getName(), key, value);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot update DB property", key, "for machine", m);
}
}
- Properties initDbProperties(Machine m)
+ Map<RmProperty, Object> initDbProperties(Machine m)
{
Node n = m.getNode();
NodeIdentity nid = n.getNodeIdentity();
- Properties props = new Properties();
- props.setProperty(RmPropName.Name.pname(), nid.getName());
- props.setProperty(RmPropName.Ip.pname(), nid.getIp());
- props.setProperty(RmPropName.Nodepool.pname(), id);
- props.put(RmPropName.Quantum.pname(), share_quantum);
+ Map<RmProperty, Object> props = new HashMap<RmProperty, Object>();
+ props.put(RmProperty.Name, nid.getName());
+ props.put(RmProperty.Ip, nid.getIp());
+ props.put(RmProperty.Nodepool, id);
+ props.put(RmProperty.Quantum, share_quantum);
- props.put(RmPropName.Memory.pname() , m.getMemory());
- props.put(RmPropName.ShareOrder.pname() , m.getShareOrder());
- props.put(RmPropName.Blacklisted.pname() , m.isBlacklisted());
+ props.put(RmProperty.Memory , m.getMemory());
+ props.put(RmProperty.ShareOrder , m.getShareOrder());
+ props.put(RmProperty.Blacklisted , m.isBlacklisted());
// init these here, but must be maintained by machine
- props.put(RmPropName.Heartbeats.pname() , 0);
- props.put(RmPropName.SharesLeft.pname() , m.countFreeShares()); // qshares remaining
- props.put(RmPropName.Assignments.pname() , m.countProcesses()); // processes
+ props.put(RmProperty.Heartbeats , 0);
+ props.put(RmProperty.SharesLeft , m.countFreeShares()); // qshares remaining
+ props.put(RmProperty.Assignments , m.countProcesses()); // processes
return props;
}
@@ -1100,7 +1099,7 @@ class NodePool
if ( offlineMachines.containsKey(node) ) { // if it's offline it can't be restored like this.
Machine m = offlineMachines.get(node);
- signalDb(m, RmPropName.Responsive, true);
+ signalDb(m, RmProperty.Responsive, true);
logger.trace(methodName, null, "Node ", m.getId(), " is offline, not activating.");
return m;
}
@@ -1125,7 +1124,7 @@ class NodePool
mlist.put(m.key(), m);
total_shares += order; // UIMA-3939
- signalDb(m, RmPropName.Responsive, true);
+ signalDb(m, RmProperty.Responsive, true);
logger.info(methodName, null, "Nodepool:", id, "Host reactivated ", m.getId(), String.format("shares %2d total %4d:", order, total_shares), m.toString());
return m;
}
@@ -1153,12 +1152,11 @@ class NodePool
String.format("shares %2d total %4d:", order, total_shares), machine.toString());
updated++;
- Properties props = initDbProperties(allMachines.get(key));
- props.put(RmPropName.Responsive.pname(), true);
- props.put(RmPropName.Online.pname(), true);
+ Map<RmProperty, Object> props = initDbProperties(allMachines.get(key));
+ props.put(RmProperty.Responsive, true);
+ props.put(RmProperty.Online, true);
try {
- Object dbid = persistence.createMachine(machine.getId(), props);
- machine.setDbId(dbid);
+ persistence.createMachine(machine.getId(), props);
} catch (Exception e) {
logger.warn(methodName, null, "Cannot write machine to DB:", machine.getId(), e);
}
@@ -1233,7 +1231,7 @@ class NodePool
void nodeLeaves(Machine m)
{
disable(m, unresponsiveMachines);
- signalDb(m, RmPropName.Responsive, false);
+ signalDb(m, RmProperty.Responsive, false);
}
// UIMA-4142
@@ -1274,7 +1272,7 @@ class NodePool
Node key = mm.key();
iter.remove();
offlineMachines.put(key, mm);
- signalDb(m, RmPropName.Online, false);
+ signalDb(m, RmProperty.Online, false);
return "VaryOff: Nodepool " + id + " - Unresponsive machine, marked offline: " + node;
}
}
@@ -1283,7 +1281,7 @@ class NodePool
}
disable(m, offlineMachines);
- signalDb(m, RmPropName.Online, false);
+ signalDb(m, RmProperty.Online, false);
return "VaryOff: " + node + " - OK.";
}
@@ -1302,7 +1300,7 @@ class NodePool
Machine mm = iter.next();
if ( mm.getId().equals(node) ) {
iter.remove();
- signalDb(mm, RmPropName.Online, true);
+ signalDb(mm, RmProperty.Online, true);
return "VaryOn: Nodepool " + id + " - Machine marked online: " + node;
}
}
@@ -1311,7 +1309,7 @@ class NodePool
while ( iter.hasNext() ) {
Machine mm = iter.next();
if ( mm.getId().equals(node) ) {
- signalDb(mm, RmPropName.Online, true);
+ signalDb(mm, RmProperty.Online, true);
return "VaryOn: Nodepool " + id + " - Machine is online but not responsive: " + node;
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java Wed Oct 28 18:12:53 2015
@@ -199,7 +199,7 @@ class ServiceInstance
logger.info(methodName, sset.getId(), "START INSTANCE");
setStopped(false);
- this.user = meta_props.getProperty(IStateServices.SvcProps.user.pname());
+ this.user = meta_props.getProperty(IStateServices.SvcMetaProps.user.pname());
// Simple use of ducc_ling, just submit as the user. The specification will have the working directory
// and classpath needed for the service, handled by the Orchestrator and Job Driver.
@@ -307,10 +307,10 @@ class ServiceInstance
if ( ! started ) {
logger.warn(methodName, sset.getId(), "Request to start service " + sset.getId().toString() + " failed.");
- meta_props.put(IStateServices.SvcProps.submit_error.pname(), submit_buffer.toString());
+ meta_props.put(IStateServices.SvcMetaProps.submit_error.pname(), submit_buffer.toString());
sset.log_errors(stdout_lines, stderr_lines);
} else {
- meta_props.remove(IStateServices.SvcProps.submit_error.pname());
+ meta_props.remove(IStateServices.SvcMetaProps.submit_error.pname());
state = JobState.Received;
}
logger.info(methodName, sset.getId(), "START INSTANCE COMPLETE");
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Wed Oct 28 18:12:53 2015
@@ -38,7 +38,7 @@ import org.apache.uima.ducc.common.crypt
import org.apache.uima.ducc.common.crypto.Crypto.AccessType;
import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.persistence.services.IStateServices;
-import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcProps;
+import org.apache.uima.ducc.common.persistence.services.IStateServices.SvcMetaProps;
import org.apache.uima.ducc.common.persistence.services.StateServicesDirectory;
import org.apache.uima.ducc.common.persistence.services.StateServicesFactory;
import org.apache.uima.ducc.common.persistence.services.StateServicesSet;
@@ -117,7 +117,7 @@ public class ServiceManagerComponent
private String state_file = null;
private DuccProperties sm_props = null;
- private String service_seqno = SvcProps.service_seqno.pname();
+ private String service_seqno = IStateServices.sequenceKey;
private DuccIdFactory idFactory = new DuccIdFactory();
private boolean signature_required = true;
@@ -827,18 +827,18 @@ public class ServiceManagerComponent
props.put(UiOption.LogDirectory.pname(), logdir);
DuccProperties meta = new DuccProperties();
- meta.setProperty(SvcProps.user.pname(), user);
- meta.setProperty(SvcProps.instances.pname(), ""+instances);
- meta.setProperty(SvcProps.endpoint.pname(), endpoint);
- meta.setProperty(SvcProps.numeric_id.pname(), id.toString());
- meta.setProperty(SvcProps.uuid.pname(), id.getUnique());
- meta.setProperty(SvcProps.registration_date_millis.pname(), Long.toString(regdate));
- meta.setProperty(SvcProps.registration_date.pname(), regdate_readable);
+ meta.setProperty(SvcMetaProps.user.pname(), user);
+ meta.setProperty(SvcMetaProps.instances.pname(), ""+instances);
+ meta.setProperty(SvcMetaProps.endpoint.pname(), endpoint);
+ meta.setProperty(SvcMetaProps.numeric_id.pname(), id.toString());
+ meta.setProperty(SvcMetaProps.uuid.pname(), id.getUnique());
+ meta.setProperty(SvcMetaProps.registration_date_millis.pname(), Long.toString(regdate));
+ meta.setProperty(SvcMetaProps.registration_date.pname(), regdate_readable);
if ( autostart == Trinary.True ) {
- meta.setProperty(SvcProps.autostart.pname(), "true");
+ meta.setProperty(SvcMetaProps.autostart.pname(), "true");
} else {
- meta.setProperty(SvcProps.autostart.pname(), "false");
+ meta.setProperty(SvcMetaProps.autostart.pname(), "false");
}
ServiceReplyEvent reply = handler.register(id, props, meta, false);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Wed Oct 28 18:12:53 2015
@@ -82,8 +82,8 @@ public class ServiceSet
// For a registered service, here is my registered id
DuccId id;
HashMap<Long, DuccId> friendly_ids = new HashMap<Long, DuccId>();
- String history_key = IStateServices.SvcProps.work_instances.pname();
- String implementors_key = IStateServices.SvcProps.implementors.pname();
+ String history_key = IStateServices.SvcMetaProps.work_instances.pname();
+ String implementors_key = IStateServices.SvcMetaProps.implementors.pname();
// incoming nodes, for dup checking
List<ServiceSet> predecessors = new ArrayList<ServiceSet>();
@@ -163,8 +163,8 @@ public class ServiceSet
String[] coOwners = null;
- String archive_key = IStateServices.archive_key;
- String archive_flag = IStateServices.archive_flag;
+ String archive_key = "true";
+ String archive_flag = IStateServices.SvcMetaProps.is_archived.columnName();
//
// Constructor for a registered service
@@ -179,19 +179,19 @@ public class ServiceSet
this.service_state = ServiceState.Stopped;
this.linger_time = props.getLongProperty(UiOption.ServiceLinger.pname(), linger_time);
- this.key = meta.getProperty(IStateServices.SvcProps.endpoint.pname());
+ this.key = meta.getProperty(IStateServices.SvcMetaProps.endpoint.pname());
parseEndpoint(key);
- this.user = meta.getProperty(IStateServices.SvcProps.user.pname());
- this.instances = meta.getIntProperty(IStateServices.SvcProps.instances.pname(), 1);
+ this.user = meta.getProperty(IStateServices.SvcMetaProps.user.pname());
+ this.instances = meta.getIntProperty(IStateServices.SvcMetaProps.instances.pname(), 1);
this.registered_instances = this.instances;
- this.autostart = meta.getBooleanProperty(IStateServices.SvcProps.autostart.pname(), false);
- this.ping_only = meta.getBooleanProperty(IStateServices.SvcProps.ping_only.pname(), false);
- this.enabled = meta.getBooleanProperty(IStateServices.SvcProps.enabled.pname(), enabled);
+ this.autostart = meta.getBooleanProperty(IStateServices.SvcMetaProps.autostart.pname(), false);
+ this.ping_only = meta.getBooleanProperty(IStateServices.SvcMetaProps.ping_only.pname(), false);
+ this.enabled = meta.getBooleanProperty(IStateServices.SvcMetaProps.enabled.pname(), enabled);
this.service_class = ServiceClass.Registered;
- this.init_failure_max = props.getIntProperty(IStateServices.SvcProps.instance_init_failures_limit.pname(), init_failure_max);
- this.reference_start = meta.getBooleanProperty(IStateServices.SvcProps.reference.pname(), this.reference_start);
+ this.init_failure_max = props.getIntProperty(IStateServices.SvcRegProps.instance_init_failures_limit.pname(), init_failure_max);
+ this.reference_start = meta.getBooleanProperty(IStateServices.SvcMetaProps.reference.pname(), this.reference_start);
@@ -208,28 +208,28 @@ public class ServiceSet
parseIndependentServices();
- meta_props.remove(IStateServices.SvcProps.references.pname()); // Will get refreshred in upcoming OR state messages
- meta_props.remove(IStateServices.SvcProps.stopped.pname()); // obsolete flag, clean out of older registrations
+ meta_props.remove(IStateServices.SvcMetaProps.references.pname()); // Will get refreshred in upcoming OR state messages
+ meta_props.remove(IStateServices.SvcMetaProps.stopped.pname()); // obsolete flag, clean out of older registrations
- meta_props.put(IStateServices.SvcProps.service_class.pname(), ""+service_class.decode());
- meta_props.put(IStateServices.SvcProps.service_type.pname(), ""+service_type.decode());
- meta_props.put(IStateServices.SvcProps.enabled.pname(), "" + enabled); // may not have been there in the first place
- meta_props.put(IStateServices.SvcProps.service_state.pname(), ""+getState());
- meta_props.put(IStateServices.SvcProps.ping_active.pname(), "false");
- meta_props.put(IStateServices.SvcProps.service_alive.pname(), "false");
- meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "false");
- meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "N/A");
+ meta_props.put(IStateServices.SvcMetaProps.service_class.pname(), ""+service_class.decode());
+ meta_props.put(IStateServices.SvcMetaProps.service_type.pname(), ""+service_type.decode());
+ meta_props.put(IStateServices.SvcMetaProps.enabled.pname(), "" + enabled); // may not have been there in the first place
+ meta_props.put(IStateServices.SvcMetaProps.service_state.pname(), ""+getState());
+ meta_props.put(IStateServices.SvcMetaProps.ping_active.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.service_alive.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.service_healthy.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.service_statistics.pname(), "N/A");
setReferenced(this.reference_start);
- setLastUse(meta_props.getLongProperty(IStateServices.SvcProps.last_use.pname(), 0L));
- setLastPing(meta_props.getLongProperty(IStateServices.SvcProps.last_ping.pname(), 0L));
- setLastRunnable(meta_props.getLongProperty(IStateServices.SvcProps.last_runnable.pname(), 0L));
+ setLastUse(meta_props.getLongProperty(IStateServices.SvcMetaProps.last_use.pname(), 0L));
+ setLastPing(meta_props.getLongProperty(IStateServices.SvcMetaProps.last_ping.pname(), 0L));
+ setLastRunnable(meta_props.getLongProperty(IStateServices.SvcMetaProps.last_runnable.pname(), 0L));
if ( (!job_props.containsKey(UiOption.ProcessExecutable.pname())) && (service_type != ServiceType.UimaAs) ) {
- meta_props.put(IStateServices.SvcProps.ping_only.pname(), "true");
+ meta_props.put(IStateServices.SvcMetaProps.ping_only.pname(), "true");
this.ping_only = true;
} else {
- meta_props.put(IStateServices.SvcProps.ping_only.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.ping_only.pname(), "false");
this.ping_only = false;
}
@@ -439,8 +439,15 @@ public class ServiceSet
Map<Long, ServiceInstance> pendingImplementors = new HashMap<Long, ServiceInstance>();
void bootImplementor(DuccId id, JobState state)
{
+ String methodName = "bootImplementor";
ServiceInstance si = new ServiceInstance(this);
+ if ( ! pending_instances.containsKey(id.getFriendly()) ) {
+ logger.warn(methodName, id, "Incoming Orchestrator state indicates active service instance but it is not in my meta data.");
+ logger.warn(methodName, id, "Instance ignored. This is usally caused by system or database failure.");
+ return;
+ }
+
si.setState(state);
si.setId(id.getFriendly());
si.setStopped(false);
@@ -567,11 +574,11 @@ public class ServiceSet
synchronized void setLastUse(long lu)
{
this.last_use = lu;
- meta_props.put(IStateServices.SvcProps.last_use.pname(), Long.toString(lu));
+ meta_props.put(IStateServices.SvcMetaProps.last_use.pname(), Long.toString(lu));
if ( last_use == 0 ) {
- meta_props.put(IStateServices.SvcProps.last_use_readable.pname(), "Unknown");
+ meta_props.put(IStateServices.SvcMetaProps.last_use_readable.pname(), "Unknown");
} else {
- meta_props.put(IStateServices.SvcProps.last_use_readable.pname(), (new Date(lu)).toString());
+ meta_props.put(IStateServices.SvcMetaProps.last_use_readable.pname(), (new Date(lu)).toString());
}
}
@@ -579,11 +586,11 @@ public class ServiceSet
synchronized void setLastPing(long lp)
{
this.last_ping = lp;
- meta_props.put(IStateServices.SvcProps.last_ping.pname(), Long.toString(lp));
+ meta_props.put(IStateServices.SvcMetaProps.last_ping.pname(), Long.toString(lp));
if ( last_ping == 0 ) {
- meta_props.put(IStateServices.SvcProps.last_ping_readable.pname(), "Unknown");
+ meta_props.put(IStateServices.SvcMetaProps.last_ping_readable.pname(), "Unknown");
} else {
- meta_props.put(IStateServices.SvcProps.last_ping_readable.pname(), (new Date(lp)).toString());
+ meta_props.put(IStateServices.SvcMetaProps.last_ping_readable.pname(), (new Date(lp)).toString());
}
}
@@ -591,11 +598,11 @@ public class ServiceSet
synchronized void setLastRunnable(long lr)
{
this.last_runnable = lr;
- meta_props.put(IStateServices.SvcProps.last_runnable.pname(), Long.toString(lr));
+ meta_props.put(IStateServices.SvcMetaProps.last_runnable.pname(), Long.toString(lr));
if ( last_runnable == 0 ) {
- meta_props.put(IStateServices.SvcProps.last_runnable_readable.pname(), "Unknown");
+ meta_props.put(IStateServices.SvcMetaProps.last_runnable_readable.pname(), "Unknown");
} else {
- meta_props.put(IStateServices.SvcProps.last_runnable_readable.pname(), (new Date(lr)).toString());
+ meta_props.put(IStateServices.SvcMetaProps.last_runnable_readable.pname(), (new Date(lr)).toString());
}
}
@@ -604,13 +611,13 @@ public class ServiceSet
run_failures = 0;
ping_failures = 0;
init_failures = 0;
- meta_props.remove(IStateServices.SvcProps.submit_error.pname());
+ meta_props.remove(IStateServices.SvcMetaProps.submit_error.pname());
excessiveRunFailures = false;
}
synchronized void setAutostart(boolean auto)
{
- meta_props.setProperty(IStateServices.SvcProps.autostart.pname(), auto ? "true" : "false");
+ meta_props.setProperty(IStateServices.SvcMetaProps.autostart.pname(), auto ? "true" : "false");
this.autostart = auto;
if ( auto ) {
// turning this on gives benefit of the doubt on failure management
@@ -696,13 +703,13 @@ public class ServiceSet
synchronized void disable(String reason)
{
- meta_props.put(IStateServices.SvcProps.disable_reason.pname(), reason);
+ meta_props.put(IStateServices.SvcMetaProps.disable_reason.pname(), reason);
this.enabled = false;
}
synchronized void enable()
{
- meta_props.remove(IStateServices.SvcProps.disable_reason.pname());
+ meta_props.remove(IStateServices.SvcMetaProps.disable_reason.pname());
resetRuntimeErrors();
this.enabled = true;
}
@@ -714,7 +721,7 @@ public class ServiceSet
synchronized String getDisableReason()
{
- return meta_props.getStringProperty(IStateServices.SvcProps.disable_reason.pname(), "Unknown");
+ return meta_props.getStringProperty(IStateServices.SvcMetaProps.disable_reason.pname(), "Unknown");
}
/**
@@ -817,8 +824,8 @@ public class ServiceSet
if ( ! isRecovered ) { // if not recovery, no need to mess with the record
stateHandler.storeProperties(id, job_props, meta_props);
} else {
- stateHandler.updateJobProperties(meta_props.get("svc_dbid"), id, (Properties) job_props);
- stateHandler.updateMetaProperties(meta_props.get("meta_dbid"), id, meta_props);
+ stateHandler.updateJobProperties(id, (Properties) job_props);
+ stateHandler.updateMetaProperties(id, meta_props);
}
}
@@ -828,7 +835,7 @@ public class ServiceSet
// no more changes
if ( isDeregistered() ) return;
- stateHandler.updateJobProperties(meta_props.get("svc_dbid"), id, (Properties) job_props);
+ stateHandler.updateJobProperties(id, (Properties) job_props);
}
synchronized void updateMetaProperties()
@@ -839,7 +846,7 @@ public class ServiceSet
// if ( isDeregistered() ) return;
prepareMetaProperties();
- stateHandler.updateMetaProperties(meta_props.get("meta_dbid"), id, meta_props);
+ stateHandler.updateMetaProperties(id, meta_props);
}
void prepareMetaProperties()
@@ -872,27 +879,27 @@ public class ServiceSet
}
- meta_props.put(IStateServices.SvcProps.reference.pname(), isReferencedStart() ? "true" : "false");
- meta_props.put(IStateServices.SvcProps.autostart.pname(), isAutostart() ? "true" : "false");
+ meta_props.put(IStateServices.SvcMetaProps.reference.pname(), isReferencedStart() ? "true" : "false");
+ meta_props.put(IStateServices.SvcMetaProps.autostart.pname(), isAutostart() ? "true" : "false");
- meta_props.put(IStateServices.SvcProps.enabled.pname(), ""+enabled);
- meta_props.put(IStateServices.SvcProps.service_state.pname(), ""+ getState());
- meta_props.put(IStateServices.SvcProps.ping_active.pname(), "" + (serviceMeta != null));
- meta_props.put(IStateServices.SvcProps.service_alive.pname(), "false");
- meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.enabled.pname(), ""+enabled);
+ meta_props.put(IStateServices.SvcMetaProps.service_state.pname(), ""+ getState());
+ meta_props.put(IStateServices.SvcMetaProps.ping_active.pname(), "" + (serviceMeta != null));
+ meta_props.put(IStateServices.SvcMetaProps.service_alive.pname(), "false");
+ meta_props.put(IStateServices.SvcMetaProps.service_healthy.pname(), "false");
if ( excessiveFailures() ) {
- meta_props.put(IStateServices.SvcProps.submit_error.pname(), "Service stopped by exessive failures. Initialization failures[" + init_failures + "], Runtime failures[" + run_failures + "]");
+ meta_props.put(IStateServices.SvcMetaProps.submit_error.pname(), "Service stopped by exessive failures. Initialization failures[" + init_failures + "], Runtime failures[" + run_failures + "]");
} else {
- meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "N/A");
+ meta_props.put(IStateServices.SvcMetaProps.service_statistics.pname(), "N/A");
}
if ( serviceMeta != null ) {
IServiceStatistics ss = serviceMeta.getServiceStatistics();
if ( ss != null ) {
- meta_props.put(IStateServices.SvcProps.service_alive.pname(), "" + ss.isAlive());
- meta_props.put(IStateServices.SvcProps.service_healthy.pname(), "" + ss.isHealthy());
- meta_props.put(IStateServices.SvcProps.service_statistics.pname(), "" + ss.getInfo());
+ meta_props.put(IStateServices.SvcMetaProps.service_alive.pname(), "" + ss.isAlive());
+ meta_props.put(IStateServices.SvcMetaProps.service_healthy.pname(), "" + ss.isHealthy());
+ meta_props.put(IStateServices.SvcMetaProps.service_statistics.pname(), "" + ss.getInfo());
if ( ss.isAlive() ) { // UIMA-4309
setLastPing(System.currentTimeMillis());
@@ -916,7 +923,7 @@ public class ServiceSet
synchronized void updateRegisteredInstances(int n)
{
- meta_props.setProperty(IStateServices.SvcProps.instances.pname(), Integer.toString(n));
+ meta_props.setProperty(IStateServices.SvcMetaProps.instances.pname(), Integer.toString(n));
registered_instances = n;
}
@@ -978,7 +985,7 @@ public class ServiceSet
String methodName = "persistReferences";
if ( references.size() == 0 ) {
- meta_props.remove(IStateServices.SvcProps.references.pname());
+ meta_props.remove(IStateServices.SvcMetaProps.references.pname());
} else {
StringBuffer sb = new StringBuffer();
for ( DuccId id : references.keySet() ) {
@@ -986,7 +993,7 @@ public class ServiceSet
sb.append(" ");
}
String s = sb.toString().trim();
- meta_props.setProperty(IStateServices.SvcProps.references.pname(), s);
+ meta_props.setProperty(IStateServices.SvcMetaProps.references.pname(), s);
}
try {
updateMetaProperties();
@@ -1079,19 +1086,19 @@ public class ServiceSet
public void setErrorString(String s)
throws Exception
{
- meta_props.put(IStateServices.SvcProps.submit_error.pname(), s);
+ meta_props.put(IStateServices.SvcMetaProps.submit_error.pname(), s);
updateMetaProperties();
}
public String getErrorString()
{
- return meta_props.getProperty(IStateServices.SvcProps.submit_error.pname());
+ return meta_props.getProperty(IStateServices.SvcMetaProps.submit_error.pname());
}
void setReferenced(boolean r)
{
this.reference_start = r;
- meta_props.put(IStateServices.SvcProps.reference.pname(), Boolean.toString(this.reference_start));
+ meta_props.put(IStateServices.SvcMetaProps.reference.pname(), Boolean.toString(this.reference_start));
}
public synchronized void reference(DuccId id)
@@ -1808,7 +1815,7 @@ public class ServiceSet
if ( isPingOnly() && (ping_failures > ping_failure_max) ) {
logger.warn(methodName, id, "Stopping ping-only service due to excessive falutes:", ping_failure_max);
- meta_props.put(IStateServices.SvcProps.submit_error.pname(), "Stopping ping-only service due to excessive falutes: " + ping_failure_max);
+ meta_props.put(IStateServices.SvcMetaProps.submit_error.pname(), "Stopping ping-only service due to excessive falutes: " + ping_failure_max);
stop(-1L); // must be -lL Long to get the right overload
implementors.remove(-1L);
@@ -2160,13 +2167,13 @@ public class ServiceSet
sd.setLinger(linger_time);
sd.setId(id.getFriendly());
sd.setUser(user);
- sd.setDisableReason(meta_props.getStringProperty(IStateServices.SvcProps.disable_reason.pname(), null));
+ sd.setDisableReason(meta_props.getStringProperty(IStateServices.SvcMetaProps.disable_reason.pname(), null));
sd.setLastUse(last_use);
sd.setLastPing(last_ping); // UIMA-4309
sd.setLastRunnable(last_runnable); // UIMA-4309
- sd.setRegistrationDate(meta_props.getStringProperty(IStateServices.SvcProps.registration_date.pname(), ""));
+ sd.setRegistrationDate(meta_props.getStringProperty(IStateServices.SvcMetaProps.registration_date.pname(), ""));
sd.setReferenceStart(reference_start);
- sd.setErrorString(meta_props.getStringProperty(IStateServices.SvcProps.submit_error.pname(), null));
+ sd.setErrorString(meta_props.getStringProperty(IStateServices.SvcMetaProps.submit_error.pname(), null));
if ( serviceMeta != null ) {
sd.setQueueStatistics(serviceMeta.getServiceStatistics());
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/pom.xml Wed Oct 28 18:12:53 2015
@@ -75,44 +75,7 @@
<artifactId>uima-ducc-container</artifactId>
<scope>compile</scope>
</dependency>
-
- <dependency>
- <groupId>com.orientechnologies</groupId>
- <artifactId>orientdb-graphdb</artifactId>
- <version>${orientdb.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.tinkerpop.blueprints</groupId>
- <artifactId>blueprints-core</artifactId>
- <version>2.6.0</version>
- </dependency>
- <dependency>
- <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
- <artifactId>concurrentlinkedhashmap-lru</artifactId>
- <version>1.4.2</version>
- </dependency>
-
- <dependency>
- <groupId>com.tinkerpop.gremlin</groupId>
- <artifactId>gremlin-groovy</artifactId>
- <version>2.6.0</version>
- </dependency>
-
- <dependency>
- <groupId>com.orientechnologies</groupId>
- <artifactId>orientdb-lucene</artifactId>
- <version>${orientdb.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.webjars</groupId>
- <artifactId>orientdb-studio</artifactId>
- <version>${orientdb.studio.version}</version>
- </dependency>
-
-
<!-- dependency>
<groupId>commons-httpclient</groupId>
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java Wed Oct 28 18:12:53 2015
@@ -68,7 +68,7 @@ public class HistoryFactory
@SuppressWarnings("unchecked")
Class<IStateServices> iss = (Class<IStateServices>) Class.forName(clname);
instance = (IHistoryPersistenceManager) iss.newInstance();
- instance.setLogger(logger);
+ instance.init(logger);
} catch ( Throwable t ) {
logger.error(methodName, null, "Cannot instantiate service persistence class", clname, ":", t);
instance = new NullHistoryManager();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java Wed Oct 28 18:12:53 2015
@@ -32,6 +32,7 @@ import java.util.Map;
import org.apache.uima.ducc.common.DuccEnvironmentHelper;
import org.apache.uima.ducc.common.IDuccEnv;
+import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.IOHelper;
@@ -66,9 +67,10 @@ public class HistoryPersistenceManager i
mkdirs();
}
- public void setLogger(DuccLogger logger)
+ public boolean init(DuccLogger logger)
{
this.logger = logger;
+ return true;
}
private void mkdirs() {
@@ -560,10 +562,10 @@ public class HistoryPersistenceManager i
return false;
}
- public boolean restore(DuccWorkMap m, Map<DuccId, DuccId> processToJob)
+ public Pair<DuccWorkMap, Map<DuccId, DuccId>> restore()
throws Exception
{
- return false;
+ return null;
}
private static int doJobs(HistoryPersistenceManager hpm)
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java Wed Oct 28 18:12:53 2015
@@ -21,6 +21,8 @@ package org.apache.uima.ducc.transport.e
import java.util.List;
import java.util.Map;
+import org.apache.uima.ducc.common.Pair;
+import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
@@ -44,9 +46,104 @@ public interface IHistoryPersistenceMana
public List<IDuccWorkService> restoreServices(long max) throws Exception;
public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception;
- public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception;
+ public Pair<DuccWorkMap, Map<DuccId, DuccId>> restore() throws Exception;
+
+ /**
+ * Establish a logger and anything else the persistence may need.
+ *
+ * @param logger This is the logger to be used. It is usually
+ * the same logger as the client of persistence, e.g.
+ * org.apache.uima.ducc.rm. The implementor is required
+ * to adjust itself to use this logger to insure
+ * messages are logged into the right log.
+ */
+ public boolean init(DuccLogger logger) throws Exception;
+
+ public enum OrWorkProps // properties for the OR work map
+ implements IDbProperty
+ {
+ JOB_TABLE {
+ public String pname() { return "job_history"; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+ RESERVATION_TABLE {
+ public String pname() { return "res_history"; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+ SERVICE_TABLE {
+ public String pname() { return "svc_history"; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+
+ // The order of the primary keys is important here as the Db assigns semantics to the first key in a compound PK
+ type {
+ public String pname() { return "type"; } // "job", "reservation", "service", ...
+ public boolean isPrimaryKey() { return true; }
+ },
+
+ ducc_dbid {
+ public String pname() { return "ducc_dbid"; }
+ public Type type() { return Type.Long; }
+ public boolean isPrimaryKey() { return true; }
+ },
+
+ history {
+ public String pname() { return "history"; } // to the future, is this a history or ckpt item?
+ public Type type() { return Type.Boolean; }
+ },
+
+ work {
+ public String pname() { return "work"; };
+ public Type type() { return Type.Blob; }
+ },
+
+ ;
+ public Type type() { return Type.String; }
+ public boolean isPrimaryKey() { return false; }
+ public boolean isPrivate() { return false; }
+ public boolean isMeta() { return false; }
+ public String columnName() { return pname(); }
+
+ };
+
+ public enum OrCkptProps // properties for the OR checkpoint
+ implements IDbProperty
+ {
+ CKPT_TABLE {
+ public String pname() { return "orckpt"; }
+ public boolean isPrivate() { return true; }
+ public boolean isMeta() { return true; }
+ },
+
+ id {
+ public String pname() { return "id"; }
+ public boolean isPrimaryKey() { return true; }
+ public Type type() { return Type.Integer; }
+ },
+
+ work {
+ public String pname() { return "work"; };
+ },
+
+ p2jmap {
+ public String pname() { return "p2jmap"; };
+ },
+
+ ;
+ public Type type() { return Type.Blob; }
+ public boolean isPrimaryKey() { return false; }
+ public boolean isPrivate() { return false; }
+ public boolean isMeta() { return false; }
+ public String columnName() {return pname(); }
+
+ };
- public void setLogger(DuccLogger logger);
//public void serviceSaveConditional(IDuccWorkService duccWorkService) throws Exception;
// public void serviceSave(IDuccWorkService duccWorkService) throws Exception;
//public IDuccWorkService serviceRestore(String fileName);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java Wed Oct 28 18:12:53 2015
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
@@ -38,7 +39,7 @@ public class NullHistoryManager
{
}
- public void setLogger(DuccLogger logger) {}
+ public boolean init(DuccLogger logger) {return true;}
public void saveJob(IDuccWorkJob duccWorkJob)
throws Exception
@@ -126,6 +127,6 @@ public class NullHistoryManager
}
public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; }
- public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; }
+ public Pair<DuccWorkMap, Map<DuccId, DuccId>> restore() throws Exception { return null; }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java Wed Oct 28 18:12:53 2015
@@ -26,28 +26,28 @@ public class IServicesRegistry {
public static final String svc = IStateServices.svc;
// meta
- public static final String autostart = IStateServices.SvcProps.autostart.pname();
- public static final String reference = IStateServices.SvcProps.reference.pname();
- public static final String enabled = IStateServices.SvcProps.enabled.pname();
- public static final String disable_reason = IStateServices.SvcProps.disable_reason.pname();
- public static final String endpoint = IStateServices.SvcProps.endpoint.pname();
- public static final String implementors = IStateServices.SvcProps.implementors.pname();
- public static final String instances = IStateServices.SvcProps.instances.pname();
- public static final String numeric_id = IStateServices.SvcProps.numeric_id.pname();
- public static final String ping_active = IStateServices.SvcProps.ping_active.pname();
- public static final String ping_only = IStateServices.SvcProps.ping_only.pname();
- public static final String service_alive = IStateServices.SvcProps.service_alive.pname();
- public static final String service_class = IStateServices.SvcProps.service_class.pname();
- public static final String service_dependency = IStateServices.SvcProps.service_dependency.pname();
- public static final String service_healthy = IStateServices.SvcProps.service_healthy.pname();
- public static final String service_state = IStateServices.SvcProps.service_state.pname();
- public static final String last_use = IStateServices.SvcProps.last_use.pname();
- public static final String service_statistics = IStateServices.SvcProps.service_statistics.pname();
- public static final String service_type = IStateServices.SvcProps.service_type.pname();
- public static final String submit_error = IStateServices.SvcProps.submit_error.pname();
- public static final String user = IStateServices.SvcProps.user.pname();
+ public static final String autostart = IStateServices.SvcMetaProps.autostart.pname();
+ public static final String reference = IStateServices.SvcMetaProps.reference.pname();
+ public static final String enabled = IStateServices.SvcMetaProps.enabled.pname();
+ public static final String disable_reason = IStateServices.SvcMetaProps.disable_reason.pname();
+ public static final String endpoint = IStateServices.SvcMetaProps.endpoint.pname();
+ public static final String implementors = IStateServices.SvcMetaProps.implementors.pname();
+ public static final String instances = IStateServices.SvcMetaProps.instances.pname();
+ public static final String numeric_id = IStateServices.SvcMetaProps.numeric_id.pname();
+ public static final String ping_active = IStateServices.SvcMetaProps.ping_active.pname();
+ public static final String ping_only = IStateServices.SvcMetaProps.ping_only.pname();
+ public static final String service_alive = IStateServices.SvcMetaProps.service_alive.pname();
+ public static final String service_class = IStateServices.SvcMetaProps.service_class.pname();
+ public static final String service_dependency = IStateServices.SvcMetaProps.service_dependency.pname();
+ public static final String service_healthy = IStateServices.SvcMetaProps.service_healthy.pname();
+ public static final String service_state = IStateServices.SvcMetaProps.service_state.pname();
+ public static final String last_use = IStateServices.SvcMetaProps.last_use.pname();
+ public static final String service_statistics = IStateServices.SvcMetaProps.service_statistics.pname();
+ public static final String service_type = IStateServices.SvcMetaProps.service_type.pname();
+ public static final String submit_error = IStateServices.SvcMetaProps.submit_error.pname();
+ public static final String user = IStateServices.SvcMetaProps.user.pname();
- public static final String service_type_CUSTOM = IStateServices.SvcProps.CUSTOM.pname();
+ public static final String service_type_CUSTOM = IStateServices.CUSTOM;
public static final String constant_Available = "Available";
public static final String constant_true = "true";
@@ -60,9 +60,9 @@ public class IServicesRegistry {
public static final String constant_OK = "OK";
// svc
- public static final String description = IStateServices.SvcProps.description.pname();
- public static final String process_memory_size = IStateServices.SvcProps.process_memory_size.pname();
- public static final String scheduling_class = IStateServices.SvcProps.scheduling_class.pname();
- public static final String log_directory = IStateServices.SvcProps.log_directory.pname();
+ public static final String description = IStateServices.SvcRegProps.description.pname();
+ public static final String process_memory_size = IStateServices.SvcRegProps.process_memory_size.pname();
+ public static final String scheduling_class = IStateServices.SvcRegProps.scheduling_class.pname();
+ public static final String log_directory = IStateServices.SvcRegProps.log_directory.pname();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java?rev=1711088&r1=1711087&r2=1711088&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java Wed Oct 28 18:12:53 2015
@@ -62,11 +62,11 @@ public class ServicesSortCache {
for(Entry<SortableService, IServiceAdapter> entry : map.entrySet()) {
IServiceAdapter payload = entry.getValue();
Properties meta = payload.getMeta();
- String key = IStateServices.SvcProps.numeric_id.pname();
+ String key = IStateServices.SvcMetaProps.numeric_id.pname();
String value = meta.getProperty(key);
int numeric_id = Integer.parseInt(value);
if(numeric_id == id) {
- meta.setProperty(IStateServices.SvcProps.enabled.pname(), Boolean.toString(bool));
+ meta.setProperty(IStateServices.SvcMetaProps.enabled.pname(), Boolean.toString(bool));
payload.setMeta(meta);
break;
}