You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/11/11 15:45:37 UTC
svn commit: r1713860 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/
src/main/resources/
uima-ducc-database/src/main/java/org/apache/uima/ducc/database/
uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/
Author: challngr
Date: Wed Nov 11 14:45:37 2015
New Revision: 1713860
URL: http://svn.apache.org/viewvc?rev=1713860&view=rev
Log:
UIME-4577 DB support updates.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Wed Nov 11 14:45:37 2015
@@ -55,7 +55,7 @@ class CheckDucc(DuccUtil):
if ( self.db_bypass == True ):
return True
- ret = self.db_alive(3)
+ ret = self.db_alive(1)
if ( ret ):
print 'The database is running'
else:
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Nov 11 14:45:37 2015
@@ -208,6 +208,11 @@ class DuccUtil(DuccBase):
if ( dbnode == None ):
print 'No database location defined.'
return False
+
+ pidfile = self.DUCC_HOME + '/state/cassandra.pid'
+ if ( not os.path.exists(pidfile) ):
+ print 'Database pid file does not exist. Checking DB connectivity.'
+
# get our log4j config into the path to shut up noisy logging
os.environ['CLASSPATH'] = os.environ['CLASSPATH'] + ':' + self.DUCC_HOME + '/resources'
@@ -269,11 +274,11 @@ class DuccUtil(DuccBase):
return True
pidfile = self.DUCC_HOME + '/state/cassandra.pid'
-
- # for cassandra, just send it a terminate signal. a pidfile is written on startup
- CMD = ['kill', '-TERM', '`cat ' + pidfile + '`']
- CMD = ' '.join(CMD)
- os.system(CMD)
+ if ( os.path.exists(pidfile) ):
+ # for cassandra, just send it a terminate signal. a pidfile is written on startup
+ CMD = ['kill', '-TERM', '`cat ' + pidfile + '`']
+ CMD = ' '.join(CMD)
+ os.system(CMD)
def find_netstat(self):
# don't you wish people would get together on where stuff lives?
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/log4j.xml Wed Nov 11 14:45:37 2015
@@ -140,12 +140,13 @@
</layout>
</appender>
- <category name="org.apache.uima.ducc.rm.scheduler" additivity="true">
+ <category name="org.apache.uima.ducc.rm" additivity="true">
<priority value="debug"/>
<appender-ref ref="rmlog" />
</category>
- <category name="org.apache.uima.ducc.rm.JobManagerConverter" additivity="true">
+ <!-- Please keep these for reference as they focus on specific components of interest during debug. -->
+ <!-- category name="org.apache.uima.ducc.rm.JobManagerConverter" additivity="true">
<priority value="info"/>
<appender-ref ref="rmlog" />
</category>
@@ -158,7 +159,7 @@
<category name="org.apache.uima.ducc.rm.ResourceManagerComponent" additivity="true">
<priority value="info"/>
<appender-ref ref="rmlog" />
- </category>
+ </category -->
<category name="org.apache.uima.ducc.orchestrator" additivity="true">
<priority value="info"/>
@@ -240,6 +241,11 @@
<appender-ref ref="jp-log" />
</category>
+ <!-- cassandra client -->
+ <category name="com.datastax.driver">
+ <priority value="warn"/>
+ </category>
+
<category name="org.apache.uima.ducc.database.DbLoader" additivity="true">
<priority value="info"/>
<appender-ref ref="db-loader-log" />
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbManager.java Wed Nov 11 14:45:37 2015
@@ -33,6 +33,8 @@ import com.datastax.driver.core.Prepared
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.policies.ConstantReconnectionPolicy;
+import com.datastax.driver.core.policies.ReconnectionPolicy;
/**
* Provide a common point for contacting the db, acquiring sessions/handles to it, managing the db,
@@ -90,9 +92,11 @@ public class DbManager
String pw = dbPassword();
PlainTextAuthProvider auth = new PlainTextAuthProvider("ducc", pw);
+ ReconnectionPolicy rp = new ConstantReconnectionPolicy(10000); // if we lose connection, keep trying every 10 seconds
cluster = Cluster.builder()
.withAuthProvider(auth)
.addContactPoint(dburl)
+ .withReconnectionPolicy(rp)
.build();
Metadata metadata = cluster.getMetadata();
@@ -107,7 +111,7 @@ public class DbManager
{
String methodName = "closeDatabase";
logger.info(methodName, null, "Closing the database.");
- cluster.close();
+ if ( cluster != null ) cluster.close();
cluster = null;
session = null;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/DbUtil.java Wed Nov 11 14:45:37 2015
@@ -20,12 +20,15 @@
package org.apache.uima.ducc.database;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.uima.ducc.common.persistence.IDbProperty;
import org.apache.uima.ducc.common.persistence.IDbProperty.Type;
+import com.datastax.driver.core.Row;
+
/**
* Static common helper methods.
*
@@ -227,6 +230,51 @@ class DbUtil
}
/**
+ * Common code to pull things from a row according to the schema, into a map
+ */
+ static Map<String, Object> getProperties(IDbProperty[] props, Row r)
+ {
+ Map<String, Object> ret = new HashMap<String, Object>();
+ for ( IDbProperty p : props ) {
+ if ( p.isPrivate() ) continue;
+ if ( p.isMeta() ) continue;
+ Object val = null;
+ switch ( p.type() ) {
+
+ case String:
+ val = r.getString(p.columnName());
+ break;
+
+ case Integer:
+ val = r.getInt(p.columnName());
+ break;
+
+ case Long:
+ val = r.getLong(p.columnName());
+ break;
+
+ case Double:
+ val = r.getDouble(p.columnName());
+ break;
+
+ case UUID:
+ val = r.getUUID(p.columnName());
+ break;
+
+ case Boolean:
+ val = r.getBool(p.columnName());
+ break;
+
+ case Blob:
+ val = r.getBytes(p.columnName());
+ break;
+ }
+ if ( val != null ) ret.put(p.pname(), val);
+ }
+ return ret;
+ }
+
+ /**
* Convert our generic "type" to the right name for this db implementation.
* We could make Type a magic enum but I want to hide DB specifics, in particular,
* how this database names various java types.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java Wed Nov 11 14:45:37 2015
@@ -26,6 +26,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -43,6 +44,7 @@ import com.datastax.driver.core.Prepared
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
@@ -75,24 +77,30 @@ public class HistoryManagerDb
String methodName = "init";
boolean ret = true;
logger.info(methodName, null, "Initializing OR persistence over the database");
- try {
- if ( dbm != null ) {
- this.dbManager = dbm;
- } else {
- dbManager = new DbManager(dburl, logger);
- dbManager.init();
- }
-
- // prepare some statements
- DbHandle h = dbManager.open();
- jobPrepare = h.prepare("INSERT INTO " + JOB_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
- reservationPrepare = h.prepare("INSERT INTO " + RES_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
- servicePrepare = h.prepare("INSERT INTO " + SVC_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
- ckptPrepare = h.prepare("INSERT INTO " + CKPT_TABLE + " (id, work, p2jmap) VALUES (?, ?, ?);");
-
- } catch ( Exception e ) {
- logger.error(methodName, null, "Cannot open the history database:", e);
- throw e;
+ while ( true ) {
+ try {
+ if ( dbm != null ) {
+ this.dbManager = dbm;
+ } else {
+ dbManager = new DbManager(dburl, logger);
+ dbManager.init();
+ }
+
+ // prepare some statements
+ DbHandle h = dbManager.open();
+ jobPrepare = h.prepare("INSERT INTO " + JOB_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
+ reservationPrepare = h.prepare("INSERT INTO " + RES_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
+ servicePrepare = h.prepare("INSERT INTO " + SVC_TABLE + " (ducc_dbid, type, history, work) VALUES (?, ?, ?, ?) IF NOT EXISTS;");
+ ckptPrepare = h.prepare("INSERT INTO " + CKPT_TABLE + " (id, work, p2jmap) VALUES (?, ?, ?);");
+ break;
+ } catch ( NoHostAvailableException e ) {
+ logger.error(methodName, null, "Cannot contact database. Retrying in 5 seconds.");
+ Thread.sleep(5000);
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Errors contacting database. No connetion made.", e);
+ ret = false;
+ break;
+ }
}
return ret;
}
@@ -182,7 +190,7 @@ public class HistoryManagerDb
ByteBuffer buf = ByteBuffer.wrap(bytes);
DbHandle h = dbManager.open();
- h.saveObject(jobPrepare, w.getDuccId().getFriendly(), type, isHistory, buf);
+ h.saveObject(s, w.getDuccId().getFriendly(), type, isHistory, buf);
logger.info(methodName, w.getDuccId(), "----------> Time to save", type, ":", System.currentTimeMillis() - nowP, "Size:", bytes.length, "bytes.");
}
@@ -383,6 +391,14 @@ public class HistoryManagerDb
byte[] bytes = baos.toByteArray();
ByteBuffer workbuf = ByteBuffer.wrap(bytes);
+ if ( logger.isTrace() ) {
+ ConcurrentHashMap<DuccId, IDuccWork> map = work.getMap();
+ for ( DuccId id : map.keySet() ) {
+ IDuccWork w = map.get(id);
+ logger.trace(methodName, id, "Checkpointing", w.getClass());
+ }
+ }
+
baos = new ByteArrayOutputStream();
out= new ObjectOutputStream(baos);
out.writeObject(processToJob);
@@ -442,6 +458,13 @@ public class HistoryManagerDb
DuccWorkReservation res = (DuccWorkReservation) work.findDuccWork(DuccType.Reservation, ""+id.getFriendly());
if ( r != null ) res.initLogger();
}
+
+ // only gets called once per boot and might be useful, let's leave at info
+ ConcurrentHashMap<DuccId, IDuccWork> map = work.getMap();
+ for ( DuccId id : map.keySet() ) {
+ IDuccWork w = map.get(id);
+ logger.info(methodName, id, "Restored", w.getClass());
+ }
ret = new Pair(work, processToJob);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/RmStatePersistence.java Wed Nov 11 14:45:37 2015
@@ -32,6 +32,7 @@ import org.apache.uima.ducc.common.utils
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
/**
* Manage saving and fetching of transient RM state. The primary consumer is
@@ -52,13 +53,22 @@ public class RmStatePersistence
private boolean init(String dburl)
throws Exception
{
+ String methodName = "init";
boolean ret = false;
- try {
- dbManager = new DbManager(dburl, logger);
- dbManager.init();
- ret = true;
- } catch ( Exception e ) {
- throw e;
+ while ( true ) {
+ try {
+ dbManager = new DbManager(dburl, logger);
+ dbManager.init();
+ ret = true;
+ break;
+ } catch ( NoHostAvailableException e ) {
+ logger.error(methodName, null, "Cannot contact database. Retrying in 5 seconds.");
+ Thread.sleep(5000);
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Errors contacting database. No connetion made.", e);
+ ret = false;
+ break;
+ }
}
return ret;
}
@@ -180,39 +190,8 @@ public class RmStatePersistence
DbHandle h = dbManager.open();
ResultSet rs = h.execute(cql);
for ( Row r : rs ) {
- Map<String, Object> mach = new HashMap<String, Object>();
- // We don't expect any nulls in this table
- for ( RmNodes n : RmNodes.values() ) {
- if ( n.isPrivate() ) continue;
- if ( n.isMeta() ) continue;
- switch ( n.type() ) {
-
- case String: {
- String v = r.getString(n.columnName());
- mach.put(n.pname(), v);
- if ( n == RmNodes.Name ) {
- ret.put(v, mach);
- }
- }
- break;
-
- case Integer: {
- int v = r.getInt(n.columnName());
- mach.put(n.pname(), v);
- }
- break;
-
- case Boolean: {
- boolean v = r.getBool(n.columnName());
- mach.put(n.pname(), v);
- }
- break;
-
- default:
- logger.warn(methodName, null, "Unexpected value in db:", n.pname(), "type", n.type(), "is not recognized.");
- break;
- }
- }
+ Map<String, Object> mach = DbUtil.getProperties(RmNodes.values(), r);
+ ret.put((String)mach.get(RmNodes.Name.pname()), mach);
}
return ret;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/StateServicesDb.java Wed Nov 11 14:45:37 2015
@@ -36,6 +36,7 @@ import org.apache.uima.ducc.common.utils
import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.SimpleStatement;
+import com.datastax.driver.core.exceptions.NoHostAvailableException;
public class StateServicesDb
implements IStateServices
@@ -72,13 +73,19 @@ public class StateServicesDb
if ( dbm != null ) {
this.dbManager = dbm;
} else {
- try {
- dbManager = new DbManager(dburl, logger);
- dbManager.init();
- ret = true;
- } catch ( Exception e ) {
- logger.error(methodName, null, "Cannot open the state database:", e);
- throw e;
+ while ( true ) {
+ try {
+ dbManager = new DbManager(dburl, logger);
+ dbManager.init();
+ break;
+ } catch ( NoHostAvailableException e ) {
+ logger.error(methodName, null, "Cannot contact the database host, Retryng connection in 5 seconds.");
+ Thread.sleep(5000);
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Cannot open the database.", e);
+ ret = false;
+ break;
+ }
}
}
@@ -112,32 +119,24 @@ public class StateServicesDb
s.setFetchSize(100);
DbHandle h = dbManager.open();
ResultSet rs = h.execute(s);
- for ( Row r: rs ) {
- long id = Long.parseLong(r.getString("numeric_id"));
- DuccId did = new DuccId(id);
- logger.debug(methodName, did, "Found properties in table", tableid);
+ for ( Row r : rs ) {
+ Map<String, Object> rowvals = DbUtil.getProperties(props, r);
DuccProperties dp = new DuccProperties();
- for ( IDbProperty p : props ) {
- String val = null;
- if ( logger.isTrace() ) {
- logger.info(methodName, null, "Fetching", p.pname(), "from", tableid);
- }
- if ( !(p.isPrivate() || p.isMeta()) ) { // collect non-private / non-meta properties into dp
- val = r.getString(p.columnName());
- if ( val != null ) {
- dp.put(p.pname(), val);
- }
- }
- if ( p.isPrimaryKey() ) { // once we find the key we set the dp int return val
- String k = null;
- if ( val == null ) {
- k = r.getString(p.columnName());
- } else {
- k = val;
- }
- ret.put(Long.parseLong(k), dp);
+ for (String k : rowvals.keySet()) {
+ dp.put(k, rowvals.get(k));
+ }
+
+ // if these don't work the DB is busted. This set of properties is now useless.
+ try {
+ String id = r.getString(IStateServices.SvcRegProps.numeric_id.pname());
+ if ( id == null ) {
+ throw new IllegalStateException("Missing numeric id for service properties.");
}
+ ret.put(Long.parseLong(id), dp);
+ } catch ( Exception e ) {
+ logger.error(methodName, null, "Fatal error recovering properties. Discarding row.");
}
+
}
return ret;
}
@@ -354,7 +353,9 @@ public class StateServicesDb
Object[] updates = new Object[props.size()*2];
int i = 0;
for ( IDbProperty k : map.keySet() ) {
- logger.info(methodName, null, "Updating", k.columnName(), "with", map.get(k));
+ if ( logger.isTrace() ) {
+ logger.trace(methodName, null, "Updating", k.columnName(), "with", map.get(k));
+ }
updates[i++] = k;
updates[i++] = map.get(k);
}
@@ -409,44 +410,4 @@ public class StateServicesDb
return ret;
}
- // static String[] mkSchemaForReg()
- // {
-
- // int size = SvcRegProps.values().length;
- // String[] ret = new String[size];
-
- // int ndx = 0;
-
- // for ( SvcRegProps p: SvcRegProps.values() ) {
- // String s = p.pname();
- // s = s + " " + DbUtil.typeToString(p.type());
- // if ( p.isPrimaryKey() ) {
- // s = s + " PRIMARY KEY";
- // }
- // ret[ndx++] = s;
- // }
- // return ret;
-
- // }
-
- // static String[] mkSchemaForMeta()
- // {
- // int size = SvcMetaProps.values().length;
- // String[] ret = new String[size];
-
- // int ndx = 0;
-
- // for ( SvcMetaProps p: SvcMetaProps.values() ) {
- // String s = p.pname();
- // s = s + " " + DbUtil.typeToString(p.type());
- // if ( p.isPrimaryKey() ) {
- // s = s + " PRIMARY KEY";
- // }
- // ret[ndx++] = s;
- // }
-
- // return ret;
-
- // }
-
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceInstance.java Wed Nov 11 14:45:37 2015
@@ -229,8 +229,6 @@ class ServiceInstance
try {
Process p = pb.start();
- // TODO: we should attach these streams to readers in threads because too much output
- // can cause blocking, deadlock, ugliness.
InputStream stdout = p.getInputStream();
InputStream stderr = p.getErrorStream();
@@ -317,6 +315,9 @@ class ServiceInstance
}
logger.info(methodName, sset.getId(), "START INSTANCE COMPLETE");
+ stdout_lines.clear();
+ stderr_lines.clear();
+
return numeric_id;
}
@@ -355,58 +356,32 @@ class ServiceInstance
ProcessBuilder pb = new ProcessBuilder(args);
Map<String, String> env = pb.environment();
env.put("DUCC_HOME", System.getProperty("DUCC_HOME"));
-
- ArrayList<String> stdout_lines = new ArrayList<String>();
- ArrayList<String> stderr_lines = new ArrayList<String>();
+
+ StdioListener sin_listener = null;
+ StdioListener ser_listener = null;
+
int rc = 0;
try {
Process p = pb.start();
-
+
+ InputStream stdout = p.getInputStream();
+ InputStream stderr = p.getErrorStream();
+
+ sin_listener = new StdioListener(1, stdout, true);
+ ser_listener = new StdioListener(2, stderr, true);
+ Thread sol = new Thread(sin_listener);
+ Thread sel = new Thread(ser_listener);
+ sol.start();
+ sel.start();
+
rc = p.waitFor();
logger.info(methodName, sset.getId(), "DuccServiceCancel returns with rc", rc);
- if (logger.isTrace() || (rc != 0)) {
- InputStream stdout = p.getInputStream();
- InputStream stderr = p.getErrorStream();
- BufferedReader stdout_reader = new BufferedReader(new InputStreamReader(stdout));
- BufferedReader stderr_reader = new BufferedReader(new InputStreamReader(stderr));
-
- String line = null;
- while ( (line = stdout_reader.readLine()) != null ) {
- stdout_lines.add(line);
- }
-
- line = null;
- while ( (line = stderr_reader.readLine()) != null ) {
- stderr_lines.add(line);
- }
- }
-
+ sin_listener.stop();
+ ser_listener.stop();
} catch (Throwable t) {
- // TODO Auto-generated catch block
logger.error(methodName, null, t);
}
-
- if ( logger.isTrace() || ( rc != 0) ) {
- boolean inhibit_cp = false;
- for ( String s : stdout_lines ) {
- // simple logic to inhibit printing the danged classpath
- if ( inhibit_cp ) {
- inhibit_cp = false;
- logger.info(methodName, sset.getId(), "Instance", numeric_id, "<INHIBITED CP>");
- } else {
- logger.info(methodName, sset.getId(), "Instance", numeric_id, "Stop stdout:", s);
- }
-
- if ( s.indexOf("-cp") >= 0 ) {
- inhibit_cp = true;
- }
- }
-
- for ( String s : stderr_lines ) {
- logger.info(methodName, sset.getId(), "Instance", numeric_id, "Stop stderr:", s);
- }
- }
}
class StdioListener
@@ -416,8 +391,9 @@ class ServiceInstance
String tag;
boolean done = false;
int which = 0;
+ boolean ignore = false;
- StdioListener(int which, InputStream in)
+ StdioListener(int which, InputStream in, boolean ignore)
{
this.in = in;
this.which = which;
@@ -425,6 +401,13 @@ class ServiceInstance
case 1: tag = "STDOUT: "; break;
case 2: tag = "STDERR: "; break;
}
+ this.ignore = ignore;
+ this.ignore = ignore;
+ }
+
+ StdioListener(int which, InputStream in)
+ {
+ this(which, in, false);
}
void stop()
@@ -450,6 +433,8 @@ class ServiceInstance
logger.info(methodName, sset.getId(), msg);
return;
}
+ if ( ignore ) continue; // just discarding it
+
switch ( which ) {
case 1:
stdout_lines.add(s);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1713860&r1=1713859&r2=1713860&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Wed Nov 11 14:45:37 2015
@@ -18,6 +18,8 @@
*/
package org.apache.uima.ducc.sm;
+import java.io.File;
+import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Timestamp;
@@ -32,6 +34,8 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
+import javax.naming.ServiceUnavailableException;
+
import org.apache.uima.UIMAFramework;
import org.apache.uima.ducc.cli.IUiOptions.UiOption;
import org.apache.uima.ducc.cli.UimaAsPing;
@@ -1048,8 +1052,14 @@ public class ServiceSet
monitor.init(null);
monitor.clearQueues();
monitor.stop( );
+ } catch (IOException e) {
+ Throwable t = e.getCause();
+ // ServiceUnavailbleException means it's gone already and we're processing some sort of stale state
+ // that's still floating around.
+ if ( ! (t instanceof ServiceUnavailableException) ) {
+ logger.info(methodName, id, e);
+ }
} catch (Throwable e) {
- // totally not a problem, just lost it
logger.info(methodName, id, e.toString());
}
}
@@ -1866,6 +1876,8 @@ public class ServiceSet
};
ProcessBuilder pb = new ProcessBuilder(args);
+ pb.redirectOutput(new File("/dev/null"));
+ pb.redirectError(new File("/dev/null"));
try {
Process p = pb.start();
int rc = p.waitFor();