You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2013/09/03 12:21:13 UTC
svn commit: r1519636 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ uima-ducc-tr...
Author: challngr
Date: Tue Sep 3 10:21:13 2013
New Revision: 1519636
URL: http://svn.apache.org/r1519636
Log:
UIMA-3215 RM Logging updates.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/id/DuccIdFactory.java Tue Sep 3 10:21:13 2013
@@ -25,11 +25,11 @@ public class DuccIdFactory implements ID
private volatile long seqno = -1;
private IPropertiesFileManager propertiesFileManager = null;
- private String propertiesFileKey = null;
-
+ private String propertiesFileKey = null;
+
public DuccIdFactory() {
}
-
+
public DuccIdFactory(IPropertiesFileManager propertiesFileManager, String propertiesFileKey) {
this.propertiesFileManager = propertiesFileManager;
this.propertiesFileKey = propertiesFileKey;
@@ -39,13 +39,23 @@ public class DuccIdFactory implements ID
seqno = seed-1;
}
+ /**
+ * During recovery, if you pass in a "friendly", you always want the passed-in friendly, but
+ * you want to ensure that at the end of recovery, the seed is set to the largest of the
+ * recovered IDs.
+ */
+ public DuccId next(long s) {
+ seqno = Math.max(s, seqno);
+ return new DuccId(s);
+ }
+
public DuccId next() {
synchronized(this) {
if(propertiesFileManager != null) {
seqno = propertiesFileManager.increment(propertiesFileKey);
}
else {
- seqno++;
+ seqno++;
}
return new DuccId(seqno);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Tue Sep 3 10:21:13 2013
@@ -111,7 +111,7 @@ public class JobManagerConverter
String getElapsedTime(ITimeWindow w)
{
if ( w == null ) return "0";
- return w.getElapsed();
+ return w.getDiff();
}
// void formatSchedulingInfo(DuccId id, IDuccSchedulingInfo si, int remaining_work)
@@ -177,10 +177,10 @@ public class JobManagerConverter
}
logger.info(methodName, job.getDuccId(),
- String.format("total: %s -> %s: %d compl: %s error: %s rem: %d mean: %.2f",
+ String.format("tot: %d %s -> %s compl: %s err: %s rem: %d mean: %f",
+ total_work,
state,
job.getStateObject(),
- total_work,
si.getWorkItemsCompleted(), // note this comes in as string (!) from OR
si.getWorkItemsError(), // also string
remaining_work,
@@ -470,10 +470,10 @@ public class JobManagerConverter
void jobRemoved(DuccId id)
{
String methodName = "jobRemoved";
- logger.debug(methodName, id, "Signalling removal");
+ logger.trace(methodName, id, "Signalling removal");
scheduler.signalCompletion(id);
localMap.removeDuccWork(id);
- logger.debug(methodName, id, "Remove signalled");
+ logger.trace(methodName, id, "Remove signalled");
}
public void reconcileProcesses(DuccId jobid, IDuccWork l, IDuccWork r)
@@ -872,7 +872,7 @@ public class JobManagerConverter
ArrayList<Share> sorted = new ArrayList<Share>(shares.values());
Collections.sort(sorted, new RmJob.ShareByInvestmentSorter());
for ( Share s : sorted ) {
- Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder());
+ Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder(), s.getInitializationTime());
all_shares.put(s.getId(), r);
}
redrive = sanityCheckForOrchestrator(j, shares, expanded.get(j.getId()));
@@ -881,7 +881,7 @@ public class JobManagerConverter
shares = shrunken.get(j.getId());
if ( shares != null ) {
for ( Share s : shares.values() ) {
- Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder());
+ Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder(), 0);
shrunken_shares.put(s.getId(), r);
}
}
@@ -889,14 +889,14 @@ public class JobManagerConverter
shares = expanded.get(j.getId());
if ( shares != null ) {
for ( Share s : shares.values() ) {
- Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder());
+ Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder(), 0);
expanded_shares.put(s.getId(), r);
}
}
if ( redrive != null ) {
for ( Share s : redrive.values() ) {
- Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder());
+ Resource r = new Resource(s.getId(), s.getNode(), s.isPurged(), s.getShareOrder(), 0);
expanded_shares.put(s.getId(), r);
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java Tue Sep 3 10:21:13 2013
@@ -318,16 +318,16 @@ public class NodepoolScheduler
HashMap<IEntity, Integer> deserved = new HashMap<IEntity, Integer>(); // qshares
+ // This section dealing with RmCounter writes the counting parameters to the log in a form
+ // that can be cut/pasted into a java properties file. This file can then be used in the
+ // RmCounter test/deveopment application. It also turns out to be very useful in the log
+ // in general so it is promoted to 'info' level.
StringBuffer enames = null;
StringBuffer eweights = null;
- if ( logger.isDebug() ) {
- // for debugging, we'll print the parts of the props
- // file needed for the RmCounter prototyper
- logger.debug(methodName, null, descr, "RmCounter Start");
- logger.debug(methodName, null, descr, "maxorder = ", NodePool.getMaxOrder());
- enames = new StringBuffer();
- eweights = new StringBuffer();
- }
+ logger.info(methodName, null, descr, "RmCounter Start");
+ logger.info(methodName, null, descr, "maxorder = ", NodePool.getMaxOrder());
+ enames = new StringBuffer();
+ eweights = new StringBuffer();
for ( IEntity e : working ) {
int[] gbo = NodePool.makeArray();
@@ -338,23 +338,20 @@ public class NodepoolScheduler
// jrc e.initWantedByOrder();
- if ( logger.isDebug() ) { // RmCounter props file
- enames.append(e.getName());
- enames.append(" ");
- eweights.append(Integer.toString(e.getShareWeight()));
- eweights.append(" ");
- }
+ enames.append(e.getName());
+ enames.append(" ");
+ eweights.append(Integer.toString(e.getShareWeight()));
+ eweights.append(" ");
}
- if ( logger.isDebug() ) { // RmCounter props file
- logger.debug(methodName, null, descr, "entity_names = ", enames.toString());
- logger.debug(methodName, null, descr, "weights = ", eweights.toString());
- for ( IEntity e : working ) {
- logger.debug(methodName, null, descr, "wantedby." + e.getName() + " = ", fmtArray(e.getWantedByOrder()));
- }
- logger.debug(methodName, null, descr, "vmachines =", fmtArray(vshares));
- logger.debug(methodName, null, descr, "RmCounter End");
+ logger.info(methodName, null, descr, "entity_names = ", enames.toString());
+ logger.info(methodName, null, descr, "weights = ", eweights.toString());
+ for ( IEntity e : working ) {
+ logger.info(methodName, null, descr, "wantedby." + e.getName() + " = ", fmtArray(e.getWantedByOrder()));
}
+ logger.info(methodName, null, descr, "vmachines =", fmtArray(vshares));
+ logger.info(methodName, null, descr, "RmCounter End");
+
int pass = 0;
do {
@@ -760,12 +757,14 @@ public class NodepoolScheduler
private void howMuchFairShare(ArrayList<ResourceClass> rcs)
{
- String methodName = "fairShare";
- logger.info(methodName, null, "Scheduling FAIR SHARE for these classes:");
- logger.info(methodName, null, " ", ResourceClass.getHeader());
- logger.info(methodName, null, " ", ResourceClass.getDashes());
- for ( ResourceClass pc : rcs ) {
- logger.info(methodName, null, " ", pc.toString());
+ String methodName = "howMuchFairShare";
+ if ( logger.isTrace() ) {
+ logger.trace(methodName, null, "Scheduling FAIR SHARE for these classes:");
+ logger.trace(methodName, null, " ", ResourceClass.getHeader());
+ logger.trace(methodName, null, " ", ResourceClass.getDashes());
+ for ( ResourceClass pc : rcs ) {
+ logger.trace(methodName, null, " ", pc.toString());
+ }
}
ArrayList<ResourceClass> eligible = new ArrayList<ResourceClass>();
@@ -1015,11 +1014,14 @@ public class NodepoolScheduler
protected void howMuchFixed(ArrayList<ResourceClass> rcs)
{
String methodName = "howMuchFixedShare";
- logger.info(methodName, null, "Scheduling FIXED SHARE for these classes:");
- logger.info(methodName, null, " ", ResourceClass.getHeader());
- logger.info(methodName, null, " ", ResourceClass.getDashes());
- for ( ResourceClass pc : rcs ) {
- logger.info(methodName, null, " ", pc.toString());
+
+ if ( logger.isTrace() ) {
+ logger.trace(methodName, null, "Scheduling FIXED SHARE for these classes:");
+ logger.trace(methodName, null, " ", ResourceClass.getHeader());
+ logger.trace(methodName, null, " ", ResourceClass.getDashes());
+ for ( ResourceClass pc : rcs ) {
+ logger.trace(methodName, null, " ", pc.toString());
+ }
}
int total_jobs = 0;
@@ -1175,11 +1177,14 @@ public class NodepoolScheduler
private void howMuchReserve(ArrayList<ResourceClass> rcs)
{
String methodName = "howMuchToreserve";
- logger.info(methodName, null, "Calculating counts for RESERVATION for these classes:");
- logger.info(methodName, null, " ", ResourceClass.getHeader());
- logger.info(methodName, null, " ", ResourceClass.getDashes());
- for ( ResourceClass pc : rcs ) {
- logger.info(methodName, null, " ", pc.toString());
+
+ if ( logger.isTrace() ) {
+ logger.trace(methodName, null, "Calculating counts for RESERVATION for these classes:");
+ logger.trace(methodName, null, " ", ResourceClass.getHeader());
+ logger.trace(methodName, null, " ", ResourceClass.getDashes());
+ for ( ResourceClass pc : rcs ) {
+ logger.trace(methodName, null, " ", pc.toString());
+ }
}
for ( ResourceClass rc : rcs ) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java Tue Sep 3 10:21:13 2013
@@ -199,7 +199,7 @@ public class RmJob
* So resched = IR & RU
* IW = !RU
*
- * We return resched so caller knows the tickle the scheduler
+ * We return resched so caller knows to tickle the scheduler
*/
public boolean setInitWait(boolean is_running)
{
@@ -511,10 +511,10 @@ public class RmJob
public void removeAllShares()
{
String methodName = "removeAllShares";
- if ( logger.isDebug() ) {
+ if ( logger.isTrace() ) {
for ( Map<Share, Share> m : sharesByMachine.values() ) {
for ( Share s : m.values() ) {
- logger.debug(methodName, getId(), "Clear share", s);
+ logger.trace(methodName, getId(), "Clear share", s);
}
}
}
@@ -1033,7 +1033,7 @@ public class RmJob
}
}
- logger.debug(methodName, getId(), username, "O", getShareOrder(), "Base cap:", base_cap, "Expected future cap:", projected_cap, "potential cap", potential_cap, "actual cap", actual_cap);
+ logger.info(methodName, getId(), username, "O", getShareOrder(), "Base cap:", base_cap, "Expected future cap:", projected_cap, "potential cap", potential_cap, "actual cap", actual_cap);
return actual_cap;
}
@@ -1248,7 +1248,6 @@ public class RmJob
shares = countInstances();
}
- if ( name == null ) name = "";
// 1 2 3 4 5 6 7 8 9 10 11 12 13
String format = "%11s %30.30s %10s %10s %6d %5d %7d %3d %6d %6d %8d %8s %9d";
String jid = String.format("%1s%10s", getShortType(), id.toString()).replace(' ', '_');
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java Tue Sep 3 10:21:13 2013
@@ -1082,7 +1082,18 @@ public class Scheduler
purgeShare(share, job);
}
- private synchronized void processRecovery(IRmJob j)
+ /**
+ * Log following / reconstruction, needed to init before recovery.
+ */
+ public void resetNodepools()
+ {
+ nodepool.reset(NodePool.getMaxOrder());
+ }
+
+ /**
+ * Make this public for log following.
+ */
+ public synchronized void processRecovery(IRmJob j)
{
String methodName = "processRecovery";
@@ -1141,8 +1152,12 @@ public class Scheduler
public synchronized static DuccId newId()
{
- DuccId id = idFactory.next();
- return id;
+ return idFactory.next();
+ }
+
+ public synchronized static DuccId newId(long id)
+ {
+ return idFactory.next(id);
}
public void queryMachines()
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java Tue Sep 3 10:21:13 2013
@@ -23,6 +23,7 @@ import org.apache.uima.ducc.common.NodeI
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IProcessState.ProcessState;
import org.apache.uima.ducc.transport.event.common.ITimeWindow;
+import org.apache.uima.ducc.transport.event.common.TimeWindow;
@@ -261,12 +262,19 @@ public class Share
* Returns only initialization time. Eventually getInvestment() may take other things into
* consideration so we separate these two (even though currently they do the same thing.)
*/
- long getInitializationTime()
+ public long getInitializationTime()
{
if ( init_time == null ) return 0;
return init_time.getElapsedMillis();
}
+ public void setInitializationTime(long millis)
+ {
+ init_time = new TimeWindow();
+ init_time.setStartLong(0);
+ init_time.setEndLong(millis);
+ }
+
boolean isInitialized()
{
if ( init_time == null ) return false;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java?rev=1519636&r1=1519635&r2=1519636&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/rm/Resource.java Tue Sep 3 10:21:13 2013
@@ -29,19 +29,20 @@ public class Resource implements IResour
private Node node; // Node id, assigned by Agent
private boolean purged; // Purged, for node failure
private int qShares; // Number of quantum shares this resource occupies
-
+ private transient long itime; // initialization time, for the toString, but not to be transmitted
// dissallow
@SuppressWarnings("unused")
private Resource()
{
}
- public Resource(DuccId duccId, Node node, boolean purged, int qShares)
+ public Resource(DuccId duccId, Node node, boolean purged, int qShares, long itime)
{
this.duccId = duccId;
this.node = node;
this.purged = purged;
this.qShares = qShares;
+ this.itime = itime;
}
public DuccId getId()
@@ -71,6 +72,6 @@ public class Resource implements IResour
public String toString()
{
- return getNodeId().getName() + "." + duccId.getFriendly() + ( purged ? "[P]" : "");
+ return getNodeId().getName() + "." + duccId.getFriendly() + "^" + itime + ( purged ? "^P" : "");
}
}