You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/01/19 19:34:15 UTC
svn commit: r1653071 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ uima-ducc...
Author: challngr
Date: Mon Jan 19 18:34:14 2015
New Revision: 1653071
URL: http://svn.apache.org/r1653071
Log:
UIMA-4168 Dynamic RM dynamic reconfig, handle removed/renamed class.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java Mon Jan 19 18:34:14 2015
@@ -53,6 +53,8 @@ public class RmAdminQOccupancyReply
StringBuffer sb = new StringBuffer();
sb.append(RmQueriedMachine.header());
sb.append("\n");
+ sb.append(RmQueriedMachine.separator());
+ sb.append("\n");
for ( RmQueriedMachine m : machines ) {
sb.append(m.toConsole());
sb.append("\n");
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java Mon Jan 19 18:34:14 2015
@@ -31,14 +31,17 @@ public class RmQueriedMachine
String nodepoolId;
long memory;
int order;
+ boolean blacklisted;
List<RmQueriedShare> shares = null;
- public RmQueriedMachine(String name, String nodepoolId, long memory, int order)
+ // UIMA-4142, account for blacklist
+ public RmQueriedMachine(String name, String nodepoolId, long memory, int order, boolean blacklisted)
{
this.name = name;
this.nodepoolId = nodepoolId;
this.memory = memory;
this.order = order;
+ this.blacklisted = blacklisted;
}
public void addShare(RmQueriedShare rqs)
@@ -47,15 +50,22 @@ public class RmQueriedMachine
shares.add(rqs);
}
- public String getId() { return name; }
- public long getMemory() { return memory; }
- public int getShareOrder() { return order; }
+ public String getId() { return name; }
+ public long getMemory() { return memory; }
+ public int getShareOrder() { return order; }
+ public boolean isBlacklisted() { return blacklisted; } // UIMA-4142
+
+ static String fmt_s = "%20s %11s %15s %10s %5s %4s %s";
+ String fmt_d = "%20s %11s %15s %10d %5d %4d" ;
- static String fmt_s = "%14s %14s %10s %5s %4s %s";
- String fmt_d = "%14s %14s %10d %5d %4d" ;
public static String header()
{
- return String.format(fmt_s, "Node", "Nodepool", "Memory", "Order", "Free", "Shares\n");
+ return String.format(fmt_s, "Node", "Blacklisted", "Nodepool", "Memory", "Order", "Free", "Shares");
+ }
+
+ public static String separator()
+ {
+ return String.format(fmt_s, "--------------------", "-----------", "---------------", "----------", "-----", "----", "----------");
}
public String toConsole()
@@ -63,17 +73,17 @@ public class RmQueriedMachine
StringBuffer sb = new StringBuffer();
if ( shares == null ) {
- sb.append(String.format(fmt_d, name, nodepoolId, memory, order, 0));
+ sb.append(String.format(fmt_d, name, blacklisted, nodepoolId, memory, order, 0));
sb.append(" [none]");
} else {
int used = 0;
for ( RmQueriedShare s : shares ) {
used += s.getShareOrder();
}
- sb.append(String.format(fmt_d, name, nodepoolId, memory, order, order - used));
+ sb.append(String.format(fmt_d, name, blacklisted, nodepoolId, memory, order, order - used));
String spacer = " ";
- String altSpacer = "\n" + String.format(fmt_s, "", "", "", "", "", ""); // yes, blank, of exactly the right size
+ String altSpacer = "\n" + String.format(fmt_s, "", "", "", "", "", "", ""); // yes, blank, of exactly the right size
for ( RmQueriedShare s : shares ) {
sb.append(spacer);
sb.append(s.toConsole());
@@ -92,6 +102,8 @@ public class RmQueriedMachine
sb.append(" ");
sb.append(nodepoolId);
sb.append(" ");
+ sb.append(Boolean.toString(blacklisted));
+ sb.append(" ");
sb.append(Long.toString(memory));
sb.append(" ");
sb.append(Integer.toString(order));
@@ -116,6 +128,6 @@ public class RmQueriedMachine
public String toString()
{
- return String.format("%14s %12d %3d %3d", name, memory, order, shares == null ? 0 : shares.size());
+ return String.format("%14s %5s %12d %3d %3d", name, blacklisted, memory, order, shares == null ? 0 : shares.size());
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java Mon Jan 19 18:34:14 2015
@@ -35,6 +35,9 @@ public class RmQueriedShare
boolean purged = false;
boolean fixed = false;
boolean initialized = false;
+
+ boolean blacklisted = false;
+
public RmQueriedShare(long job_id, long share_id, int order, long investment_init, long investment_rt)
{
this.jobId = job_id;
@@ -44,10 +47,11 @@ public class RmQueriedShare
this.investmentRt = investment_rt;
}
- public void setEvicted(boolean v){ this.evicted = v; }
- public void setPurged(boolean v){ this.purged = v; }
- public void setFixed(boolean v) { this.fixed = v; }
- public void setInitialized(boolean v) {this.initialized = v; }
+ public void setEvicted(boolean v) { this.evicted = v; }
+ public void setPurged(boolean v) { this.purged = v; }
+ public void setFixed(boolean v) { this.fixed = v; }
+ public void setInitialized(boolean v) { this.initialized = v; }
+ public void setBlacklisted() { this.blacklisted = true; }
public long getJobId() { return this.jobId; }
public long getId() { return this.shareId; }
@@ -63,11 +67,19 @@ public class RmQueriedShare
public String toCompact()
{
- return String.format("%d %d %d %d %d %s %s %s %s", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ if ( blacklisted ) {
+ return String.format("BLACKLISTED %8d %8d %d", jobId, shareId, order);
+ } else {
+ return String.format("%d %d %d %d %d %s %s %s %s", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ }
}
public String toConsole()
{
- return String.format("J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] I[%5s]", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ if ( blacklisted ) {
+ return String.format("J[%8d] S[%8d] O[%d] BLACKLISTED", jobId, shareId, order);
+ } else {
+ return String.format("J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] I[%5s]", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ }
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Mon Jan 19 18:34:14 2015
@@ -81,7 +81,9 @@ public class JobManagerConverter
JobManagerUpdate lastJobManagerUpdate = new JobManagerUpdate();
Map<IRmJob, IRmJob> refusedJobs = new HashMap<IRmJob, IRmJob>();
-
+
+ Map<DuccId, IRmJobState> blacklistedResources = new HashMap<DuccId, IRmJobState>(); // UIMA-4142 to tell OR
+
boolean recovery = false;
public JobManagerConverter(ISchedulerMain scheduler, NodeStability ns)
@@ -113,6 +115,149 @@ public class JobManagerConverter
}
}
+ /**
+ * Purge everything in the world for this job.
+ * UIMA-4142
+ */
+ void blacklistJob(IDuccWork job, long memory, boolean evict)
+ {
+ String methodName = "blacklistJob";
+
+ int order = scheduler.calcShareOrder(memory);
+ Map<DuccId, IResource> all_shares = null;
+ Map<DuccId, IResource> shrunken_shares = null;
+ Map<DuccId, IResource> expanded_shares = null;
+
+ if ( evict ) {
+ all_shares = new LinkedHashMap<DuccId, IResource>();
+ shrunken_shares = new LinkedHashMap<DuccId, IResource>();
+ expanded_shares = new LinkedHashMap<DuccId, IResource>();
+ }
+
+ // first time - everything must go
+ IDuccProcessMap pm = ((IDuccWorkExecutable)job).getProcessMap();
+ for ( IDuccProcess proc : pm.values() ) { // build up Shares from the incoming state
+ NodeIdentity ni = proc.getNodeIdentity();
+ Machine m = scheduler.getMachine(ni);
+ int share_order = 1;
+
+ if ( m != null ) {
+ if ( proc.isActive() || (proc.getProcessState() == ProcessState.Undefined) ) {
+ logger.info(methodName, job.getDuccId(), "blacklist", proc.getDuccId(),
+ "state", proc.getProcessState(), "isActive", proc.isActive(), "isComplete", proc.isComplete());
+ m.blacklist(job.getDuccId(), proc.getDuccId(), order);
+ if ( evict ) {
+ share_order = m.getShareOrder(); // best guess
+ Resource r = new Resource(proc.getDuccId(), proc.getNode(), false, share_order, 0);
+ all_shares.put(proc.getDuccId(), r);
+ shrunken_shares.put(proc.getDuccId(), r);
+ }
+ } else {
+ logger.info(methodName, job.getDuccId(), "whitelist", proc.getDuccId(),
+ "state", proc.getProcessState(), "isActive", proc.isActive(), "isComplete", proc.isComplete());
+ m.whitelist(proc.getDuccId());
+ }
+ }
+ }
+
+ if ( evict && (shrunken_shares.size() > 0) ) {
+ RmJobState rjs = new RmJobState(job.getDuccId(), all_shares, shrunken_shares, expanded_shares);
+ rjs.setDuccType(job.getDuccType());
+ blacklistedResources.put(job.getDuccId(), rjs); // to tell OR
+ }
+ }
+
+ // UIMA-4142
+ void blacklistReservation(IDuccWork job)
+ {
+ String methodName = "blacklistReservation";
+ logger.trace(methodName, job.getDuccId(), "enter");
+
+ IDuccReservationMap drm = ((IDuccWorkReservation) job).getReservationMap();
+
+ for ( IDuccReservation idr : drm.values() ) {
+ NodeIdentity ni = idr.getNodeIdentity();
+ Machine m = scheduler.getMachine(ni);
+ if ( m == null ) { // not known, huh? maybe next epoch it will have checked in
+ logger.warn(methodName, job.getDuccId(), "Problem whitelisting: cannot find machine", ni.getName());
+ } else {
+ m.blacklist(job.getDuccId(), idr.getDuccId(), -1);
+ }
+ }
+
+ }
+
+ // UIMA-4142
+ void blacklist(IDuccWork job, int memory)
+ {
+ String methodName = "blacklist";
+ logger.trace(methodName, job.getDuccId(), "enter");
+
+ switch ( job.getDuccType() ) {
+ case Job:
+ blacklistJob(job, memory, true);
+ break;
+ case Service:
+ case Pop:
+ switch ( ((IDuccWorkService)job).getServiceDeploymentType() )
+ {
+ case uima:
+ case custom:
+ blacklistJob(job, memory, true);
+ break;
+ case other:
+ blacklistJob(job, memory, false);
+ break;
+ }
+ break;
+ case Reservation:
+ blacklistReservation(job);
+ break;
+ default:
+ // illegal - internal error if this happens
+ logger.error(methodName, job.getDuccId(), "Unknown job type", job.getDuccType(), "ignoring in blacklist.");
+ break;
+ }
+ }
+
+ // UIMA-4142
+ void whitelist(IDuccWork job)
+ {
+ String methodName = "whitelist";
+
+ switch ( job.getDuccType() ) {
+ case Job:
+ case Service:
+ case Pop:
+ for ( IDuccProcess idp : ((IDuccWorkJob) job).getProcessMap().values() ) {
+ NodeIdentity ni = idp.getNodeIdentity();
+ Machine m = scheduler.getMachine(ni);
+ if ( m == null ) { // not known, huh? maybe next epoch it will have checked in
+ logger.warn(methodName, job.getDuccId(), "Problem whitelisting: cannot find machine", ni.getName());
+ } else {
+ m.whitelist(idp.getDuccId());
+ }
+ }
+
+ break;
+ case Reservation:
+ for ( IDuccReservation idp : ((IDuccWorkReservation) job).getReservationMap().values() ) {
+ NodeIdentity ni = idp.getNodeIdentity();
+ Machine m = scheduler.getMachine(ni);
+ if ( m == null ) { // not known, huh? maybe next epoch it will have checked in
+ logger.warn(methodName, job.getDuccId(), "Problem whitelisting: cannot find machine", ni.getName());
+ } else {
+ m.whitelist(idp.getDuccId());
+ }
+ }
+ break;
+ default:
+ // illegal - internal error if this happens
+ logger.error(methodName, job.getDuccId(), "Unknown job type", job.getDuccType(), "ignoring in blacklist.");
+ break;
+ }
+ }
+
String getElapsedTime(ITimeWindow w)
{
if ( w == null ) return "0";
@@ -189,7 +334,7 @@ public class JobManagerConverter
return;
} else {
int total_work = toInt(si.getWorkItemsTotal(), scheduler.getDefaultNTasks());
- int completed_work = toInt(si.getWorkItemsCompleted(), 0) + toInt(si.getWorkItemsError(), 0)+ toInt(si.getWorkItemsLost(), 0);
+ int completed_work = toInt(si.getWorkItemsCompleted(), 0) + toInt(si.getWorkItemsError(), 0);
int max_shares = toInt(si.getSharesMax(), Integer.MAX_VALUE);
int existing_max_shares = j.getMaxShares();
@@ -448,8 +593,15 @@ public class JobManagerConverter
j.setResourceClass(rescl);
if ( rescl == null ) {
- // ph darn, we can't continue past this point
+ // oh darn, we can't continue past this point
refuse(j, "Cannot find priority class " + className + " for job");
+
+ // UIMA-4142
+ // However, is this is recovery and we get here, it's because somehow the class definition
+ // got deleted. In this case there might be resources assigned. We must evict if possible.
+ // All affected hosts must be blacklisted. We need to remember all this so we can unblacklist them
+ // if the resources ever become free.
+ blacklist(job, memory);
return false;
}
@@ -689,6 +841,7 @@ public class JobManagerConverter
// must do this independently of isInitialized() since reinit could happen fully between OR pubs
localMap = new DuccWorkMap(); // as if RM had been booted
lastJobManagerUpdate = new JobManagerUpdate();
+ blacklistedResources.clear(); // UIMA-4142
refusedJobs.clear();
first_or_state = true;
}
@@ -739,6 +892,7 @@ public class JobManagerConverter
}
} else {
logger.info(methodName, w.getDuccId(), "Received non-schedulable job, state = ", w.getStateObject());
+ whitelist(w); // UIMA-4142 if blacklisted, clear everything
}
}
@@ -978,6 +1132,10 @@ public class JobManagerConverter
rmJobState.put(j.getId(), rjs);
}
+ // UIMA-4142 Add the blacklist to the mix
+ rmJobState.putAll(blacklistedResources);
+ blacklistedResources.clear();
+
previousJobState = rmJobState;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java Mon Jan 19 18:34:14 2015
@@ -61,6 +61,7 @@ public interface ISchedulerMain
IRmJob getJob(DuccId id);
Share getShare(DuccId id);
Machine getMachine(NodeIdentity id);
+ int calcShareOrder(long mem); // UIMA-4142 expose to JobManagerConverter
void queryMachines();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java Mon Jan 19 18:34:14 2015
@@ -19,18 +19,21 @@
package org.apache.uima.ducc.rm.scheduler;
import java.util.HashMap;
+import java.util.Map;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.admin.event.RmQueriedMachine;
import org.apache.uima.ducc.common.admin.event.RmQueriedShare;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
public class Machine
implements SchedConstants
{
- //private DuccLogger logger = DuccLogger.getLogger(Machine.class, COMPONENT_NAME);
+ private static DuccLogger logger = DuccLogger.getLogger(Machine.class, COMPONENT_NAME);
private String id;
private int hbcounter = 0; // heartbeat counter
@@ -56,6 +59,11 @@ public class Machine
private int virtual_share_order = 0;
private int shares_left = 0;
+ // UIMA-4142
+ // count of shares unavailable because of blacklisting
+ private int blacklisted_shares = 0;
+ private Map<DuccId, Share> blacklistedWork = new HashMap<DuccId, Share>(); // id of process or reservation, share order
+
Node node;
private HashMap<Share, Share> activeShares = new HashMap<Share, Share>();
@@ -81,6 +89,61 @@ public class Machine
return node;
}
+ // UIMA-4142
+ // Black list some number of shres for a specific job and proc. This reduces the number of
+ // schedulable shares until they are whitelisted.
+ public synchronized void blacklist(DuccId jobid, DuccId procid, int nshares)
+ {
+ String methodName = "blacklist";
+
+ if ( ! blacklistedWork.containsKey(procid) ) { // already condemned?
+ if ( nshares == -1 ) nshares = share_order; // whole machine - reservations
+
+ Share s = new Share(procid, this, jobid, nshares);
+ blacklistedWork.put(procid, s);
+ shares_left -= nshares;
+ blacklisted_shares += nshares;
+
+ if ( shares_left < 0 ) {
+ try {
+ throw new IllegalStateException("shares_left went negative");
+ } catch ( Exception e ) {
+ logger.error(methodName, null , e, "shares went negative on", this.id, "share", procid.toString(), "nshares", nshares, "shares_left", shares_left, "share_order", share_order);
+ }
+ }
+ }
+ logger.info(methodName, null, this.id, procid.toString(), procid.getUUID(), "procs", blacklistedWork.size(), "shares left", shares_left, "/", share_order);
+ }
+
+
+ // UIMA-4142
+ // Whitelist shares if they were previously blacklisted, making them available for scheduling.
+ public synchronized void whitelist(DuccId procid)
+ {
+ String methodName = "whitelist";
+ if ( blacklistedWork.containsKey(procid) ) {
+ Share s = blacklistedWork.remove(procid);
+ int nshares = s.getShareOrder();
+ shares_left += nshares;
+ blacklisted_shares -= nshares;
+
+ if ( shares_left > share_order ) {
+ try {
+ throw new IllegalStateException("shares_left exceeds share_order");
+ } catch ( Exception e ) {
+ logger.error(methodName, null , e, "shares left exceeds share_order on", this.id, "share", procid.toString(), "nshares", nshares, "shares_left", shares_left, "share_order", share_order);
+ }
+ }
+ }
+ logger.info(methodName, null, this.id, procid.toString(), procid.getUUID(), "procs", blacklistedWork.size(), "shares left", shares_left, "/", share_order);
+ }
+
+ // UIMA-4142
+ public synchronized boolean isBlacklisted()
+ {
+ return blacklistedWork.size() > 0;
+ }
+
public synchronized void heartbeat_down()
{
hbcounter = 0;
@@ -120,7 +183,8 @@ public class Machine
//
// We use this trick so we can use the "normal" allocation mechanisms for bookeeping without special-casing reservations.
//
- return ( (activeShares.size()) == 0 && (virtual_share_order == share_order));
+ // UIMA-4142, include blacklist considerations
+ return ( (activeShares.size()) == 0 && (virtual_share_order == share_order) && ( !isBlacklisted() ) );
}
/**
@@ -129,6 +193,9 @@ public class Machine
public boolean isFreeable()
{
boolean answer = true;
+ // UIMA-4142, include blacklist considerations
+ if ( isBlacklisted() ) return false;
+
for ( Share s : activeShares.values() ) {
if ( s.isFixed() ) {
return false;
@@ -192,7 +259,7 @@ public class Machine
{
this.share_order = o;
this.shares_left = share_order;
- this.virtual_share_order = share_order;
+ resetVirtualShareOrder(); // UIMA-4142 use common code to calculate this
}
public void setVirtualShareOrder(int o)
@@ -202,7 +269,8 @@ public class Machine
public void resetVirtualShareOrder()
{
- this.virtual_share_order = share_order;
+ // UIMA-4142, include blacklist considerations
+ this.virtual_share_order = share_order - blacklisted_shares;
}
public void removeShare(Share s)
@@ -213,7 +281,7 @@ public class Machine
}
/**
- * How many shares of the given order can I support without preemption? (Virtual, not quantum shares.)
+ * How many shares of the given order can I support without preemption?
*/
public int countFreeShares(int order)
{
@@ -222,7 +290,8 @@ public class Machine
for ( Share s : activeShares.values() ) {
in_use += s.getShareOrder();
}
- return (share_order - in_use) / order ;
+ // UIMA-4142, include blacklist considerations
+ return (share_order - in_use - blacklisted_shares) / order ;
}
public int countFreeShares()
@@ -250,7 +319,8 @@ public class Machine
RmQueriedMachine queryMachine()
{
- RmQueriedMachine ret = new RmQueriedMachine(id, nodepool.getId(), memory, share_order);
+ // UIMA-4142, include blacklist considerations
+ RmQueriedMachine ret = new RmQueriedMachine(id, nodepool.getId(), memory, share_order, isBlacklisted());
for ( Share s : activeShares.values() ) {
RmQueriedShare rqs = new RmQueriedShare(s.getJob().getId().getFriendly(),
s.getId().getFriendly(),
@@ -263,6 +333,17 @@ public class Machine
rqs.setInitialized(s.isInitialized());
ret.addShare(rqs);
}
+
+ for (Share s : blacklistedWork.values() ) {
+ RmQueriedShare rqs = new RmQueriedShare(s.getBlJobId().getFriendly(),
+ s.getId().getFriendly(),
+ s.getShareOrder(),
+ s.getInitializationTime(),
+ s.getInvestment());
+ rqs.setBlacklisted();
+ ret.addShare(rqs);
+ }
+
return ret;
}
@@ -280,12 +361,13 @@ public class Machine
public static String getDashes()
{
- return String.format("%20s %5s %13s %13s %11s %s", "--------------------", "-----", "-------------", "-------------", "-----------", "------ ...");
+ return String.format("%20s %12s %5s %13s %13s %11s %s", "--------------------", "------------", "-----", "-------------", "-------------", "-----------", "------ ...");
}
public static String getHeader()
{
- return String.format("%20s %5s %13s %13s %11s %s", "Name", "Order", "Active Shares", "Unused Shares", "Memory (MB)", "Jobs");
+ // UIMA-4142, include blacklist considerations
+ return String.format("%20s %12s %5s %13s %13s %11s %s", "Name", "Blacklisted", "Order", "Active Shares", "Unused Shares", "Memory (MB)", "Jobs");
}
/**
@@ -347,6 +429,7 @@ public class Machine
unused = share_order - qshares;
}
- return String.format("%20s %5d %13d %13d %11d %s", id, share_order, qshares, unused, (memory/1024), jobs);
+ // UIMA-4142, include blacklist considerations
+ return String.format("%20s %12s %5d %13d %13d %11d %s", id, isBlacklisted(), share_order, qshares, unused, (memory/1024), jobs);
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Mon Jan 19 18:34:14 2015
@@ -36,7 +36,7 @@ import org.apache.uima.ducc.transport.ev
class NodePool
implements SchedConstants
{
- DuccLogger logger = DuccLogger.getLogger(NodePool.class, COMPONENT_NAME);
+ static DuccLogger logger = DuccLogger.getLogger(NodePool.class, COMPONENT_NAME);
String id;
NodePool parent = null;
@@ -52,7 +52,7 @@ class NodePool
HashMap<Node, Machine> allMachines = new HashMap<Node, Machine>(); // all active machines in the system
HashMap<Node, Machine> unresponsiveMachines = new HashMap<Node, Machine>(); // machines with excessive missed heartbeats
HashMap<Node, Machine> offlineMachines = new HashMap<Node, Machine>();
- HashMap<Integer, HashMap<Node, Machine>> machinesByOrder = new HashMap<Integer, HashMap<Node, Machine>>(); // Schedulable, not necessarily fee
+ HashMap<Integer, HashMap<Node, Machine>> machinesByOrder = new HashMap<Integer, HashMap<Node, Machine>>(); // All schedulable machines, not necessarily free
HashMap<String, Machine> machinesByName = new HashMap<String, Machine>(); // by name, for nodepool support
HashMap<String, Machine> machinesByIp = new HashMap<String, Machine>(); // by IP, for nodepool support
@@ -76,9 +76,9 @@ class NodePool
//
// assume 2x16G, 0x32G, 1x48G, 4x64G machines ------------------------------+
// the arrays look like this for each init of the scheduler: v
- int nMachinesByOrder[]; // number of machines of each share order [ 0 2 0 1 4 ] - physical machines
- int vMachinesByOrder[];
- int nSharesByOrder[]; // shares of each size for each share order [ 0 21 9 5 4 ] - collective N Shares for each order
+ int nMachinesByOrder[]; // number of full, free machines of each share order [ 0 2 0 1 4 ] - physical machines
+ int vMachinesByOrder[]; // number of partial machines, indexed by free space
+ int nSharesByOrder[]; // shares of each size for each share order [ 0 21 9 5 4 ] - collective N Shares for each order
//int nFreeSharesByOrder[]; // for each order, the theoretical number of shares to give away [ 0 1 0 3 16 ] - free Q shares per order
int machinesToPreempt[]; // number of full machines to preempt for reservations, by order
@@ -89,7 +89,8 @@ class NodePool
// int shareExpansion[];
Map<Integer, Integer> onlineMachinesByOrder = new HashMap<Integer, Integer>(); // all online machines
- HashMap<Integer, HashMap<Node, Machine>> virtualMachinesByOrder;
+ // Indexed by available free shares, the specific machines with the indicated free space
+ HashMap<Integer, Map<Node, Machine>> virtualMachinesByOrder = new HashMap<Integer, Map<Node, Machine>>(); // UIMA-4142
static int maxorder = 0;
// NodePool(NodePool parent, String id, EvictionPolicy ep, int order)
@@ -563,12 +564,13 @@ class NodePool
}
@SuppressWarnings("unchecked")
- HashMap<Node, Machine> getVirtualMachinesByOrder(int order)
+ Map<Node, Machine> getVirtualMachinesByOrder(int order)
{
- HashMap<Node, Machine> machs;
+ Map<Node, Machine> machs;
if( virtualMachinesByOrder.containsKey(order) ) {
- machs = (HashMap<Node, Machine>) virtualMachinesByOrder.get(order).clone();
+ HashMap<Node, Machine> tmp = (HashMap<Node, Machine>) virtualMachinesByOrder.get(order);
+ machs = (HashMap<Node, Machine>) tmp.clone();
} else {
machs = new HashMap<Node, Machine>();
}
@@ -656,7 +658,7 @@ class NodePool
public synchronized void connectShare(Share s, Machine m, IRmJob j, int order)
{
String methodName = "connectShare";
- logger.trace(methodName, j.getId(), "share", s, "order", order, "machine", m);
+ logger.info(methodName, j.getId(), "share", s, "order", order, "machine", m);
j.assignShare(s);
m.assignShare(s);
rearrangeVirtual(m, order);
@@ -665,18 +667,20 @@ class NodePool
void rearrangeVirtual(Machine m, int order)
{
- //String methodName = "rearrangeVirtual";
+ String methodName = "rearrangeVirtual";
if ( allMachines.containsKey(m.key()) ) {
int v_order = m.getVirtualShareOrder();
int r_order = m.getShareOrder();
+ logger.info(methodName, null, m.getId(), "order", order, "v_order", v_order, "r_order", r_order);
+
if ( v_order == r_order ) {
nMachinesByOrder[r_order]--;
} else {
vMachinesByOrder[v_order]--;
}
- HashMap<Node, Machine> vlist = virtualMachinesByOrder.get(v_order);
+ Map<Node, Machine> vlist = virtualMachinesByOrder.get(v_order);
vlist.remove(m.key());
v_order -= order;
@@ -734,20 +738,45 @@ class NodePool
machinesToPreempt = new int[maxorder + 1];
- virtualMachinesByOrder = new HashMap<Integer, HashMap<Node, Machine>>();
- for ( Integer i : machinesByOrder.keySet() ) {
+ // UIMA-4142 Must set vMachinesByOrder and virtualMachinesByOrder independently of
+ // machinesByOrder because blacklisting can cause v_order != r_order
+ // during reset.
+ virtualMachinesByOrder.clear();
+ for ( Machine m : allMachines.values() ) {
+ m.resetVirtualShareOrder();
+ int v_order = m.getVirtualShareOrder();
+ int r_order = m.getShareOrder();
+
+ Map<Node, Machine> ml = null;
+ if ( v_order == r_order ) {
+ nMachinesByOrder[r_order]++;
+ } else {
+ vMachinesByOrder[v_order]++;
+ }
- @SuppressWarnings("unchecked")
- HashMap<Node, Machine> ml = (HashMap<Node, Machine>) machinesByOrder.get(i).clone();
+ ml = virtualMachinesByOrder.get(v_order);
+ if ( ml == null ) {
+ ml = new HashMap<Node, Machine>();
+ virtualMachinesByOrder.put(v_order, ml);
+ }
- virtualMachinesByOrder.put(i, ml);
- nMachinesByOrder[i] = ml.size();
+ ml.put(m.key(), m);
}
- calcNSharesByOrder();
- for ( Machine m : allMachines.values() ) {
- m.resetVirtualShareOrder();
- }
+ // UIMA 4142 this old calc isn't right any more because blacklisting can cause
+ // v_order != r_order during reset
+ // virtualMachinesByOrder = new HashMap<Integer, HashMap<Node, Machine>>();
+ // for ( Integer i : machinesByOrder.keySet() ) {
+
+ // @SuppressWarnings("unchecked")
+ // HashMap<Node, Machine> ml = (HashMap<Node, Machine>) machinesByOrder.get(i).clone();
+
+ // virtualMachinesByOrder.put(i, ml);
+ // nMachinesByOrder[i] = ml.size();
+ // }
+
+
+ calcNSharesByOrder();
for ( NodePool np : children.values() ) {
np.reset(order);
@@ -1000,6 +1029,14 @@ class NodePool
disable(m, unresponsiveMachines);
}
+ // UIMA-4142
+ // helper for CLI things that refer to things by name only. do we know about anything by this
+ // name? see resolve() in Scheduler.java.
+ boolean hasNode(String n)
+ {
+ return machinesByName.containsKey(n);
+ }
+
String varyoff(String node)
{
Machine m = machinesByName.get(node);
@@ -1542,11 +1579,12 @@ class NodePool
continue; // nothing here to give
}
- HashMap<Node, Machine> machs = getVirtualMachinesByOrder(i);
+ Map<Node, Machine> machs = getVirtualMachinesByOrder(i);
ArrayList<Machine> ml = new ArrayList<Machine>();
ml.addAll(machs.values());
for ( Machine m : ml ) { // look for space
+ if ( m.isBlacklisted() ) continue; // nope
int g = Math.min(needed, m.countFreeShares(order)); // adjust by the order supported on the machine
for ( int ndx= 0; ndx < g; ndx++ ) {
Share s = new Share(m, j, order);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java Mon Jan 19 18:34:14 2015
@@ -35,7 +35,7 @@ public class ResourceClass
implements SchedConstants,
IEntity
{
- private DuccLogger logger = DuccLogger.getLogger(this.getClass(), COMPONENT_NAME);
+ private static DuccLogger logger = DuccLogger.getLogger(ResourceClass.class, COMPONENT_NAME);
private String id;
private Policy policy;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java Mon Jan 19 18:34:14 2015
@@ -185,7 +185,9 @@ public class Scheduler
dramOverride = dramOverride * (1024 * 1024); // convert to KB
}
- idFactory = new DuccIdFactory(1);
+ if ( idFactory == null ) { // UIMA-4142 only remake it on first boot
+ idFactory = new DuccIdFactory(1);
+ }
// try {
// schedImplName = SystemPropertyResolver.getStringProperty("ducc.rm.scheduler", "org.apache.uima.ducc.rm.ClassBasedScheduler");
@@ -203,10 +205,10 @@ public class Scheduler
String class_definitions = SystemPropertyResolver
.getStringProperty(DuccPropertiesResolver
.ducc_rm_class_definitions, "scheduler.classes");
-
class_definitions = System.getProperty("DUCC_HOME") + "/resources/" + class_definitions;
+
try {
- initClasses(class_definitions);
+ initClasses();
} catch ( Exception e ) {
logger.error(methodName, null, e);
throw e;
@@ -266,6 +268,16 @@ public class Scheduler
setInitialized(false); // stop receipt of OR and Agent publications
+ // First we run the logic that reads the configuration, and if it fails, we abort the reconfig without crashing
+ // We'll throw it away because we call init() in a minute, which will do the actual configuration, as if booting
+ try {
+ readConfiguration(); // UIMA-4142
+ } catch (Throwable e) {
+ setInitialized(true);
+ logger.warn(methodName, null, "Reconfiguration aborted:", e.toString());
+ return "Reconfiguration failed: " + e.toString();
+ }
+
HashMap<Node, Machine> offlineMachines = new HashMap<Node, Machine>();
for (NodePool np : nodepools) {
offlineMachines.putAll(np.getOfflineMachines());
@@ -404,7 +416,7 @@ public class Scheduler
/**
* Calculate share order, given some memory size in GB (as in from a job spec)
*/
- int calcShareOrder(long mem)
+ public int calcShareOrder(long mem)
{
// Calculate its share order
mem = mem * 1024 * 1024; // to GB
@@ -416,95 +428,6 @@ public class Scheduler
return share_order;
}
-// /**
-// * Use the NodeIdentity to infer my the domain name.
-// *
-// * Itertate through the possible names - if one of them has a '.'
-// * the we have to assume the following stuff is the domain name.
-// * We only get one such name, so we give up the search if we find
-// * it.
-// */
-// static String cached_domain = null;
-// private String getDomainName()
-// {
-// String methodName = "getDomainName";
-
-// String answer = System.getProperty("SIM_RM_DOMAIN"); // for the simulation wrapper, to replay logs from other domains correctly
-// if ( answer != null ) {
-// return answer;
-// }
-
-// if ( cached_domain != null ) return cached_domain;
-// try {
-// NodeIdentity ni = new NodeIdentity();
-// for ( IIdentity id : ni.getNodeIdentities()) {
-// String n = id.getName();
-// int ndx = n.indexOf(".");
-// if ( ndx > 0 ) {
-// cached_domain = n.substring(ndx + 1);
-// return cached_domain;
-// }
-// }
-// } catch (Exception e) {
-// // TODO Auto-generated catch block
-// logger.warn(methodName, null, "Cannot create my own node identity:", e);
-// }
-// return null; // crappy config if this happens, some stuff may not match nodepools and
-// // nothing to do about it.
-// }
-
-// Map<String, String> readNodepoolFile(String npfile)
-// {
-// String methodName = "readNodepoolFile";
-// String my_domain = getDomainName();
-// String ducc_home = System.getProperty("DUCC_HOME");
-// npfile = ducc_home + "/resources/" + npfile;
-
-// logger.info(methodName, null, "Domain name:", my_domain);
-// Map<String, String> response = new HashMap<String, String>();
-
-// try {
-// BufferedReader br = new BufferedReader(new FileReader(npfile));
-// String node = "";
-// while ( (node = br.readLine()) != null ) {
-// int ndx = node.indexOf("#");
-// if ( ndx >= 0 ) {
-// node = node.substring(0, ndx);
-// }
-// node = node.trim();
-// if (node.equals("") ) {
-// continue;
-// }
-
-// if ( node.startsWith("import") ) {
-// String[] tmp = node.split("\\s");
-// response.putAll(readNodepoolFile(tmp[1]));
-// continue;
-// }
-// response.put(node, node);
-
-// // include fully and non-fully qualified names to allow sloppiness of config
-// ndx = node.indexOf(".");
-// String dnode;
-// if ( ndx >= 0 ) {
-// dnode = node.substring(0, ndx);
-// response.put(dnode, dnode);
-// } else if ( my_domain != null ) {
-// dnode = node + "." + my_domain;
-// response.put(dnode, dnode);
-// }
-// }
-// br.close();
-
-// } catch (FileNotFoundException e) {
-// throw new SchedulingException(null, "Cannot open NodePool file \"" + npfile + "\": file not found.");
-// } catch (IOException e) {
-// throw new SchedulingException(null, "Cannot read NodePool file \"" + npfile + "\": I/O Error.");
-// }
-
-// return response;
-// }
-
/**
* Collect all the classes served by the indicated nodepool (property set). This fills
* in the 'ret' map from the parameter 'dp' and recursive calls to the children in dp.
@@ -565,17 +488,34 @@ public class Scheduler
}
}
- void initClasses(String filename)
+ // UIMA-4142 better modularize this code
+ NodeConfiguration readConfiguration()
+ throws Exception
{
- String methodName = "initClasses";
+ String class_definitions = SystemPropertyResolver
+ .getStringProperty(DuccPropertiesResolver
+ .ducc_rm_class_definitions, "scheduler.classes");
+ class_definitions = System.getProperty("DUCC_HOME") + "/resources/" + class_definitions;
String me = Scheduler.class.getName() + ".Config";
DuccLogger initLogger = new DuccLogger(me, COMPONENT_NAME);
- configuration = new NodeConfiguration(filename, initLogger); // UIMA-4142 make the config global
+ NodeConfiguration nc = new NodeConfiguration(class_definitions, initLogger); // UIMA-4142 make the config global
+ nc.readConfiguration();
+ return nc; // UIMA-4142
+ }
+
+ // UIMA-4142, don't pass in class def file, instead use common readConfiguration
+ void initClasses()
+ {
+ String methodName = "initClasses";
+
try {
- configuration.readConfiguration(); // UIMA-4142
+ configuration = readConfiguration();
} catch (Throwable e) {
+ // RM boot. We must abort being has we have no prior working configuration to fall back to.
logger.error(methodName, null, e);
logger.error(methodName, null, "Scheduler exits: unable to read configuration.");
+ System.out.println("Scheduler exits: unable to read configuration.");
+ e.printStackTrace();
System.exit(1);
}
@@ -1146,22 +1086,42 @@ public class Scheduler
}
}
+ /**
+ * User passed us a node by name. Maybe did and maybe didn't qualify it.
+ * Maybe the node checked in qualified maybe it didn't. Here we try to find
+ * something that kind of matches.
+ * UIMA-4142. Technically a bug on vary-on and vary-off but found and fixed as part of
+ * the indicated Jira.
+ */
+ synchronized String resolve(String node)
+ {
+ NodePool np = nodepoolsByNode.get(node);
+ if ( np == null ) return null; // indexed by long and short so if not found we're stuck
+
+ if ( np.hasNode(node) ) return node; // he knows it by this name we're done
+
+ int ndx = node.indexOf(".");
+ if ( ndx > 0 ) {
+ // np MUST know it by either long or short or it wouldn't be in nodepoolsByNode
+ // so it must be short
+ return node.substring(0, ndx);
+ } else {
+ // and vice-versa, it must be the long
+ return shortToLongNode.get(node);
+ }
+ }
+
public synchronized String varyon(String[] nodes)
{
String methodName = "varyon";
StringBuffer reply = new StringBuffer();
for (String n : nodes ) {
- if ( shortToLongNode.containsKey(n) ) { // internally everything is by 'long'
- n = shortToLongNode.get(n);
- }
-
- NodePool np = nodepoolsByNode.get(n);
- if ( np == null ) {
- reply.append("No nodepool found for node ");
- reply.append(n);
- reply.append("\n");
+ n = resolve(n);
+ if ( n == null ) {
+ reply.append("VaryOn: The node cannot be found in the RM.\n");
} else {
+ NodePool np = nodepoolsByNode.get(n); // if null, resolve will fail
String repl = np.varyon(n);
logger.info(methodName, null, repl);
reply.append(repl);
@@ -1177,16 +1137,11 @@ public class Scheduler
StringBuffer reply = new StringBuffer();
for (String n : nodes ) {
- if ( shortToLongNode.containsKey(n) ) { // internally everything is by 'long'
- n = shortToLongNode.get(n);
- }
-
- NodePool np = nodepoolsByNode.get(n);
- if ( np == null ) {
- reply.append("No nodepool found for node ");
- reply.append(n);
- reply.append("\n");
- } else {
+ n = resolve(n);
+ if ( n == null ) {
+ reply.append("VaryOff: The node cannot be found in the RM.\n");
+ } else {
+ NodePool np = nodepoolsByNode.get(n); // if null, resolve will fail
String repl = np.varyoff(n);
logger.info(methodName, null, repl);
reply.append(repl);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java Mon Jan 19 18:34:14 2015
@@ -37,8 +37,9 @@ public class Share
{
//private transient DuccLogger logger = DuccLogger.getLogger(Share.class, COMPONENT_NAME);
private Machine machine; // machine associatede with this share, assigned after "what-of"
- private DuccId id; // unique *within this machine* assigned after "what-of"
- private IRmJob job; // job executing in this share, if any, assigned after "what-of"
+ private DuccId id = null; // unique *within this machine* assigned after "what-of"
+ private IRmJob job = null;; // job executing in this share, if any, assigned after "what-of"
+ private DuccId bljobid = null; // UIMA-4142 ID of blacklisted job
private int share_order; // may not be same as machine's order
private ITimeWindow init_time = null; // how much time this process spends initializing
@@ -68,6 +69,21 @@ public class Share
this.id = id;
this.machine = machine;
this.job = job;
+ this.bljobid = null; // UIMA-4142
+ this.share_order = share_order;
+ }
+
+ /**
+ * UIMA-4142
+ * This constructor is for a blacklisted share. We can't make a job but we know what the
+ * job's id would have been if we could.
+ */
+ public Share(DuccId id, Machine machine, DuccId jobid, int share_order)
+ {
+ this.id = id;
+ this.machine = machine;
+ this.job = null;
+ this.bljobid = jobid; // UIMA-4142
this.share_order = share_order;
}
@@ -79,6 +95,7 @@ public class Share
this.machine = machine;
this.id = Scheduler.newId();
this.job = job;
+ this.bljobid = null; // UIMA-4142
this.share_order = share_order;
}
@@ -134,6 +151,18 @@ public class Share
return id;
}
+ // UIMA-4142
+ public boolean isBlacklisted()
+ {
+ return bljobid != null;
+ }
+
+ // UIMA-4142
+ public DuccId getBlJobId()
+ {
+ return bljobid;
+ }
+
Machine getMachine()
{
return machine;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java?rev=1653071&r1=1653070&r2=1653071&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java Mon Jan 19 18:34:14 2015
@@ -285,6 +285,7 @@ public class DuccProcess implements IDuc
switch(processState) {
case Initializing:
case Running:
+ case Starting: // uima-4142 This state found missing. Added by jrc 2015-01-15
retVal = true;
break;
}