You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/04/27 22:40:29 UTC
svn commit: r1676375 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/
uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/
Author: challngr
Date: Mon Apr 27 20:40:28 2015
New Revision: 1676375
URL: http://svn.apache.org/r1676375
Log:
UIMA-4275 Class-based fair-share caps.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IEntity.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java Mon Apr 27 20:40:28 2015
@@ -97,13 +97,12 @@ public class NodeConfiguration
defaultFairShareClass.put("policy", "FAIR_SHARE");
defaultFairShareClass.put("weight", "100");
defaultFairShareClass.put("priority", "10");
- defaultFairShareClass.put("fair-share-cap", Integer.toString(Integer.MAX_VALUE));
- defaultFairShareClass.put("cap", "<optional>");
+ defaultFairShareClass.put("cap", Integer.toString(Integer.MAX_VALUE));
defaultFairShareClass.put("expand-by-doubling", ""+SystemPropertyResolver.getBooleanProperty("ducc.rm.expand.by.doubling", true));
defaultFairShareClass.put("initialization-cap", ""+SystemPropertyResolver.getIntProperty("ducc.rm.initialization.cap", 2));
defaultFairShareClass.put("use-prediction", ""+SystemPropertyResolver.getBooleanProperty("ducc.rm.prediction", true));
defaultFairShareClass.put("prediction-fudge", ""+SystemPropertyResolver.getIntProperty("ducc.rm.prediction.fudge", 60000));
- defaultFairShareClass.put("max-processes", "<optional>");
+ defaultFairShareClass.put("max-processes", "<optional>"); // this is deprecated. it remains here to ease migration to DUCC 2.0. It has no effect.
defaultFairShareClass.put("nodepool", "<required>");
defaultFairShareClass.put("users", "<optional>");
defaultFairShareClass.put("debug", "fixed");
@@ -122,9 +121,8 @@ public class NodeConfiguration
defaultFixedShareClass.put("policy", "FIXED_SHARE");
defaultFixedShareClass.put("priority", "5");
defaultFixedShareClass.put("default", "<optional>");
- defaultFixedShareClass.put("max-processes", "<optional>");
- defaultFixedShareClass.put("max-allotment", "<optional>");
- defaultFixedShareClass.put("cap", "<optional>");
+ defaultFixedShareClass.put("max-processes", "<optional>"); // this is deprecated. it remains here to ease migration to DUCC 2.0. It has no effect.
+ defaultFixedShareClass.put("cap", "<optional>"); // this is deprecated. it remains here to ease migration to DUCC 2.0. It has no effect.
defaultFixedShareClass.put("nodepool", "<required>");
defaultFixedShareClass.put("users", "<optional>");
@@ -136,9 +134,8 @@ public class NodeConfiguration
defaultReserveClass.put("policy", "RESERVE");
defaultReserveClass.put("priority", "1");
defaultReserveClass.put("default", "<optional>");
- defaultReserveClass.put("max-machines", "<optional>");
- defaultReserveClass.put("max-allotment", "<optional>");
- defaultReserveClass.put("cap", "<optional>");
+ defaultReserveClass.put("max-machines", "<optional>"); // this is deprecated. it remains here to ease migration to DUCC 2.0. It has no effect.
+ defaultReserveClass.put("cap", "<optional>"); // this is deprecated. it remains here to ease migration to DUCC 2.0. It has no effect.
defaultReserveClass.put("nodepool", "<required>");
defaultReserveClass.put("users", "<optional>");
defaultReserveClass.put("enforce", "true");
@@ -1154,8 +1151,7 @@ public class NodeConfiguration
printProperty("Priority", cl.get("priority"));
printProperty("Weight", cl.get("weight"));
printProperty("Debug", cl.get("debug"));
- printProperty("Fair-Share Cap", cl.get("fair-share-cap"));
- printDeprecatedProperty("Cap", cl.get("cap"), "IGNORED Use max-allotment = [mem in GB] instead.");
+ printProperty("Cap", cl.get("cap"));
printProperty("Expand By Doubling", cl.get("expand-by-doubling"));
printProperty("Initialization Cap", cl.get("initialization-cap"));
printProperty("Use Prediction", cl.get("use-prediction"));
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Mon Apr 27 20:40:28 2015
@@ -409,7 +409,8 @@ public class JobManagerConverter
return false; // so we don't add it to global tables
} else {
DuccId id = proc.getDuccId();
- Share s = new Share(id, m, j, m.getShareOrder()); // guess share order; scheduler will reset when it recovers job
+ Share s = new Share(id, m, j, m.getShareOrder()); // guess share order; scheduler will reset when it recovers job
+ // UIMA-4275 include depth
long mem = proc.getResidentMemory();
long investment = proc.getWiMillisInvestment();
logger.info(methodName, j.getId(), "Assigning share in state", state, "pid", pid, "for recovery", s.toString());
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IEntity.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IEntity.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IEntity.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IEntity.java Mon Apr 27 20:40:28 2015
@@ -56,9 +56,10 @@ interface IEntity
boolean canUseBonus(int order); // can I use one more share of this size
- int calculateCap(int order, int basis); // The entity must work out any caps that may restrict the counts
+ int calculateCap(); // The entity must work out any caps that may restrict the counts
+ // UIMA-4275 Simplify (delete) args
- long getTimestamp(); // for tiebreaks
+ long getTimestamp(); // for tiebreaks
Comparator<IEntity> getApportionmentSorter();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IRmJob.java Mon Apr 27 20:40:28 2015
@@ -280,4 +280,9 @@ public interface IRmJob
// counted afresh in the current scheduling cycle, for allotments
public int countOccupancy(); // UIMA-4275
+ // UIMA-4275 Must lose some number of shares unconditionally
+ public void shrinkBy(int howmany);
+
+ // UIMA-4275
+ public boolean exceedsFairShareCap();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java Mon Apr 27 20:40:28 2015
@@ -250,6 +250,11 @@ public class Machine
this.memory = memory;
}
+ public int getNodepoolDepth() // UIMA-4275
+ {
+ return nodepool.getDepth();
+ }
+
public int getShareOrder()
{
return share_order;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Mon Apr 27 20:40:28 2015
@@ -711,6 +711,7 @@ class NodePool
}
void rearrangeVirtual(Machine m, int order)
+
{
String methodName = "rearrangeVirtual";
if ( allMachines.containsKey(m.key()) ) {
@@ -726,6 +727,16 @@ class NodePool
}
Map<Node, Machine> vlist = virtualMachinesByOrder.get(v_order);
+ if ( vlist == null ) {
+ // Delivered under UIMA-4275 as that is when I decided to try to avoid NPE here.
+ //
+ // This is fatal, the internal records are all wrong. Usually this is because of some
+ // external snafu, such as mixing and matching ducc clusters on the same broker.
+ // There's really not much we can do though. There's a good chance that continuing
+ // will cause NPE elsewhere. Maybe we can just ignore it and let it leak?
+ logger.error(methodName, null, "ERROR: bad virtual machine list.", m.getId(), "order", order, "v_order", v_order, "r_order", r_order);
+ return;
+ }
vlist.remove(m.key());
v_order -= order;
@@ -1521,9 +1532,10 @@ class NodePool
}
/**
- * Trying again.
+ * Here we tell the NP how much we need cleared up. It will look around and try to do that.
+ * @deprecated No longer used, the doEvictions code in NodepoolScheduler handles evictions by itself.
+ * Keeping this for a while for reference. UIMA-4275
*/
- //^jrc
void doEvictionsByMachine(int [] neededByOrder, boolean force)
{
String methodName = "doEvictions";
@@ -1605,6 +1617,7 @@ class NodePool
int needed = (counted - current);
int order = j.getShareOrder();
int given = 0;
+ boolean expansionStopped = false; // UIMA-4275
logger.info(methodName, j.getId(), "counted", counted, "current", current, "needed", needed, "order", order, "given", given);
@@ -1623,13 +1636,19 @@ class NodePool
if ( m.isBlacklisted() ) continue; // nope
int g = Math.min(needed, m.countFreeShares(order)); // adjust by the order supported on the machine
for ( int ndx= 0; ndx < g; ndx++ ) {
- Share s = new Share(m, j, order);
- connectShare(s, m, j, order);
- logger.info(methodName, j.getId(), "Connecting new share", s.toString());
- //j.assignShare(s);
- //m.assignShare(s);
- //rearrangeVirtual(m, order);
- //allShares.put(s, s);
+ if ( j.exceedsFairShareCap() ) { // UIMA-4275
+ // can't take any more shares, probably because of caps
+ expansionStopped = true;
+ break whatof;
+ } else {
+ Share s = new Share(m, j, order);
+ connectShare(s, m, j, order);
+ logger.info(methodName, j.getId(), "Connecting new share", s.toString());
+ //j.assignShare(s);
+ //m.assignShare(s);
+ //rearrangeVirtual(m, order);
+ //allShares.put(s, s);
+ }
}
given += g;
@@ -1644,7 +1663,7 @@ class NodePool
//calcNSharesByOrder();
}
- if ( needed > 0 ) {
+ if ( (needed > 0) && ( !expansionStopped ) ) { // UIMA-4275
for ( NodePool np : getChildrenAscending() ) {
StringBuffer sb = new StringBuffer();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java Mon Apr 27 20:40:28 2015
@@ -19,7 +19,6 @@
package org.apache.uima.ducc.rm.scheduler;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -377,7 +376,8 @@ public class NodepoolScheduler
return String.format(sb.toString(), vals);
}
- protected void apportion_qshares(List<IEntity> entities, int[] vshares, int total_shares, String descr)
+ // UIMA-4275 Don't pass in total shares any more, we used only for class caps, which work differently now.
+ protected void apportion_qshares(List<IEntity> entities, int[] vshares, String descr)
{
String methodName = "apportion_qshares";
boolean shares_given = false;
@@ -493,10 +493,18 @@ public class NodepoolScheduler
int des = deserved.get(e); // total deserved this round QShares
int gpr = given_per_round.get(e); // total given this round
int mpr = Math.max(0, des-gpr); // max this round, deserved less what I aleady was given
- int tgiven = Math.min(mpr, (int) Math.floor(dgiven)); // what is calculated, capped by what I alreay have
- int cap = e.calculateCap(o, total_shares); // get caps, if any
+ //int tgiven = Math.min(mpr, (int) Math.floor(dgiven)); // what is calculated, capped by what I alreay have
+ // UIMA-4275, floor to ciel. Below with tgiven and rgiven we deal with ther emainder also. The floor plus the residual below
+ // seemed really agressive and some small allocation were working out to 0 when they shouldn't
+ int tgiven = Math.min(mpr, (int) Math.ceil(dgiven)); // what is calculated, capped by what I alreay have
+ int cap = e.calculateCap(); // get caps, if any, in qshares (simplified in UIMA-4275)
logger.trace(methodName, null, descr, "O", o, ":", e.getName(), "Before caps, given", tgiven, "cap", cap);
+ if ( gbo[0] >= cap ) { // UIMA-4275
+ logger.trace(methodName, null, descr, "O", o, "Entity", e.getName(), "cap prevents further allocation.");
+ continue;
+ }
+
int given = tgiven / o; // tentatively given, back to NShares
int rgiven = tgiven % o; // residual - remainder
//int twanted = wbo[0] + gbo[0]; // actual wanted: still wanted plus alredy given
@@ -545,11 +553,11 @@ public class NodepoolScheduler
if ( allweights <=0 ) break; // JRC JRC
}
- // Remove entities that have everything they want
+ // Remove entities that have everything they want or could otherwise get
Iterator<IEntity> iter = working.iterator();
while ( iter.hasNext() ) {
IEntity e = iter.next();
- if ( e.getWantedByOrder()[0] == 0 ) {
+ if ( (e.getWantedByOrder()[0] == 0) || (e.getGivenByOrder()[0] >= e.calculateCap()) ) { // UIMA-4275, checking fair-share cap
// logger.info(methodName, null, descr, e.getName(), "reaped, nothing more wanted:", fmtArray(e.getWantedByOrder()));
iter.remove();
}
@@ -585,10 +593,10 @@ public class NodepoolScheduler
} while ( shares_given );
if ( logger.isTrace() ) {
- logger.trace(methodName, null, descr, "Final before bonus:");
+ logger.info(methodName, null, descr, "Final before bonus:");
for ( IEntity e : entities ) {
int[] gbo = e.getGivenByOrder();
- logger.trace(methodName, null, descr, String.format("%12s %s", e.getName(), fmtArray(gbo)));
+ logger.info(methodName, null, descr, String.format("%12s %s", e.getName(), fmtArray(gbo)));
}
}
@@ -617,6 +625,7 @@ public class NodepoolScheduler
// UIMA-4065
while ( (e.canUseBonus(o) ) && (vshares[o] > 0) ) {
gbo[o]++;
+ gbo[0]++;
removeSharesByOrder(vshares, nshares, 1, o);
given = true;
break;
@@ -641,7 +650,7 @@ public class NodepoolScheduler
logger.debug(methodName, null, descr, "Final apportionment:");
for ( IEntity e : entities ) {
int[] gbo = e.getGivenByOrder(); // nshares
- logger.debug(methodName, null, descr, String.format("%12s gbo%s", e.getName(), fmtArray(gbo)));
+ logger.debug(methodName, null, descr, String.format("%12s gbo %s", e.getName(), fmtArray(gbo)));
}
logger.debug(methodName, null, descr, "vshares", fmtArray(vshares));
logger.debug(methodName, null, descr, "nshares", fmtArray(nshares));
@@ -684,7 +693,6 @@ public class NodepoolScheduler
// pull the counts. these don't get updated by the counting routines per-se. after doing counting the np's are
// expected to do the 'what-of' calculations that do acutall allocation and which update the counts
int[] vshares = np.cloneVMachinesByOrder();
- int total_shares = np.countTotalShares(); // for caps, Qshares
ArrayList<IEntity> l = new ArrayList<IEntity>();
l.addAll(rcs);
@@ -693,14 +701,10 @@ public class NodepoolScheduler
e.initWantedByOrder((ResourceClass) e);
}
- apportion_qshares((List<IEntity>) l, vshares, total_shares, methodName);
+ apportion_qshares((List<IEntity>) l, vshares, methodName); // UIMA-4275, remove 'total'
int sum_of_weights = 0;
for ( ResourceClass rc : rcs ) {
- rc.markSubpoolCounted(); // Remember that we've attempted to apportion shares to this guy.
- // This is specific to rc only, not IEntity, so we don't
- // generalize this into apportion_shares.
-
sum_of_weights += rc.getShareWeight(); // see next loop
}
@@ -710,7 +714,7 @@ public class NodepoolScheduler
// for each rc. Here is a great place to do that calculation.
//
for ( ResourceClass rc : rcs ) {
- int fair_share = (int) Math.floor(total_shares * ( (double) rc.getShareWeight() / sum_of_weights ));
+ int fair_share = (int) Math.floor(np.countTotalShares() * ( (double) rc.getShareWeight() / sum_of_weights ));
rc.setPureFairShare(fair_share);
}
}
@@ -729,6 +733,11 @@ public class NodepoolScheduler
return false; // nothing to do ...
}
+ // if ( rc.getName().equals("urgent") ) {
+ // int stop_here = 1;
+ // stop_here++;
+ // }
+
int[] vshares = rc.getGivenByOrder(); // assigned in countClassShares
HashMap<User, User> users = new HashMap<User, User>(); // get a map of all users in this class by examining the curent jobs
@@ -741,7 +750,7 @@ public class NodepoolScheduler
ArrayList<IEntity> l = new ArrayList<IEntity>();
l.addAll(users.values());
- apportion_qshares((List<IEntity>) l, vshares, 0, methodName);
+ apportion_qshares((List<IEntity>) l, vshares, methodName); // UIMA-4275 remove 'total'
//
// For final eviction if needed, calculate the pure uncapped un-bonused count for
@@ -776,7 +785,7 @@ public class NodepoolScheduler
e.initWantedByOrder(rc);
}
- apportion_qshares((List<IEntity>) l, vshares, 0, methodName);
+ apportion_qshares((List<IEntity>) l, vshares, methodName); // UIMA-4275, remove "total"
//
// For final eviction if needed, calculate the pure uncapped un-bonused count for
@@ -939,11 +948,11 @@ public class NodepoolScheduler
{
String methodName = "howMuchFairShare";
if ( logger.isTrace() ) {
- logger.trace(methodName, null, "Scheduling FAIR SHARE for these classes:");
- logger.trace(methodName, null, " ", ResourceClass.getHeader());
- logger.trace(methodName, null, " ", ResourceClass.getDashes());
+ logger.info(methodName, null, "Scheduling FAIR SHARE for these classes:");
+ logger.info(methodName, null, " ", ResourceClass.getHeader());
+ logger.info(methodName, null, " ", ResourceClass.getDashes());
for ( ResourceClass pc : rcs ) {
- logger.trace(methodName, null, " ", pc.toString());
+ logger.info(methodName, null, " ", pc.toString());
}
}
@@ -1690,13 +1699,14 @@ public class NodepoolScheduler
}
int neededByOrder[] = NodePool.makeArray(); // for each order, how many N-shares do I want to add?
- int total_needed = 0;
+ // int total_needed = 0;
+ Map<IRmJob, Integer> overages = new HashMap<IRmJob, Integer>(); // UIMA-4275
for ( ResourceClass cl : resourceClasses.values() ) {
if ( cl.getNodepoolName().equals(nodepool.getId()) && (cl.getAllJobs().size() > 0) ) {
HashMap<IRmJob, IRmJob> jobs = cl.getAllJobs();
String npn = cl.getNodepoolName();
- logger.trace(methodName, null, String.format("%12s %7s %7s %6s %5s", npn, "Counted", "Current", "Needed", "Order"));
+ logger.info(methodName, null, String.format("%12s %7s %7s %6s %5s", npn, "Counted", "Current", "Needed", "Order"));
for ( IRmJob j : jobs.values() ) {
int counted = j.countNSharesGiven(); // allotment from the counter
@@ -1708,20 +1718,41 @@ public class NodepoolScheduler
// if needed < 0, that's shares we need to dump because the
// counts say so.
// if needed == 0 then clearly nothing
- needed = Math.abs(needed);
- // needed = Math.max(0, needed);
- logger.trace(methodName, j.getId(), String.format("%12s %7d %7d %6d %5d", npn, counted, current, needed, order));
- neededByOrder[order] += needed;
- total_needed += needed;
+ if ( needed < 0 ) {
+ // UIMA-4275 these guys must be forced to shrink
+ overages.put(j, -needed);
+ } else {
+ // needed = Math.abs(needed);
+ // needed = Math.max(0, needed);
+
+ logger.info(methodName, j.getId(), String.format("%12s %7d %7d %6d %5d", npn, counted, current, needed, order));
+ neededByOrder[order] += needed;
+ // total_needed += needed;
+ }
}
}
}
- logger.debug(methodName, null, nodepool.getId(), "NeededByOrder before any eviction:", Arrays.toString(neededByOrder));
- if ( (nodepool.countOccupiedShares() > 0) && (total_needed > 0) ) {
- nodepool.doEvictionsByMachine(neededByOrder, false);
- }
+
+ // Every job in overages is required to lose the indicated number of share. If this is done optimally it
+ // will leave suficcient space for the counted shares of all the expansions. Therein lies the rub.
+ //
+ // The older code below does its best to make space for the 'needed' array but it fails to fully evict
+ // an over-deployed job in a number of situations. The loop here is going to rely on defragmentation, which
+ // we did not have originally to do final cleanup. The job will be asked to dump its extra processes according
+ // to the mechanism in its shrinkBy() method. See that method for details.
+ // UIMA-4275
+ for (IRmJob j : overages.keySet()) {
+ j.shrinkBy(overages.get(j));
+ }
+
+ // First we try to make enough space in the right places for under-allocation jobs
+ // logger.debug(methodName, null, nodepool.getId(), "NeededByOrder before any eviction:", Arrays.toString(neededByOrder));
+ // if ( (nodepool.countOccupiedShares() > 0) && (total_needed > 0) ) {
+ // nodepool.doEvictionsByMachine(neededByOrder, false);
+
+
}
/**
@@ -2852,6 +2883,10 @@ public class NodepoolScheduler
User u2 = j2.getUser();
s2wealth = u2.getShareWealth();
+ if ( s2wealth == s1wealth ) {
+ return RmJob.compareInvestment(s1, s2); // UIMA-4275
+ }
+
return s2wealth - s1wealth;
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java Mon Apr 27 20:40:28 2015
@@ -47,9 +47,8 @@ public class ResourceClass
private int share_quantum; // for limits, to convert shares to GB
private int max_allotment; // All allocation policies, max in GB
- // for shares, this caps shares
- private int absolute_cap; // max shares or machines this class can hand out
- private double percent_cap; // max shares or machines this class can hand out as a percentage of all shares
+ // for shares, this caps shares UIMA-4275
+ private int fair_share_cap; // max shares or machines this class can hand out
private int true_cap; // set during scheduling, based on actual current resource availability
private int pure_fair_share; // the unmodified fair share, not counting caps, and not adding in bonuses
@@ -62,9 +61,6 @@ public class ResourceClass
private NodePool nodepool = null;
- // private HashMap<Integer, Integer> nSharesByOrder = new HashMap<Integer, Integer>(); // order, N shares of that order
- private boolean subpool_counted = false;
-
// the physical presence of nodes in the pool is somewhat dynamic - we'll store names only, and generate
// a map of machines on demand by the schedler from currently present machnes
private String nodepoolName = null;
@@ -108,24 +104,18 @@ public class ResourceClass
if ( policy != Policy.FAIR_SHARE ) {
this.max_allotment = props.getIntProperty("max-allotment", Integer.MAX_VALUE);
+ if ( max_allotment == 0 ) max_allotment = Integer.MAX_VALUE; // UIMA-4275, remember to set default if allotment is 0
}
if ( policy == Policy.RESERVE ) {
this.enforce_memory = props.getBooleanProperty("enforce", true);
}
- this.absolute_cap = Integer.MAX_VALUE;
- this.percent_cap = 1.0;
+ this.fair_share_cap = Integer.MAX_VALUE; // UIMA-4275
if ( this.policy == Policy.FAIR_SHARE ) {
- String cap = props.getStringProperty("fair-share-cap");
- if ( cap.endsWith("%") ) {
- int t = Integer.parseInt(cap.substring(0, cap.length()-1));
- this.percent_cap = (t * 1.0 ) / 100.0;
- } else {
- absolute_cap = Integer.parseInt(cap);
- if (absolute_cap == 0) absolute_cap = Integer.MAX_VALUE;
- }
+ fair_share_cap = props.getIntProperty("cap", Integer.MAX_VALUE);
+ if (fair_share_cap == 0) fair_share_cap = Integer.MAX_VALUE;
this.share_weight = props.getIntProperty("weight");
if ( props.containsKey("expand-by-doubling") ) {
@@ -244,12 +234,10 @@ public class ResourceClass
return true_cap;
}
- public double getPercentCap() {
- return percent_cap;
- }
- public int getAbsoluteCap() {
- return absolute_cap;
+ // get the absolute cap, in GB
+ private int getFairShareCap() {
+ return fair_share_cap;
}
public int getAllotment(IRmJob j)
@@ -289,10 +277,6 @@ public class ResourceClass
* See if the total memory for job 'j' plus the occupancy of the 'jobs' exceeds 'max'
* Returns 'true' if occupancy is exceeded, else returns 'false'
* UIMA-4275
- *
- * NOTE: After discussion this may not be the ideal place for this, in favor of a global
- * cap on all NPShares, but I suspect it will come back to haunt us that we need
- * the finer granularity of class-based caps, so the logic is staying here for now
*/
private boolean occupancyExceeded(int max, IRmJob j, Map<IRmJob, IRmJob> jobs)
{
@@ -350,6 +334,15 @@ public class ResourceClass
}
}
+ // UIMA-4275
+ public boolean fairShareCapExceeded(IRmJob j)
+ {
+ if ( policy != Policy.FAIR_SHARE ) return false;
+
+ if ( j.getShareOrder() + countActiveShares() > calculateCap() ) return true;
+ return false;
+ }
+
/**
* Return my share weight, if I have any jobs of the given order or less. If not,
* return 0;
@@ -386,6 +379,7 @@ public class ResourceClass
int wbo = getWantedByOrder()[order]; // calculated by caller so we don't need to check caps
int gbo = getGivenByOrder()[order];
+ if ( getGivenByOrder()[0] >= calculateCap() ) return false; // don't exceed cap UIMA-4275
//
// we want to ask the nodepool and its subpools:
// how many open shares of "order" will you have after we give way
@@ -427,14 +421,8 @@ public class ResourceClass
{
//class_shares = 0;
given_by_order = null;
- subpool_counted = false;
}
- public void markSubpoolCounted()
- {
- subpool_counted = true;
- }
-
void addJob(IRmJob j)
{
allJobs.put(j, j);
@@ -542,39 +530,11 @@ public class ResourceClass
}
}
- public int calculateCap(int order, int basis)
+ // This is used for the counting code apportion_qshares in NodepoolScheduler. Returns qshares.
+ public int calculateCap()
{
- int perccap = Integer.MAX_VALUE; // the cap, calculated from percent
- int absolute = getAbsoluteCap();
- double percent = getPercentCap();
-
- if ( percent < 1.0 ) {
- double b = basis;
- b = b * percent;
- perccap = (int) Math.round(b);
- } else {
- perccap = basis;
- }
-
- int cap = Math.min(absolute, perccap) / order; // cap on total shares available
-
- //
- // If this RC is defined over a nodepool that isn't the global nodepool then its share
- // gets calculated multiple times. The first time when it is encountered during the
- // depth-first traversal, to work out the fair-share for all classes defined over the
- // nodepool. Subpool resources are also available to parent pools however, and must
- // be reevaluated to insure the resources are fairly allocated over the larger pool
- // of work.
- //
- // So at this point there might already be shares assigned. If so, we need to
- // recap on whatever is already given to avoid over-allocating "outside" of the
- // assigned shares.
- //
- if ( (given_by_order != null) && subpool_counted ) {
- cap = Math.min(cap, given_by_order[order]);
- } // else - never been counted or assigned at this order, no subpool cap
-
- return cap;
+ // significant rework, UIMA-4275
+ return getFairShareCap() / share_quantum; // cap on total shares available, converted to qshares
}
@@ -583,11 +543,12 @@ public class ResourceClass
return ( (given_by_order != null) && (given_by_order[0] > 0) );
}
- private int countActiveShares()
+ // number of quantum shares assigned
+ int countActiveShares()
{
int sum = 0;
for ( IRmJob j : allJobs.values() ) {
- sum += (j.countNShares() * j.getShareOrder()); // in quantum shares
+ sum += (j.countOccupancy() * j.getShareOrder()); // in quantum shares UIMA-4275
}
return sum;
}
@@ -626,15 +587,15 @@ public class ResourceClass
}
// note we assume Nodepool is the last token so we don't set a len for it!
- private static String formatString = "%12s %11s %4s %5s %6s %6s %7s %6s %6s %7s %5s %7s %s";
+ private static String formatString = "%12s %11s %4s %5s %6s %7s %6s %6s %7s %5s %7s %s";
public static String getDashes()
{
- return String.format(formatString, "------------", "-----------", "----", "-----", "------", "------", "-------", "------", "------", "-------", "-----", "-------", "--------");
+ return String.format(formatString, "------------", "-----------", "----", "-----", "------", "-------", "------", "------", "-------", "-----", "-------", "--------");
}
public static String getHeader()
{
- return String.format(formatString, "Class Name", "Policy", "Prio", "Wgt", "AbsCap", "PctCap", "InitCap", "Dbling", "Prdct", "PFudge", "Shr", "Enforce", "Nodepool");
+ return String.format(formatString, "Class Name", "Policy", "Prio", "Wgt", "AbsCap", "InitCap", "Dbling", "Prdct", "PFudge", "Shr", "Enforce", "Nodepool");
}
@Override
@@ -649,8 +610,7 @@ public class ResourceClass
policy.toString(),
priority,
share_weight,
- makeReadable(absolute_cap),
- (int) (percent_cap *100),
+ makeReadable(fair_share_cap),
initialization_cap,
expand_by_doubling,
use_prediction,
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/RmJob.java Mon Apr 27 20:40:28 2015
@@ -26,6 +26,7 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -330,7 +331,8 @@ public class RmJob
return wanted_by_order;
}
- public int calculateCap(int order, int total)
+ // UIMA-4275 simplifies this
+ public int calculateCap()
{
return Integer.MAX_VALUE; // no cap for jobs
}
@@ -339,13 +341,16 @@ public class RmJob
// UIMA-4275
public int countOccupancy()
{
- if ( (given_by_order == null) || (given_by_order[share_order] == 0) ) {
- // must use current allocation because we haven't been counted yet
- return countNShares() * share_order;
- } else {
- // new allocation may not match current, so we use that one
- return given_by_order[share_order] * share_order;
- }
+
+ return assignedShares.size() + pendingShares.size();
+
+ // if ( (given_by_order == null) || (given_by_order[share_order] == 0) ) {
+ // // must use current allocation because we haven't been counted yet
+ // return countNShares() * share_order;
+ // } else {
+ // // new allocation may not match current, so we use that one
+ // return given_by_order[share_order] * share_order;
+ //}
}
public int countNSharesGiven()
@@ -383,6 +388,9 @@ public class RmJob
public boolean canUseBonus(int order) // UIMA-4065
{
if ( order != share_order) return false;
+
+ if ( getGivenByOrder()[0] >= getResourceClass().calculateCap() ) return false; // don't exceed cap UIMA-4275
+
return (getWantedByOrder()[order] > 0); // yep, still want
}
@@ -721,7 +729,6 @@ public class RmJob
return 0;
}
-
if ( logger.isTrace() ) {
logger.trace(methodName, getId(), "Shares Before Sort - id, isInitialized, investment:");
for ( Share s : sharesSorted ) {
@@ -816,21 +823,33 @@ public class RmJob
}
/**
- * I'm shrinking by this many. Put into pendingRemoves but don't take them off the assigned list
- * until we know job manager has been told.
+ * This is an investment shrink. We don't have to walk nodepools because we use nodepool depth of each share
+ * to select deeper pools first.
*
- * TODO: this will likely change when the Scheduler starts deciding - either we'll put decision logic
- * into Job proper, or defer it to the IScheduler implementation (probably the latter).
+ * The selection strategy is this:
+ * 1. Lowest investment always wins.
+ * 2. Deepest nodepool is secondary sort.
+ * 3. Largest machine is tertiary.
*
- * @Deprecated. Keeping until I can nuke or update the simple fair share code.
+ * The assumption is that the job must give up the indicated shares unconditionally. We let the
+ * defragmentation routine to any additional cleanup if this isn't sufficient to satisfy pending expansions.
+ * UIMA-4275
*/
- //public void shrinkTo(int k)
- //{
-// int count = assignedShares.size() - k;
-// for ( int i = 0; i < count; i++ ) {
-// pendingRemoves.add(assignedShares.get(i));
-// }
- //}
+ public void shrinkBy(int howmany)
+ {
+ String methodName = "shrinkBy";
+ List<Share> sharesSorted = new ArrayList<Share>(assignedShares.values());
+ Collections.sort(sharesSorted, new ShareByInvestmentSorter());
+
+ int tolose = Math.min(howmany, sharesSorted.size());
+
+ for ( int i = 0; i < tolose; i++ ) {
+ Share ss = sharesSorted.get(i);
+ logger.info(methodName, getId(), "Removing share", ss.toString());
+ pendingRemoves.put(ss, ss);
+ ss.evict();
+ }
+ }
/**
* Waiting for somebody to deal with my shrinkage?
@@ -1184,6 +1203,12 @@ public class RmJob
return resource_class;
}
+ // UIMA-4275
+ public boolean exceedsFairShareCap()
+ {
+ return getResourceClass().fairShareCapExceeded(this);
+ }
+
// @deprecated
//public int countInstances() {
// return n_machines;
@@ -1368,6 +1393,53 @@ public class RmJob
}
}
+ // pull this out of the ShareByInvest sorter so other sorters can use it - see the
+ // ShareByWealth sorter in NodePoolScheduler
+ // UIMA-4275
+ public static int compareInvestment(Share s1, Share s2)
+ {
+ if ( s1.equals(s2) ) return 0;
+
+ int mask = 0;
+ if ( s1.isInitialized() ) mask |= 0x02;
+ if ( s2.isInitialized() ) mask |= 0x01;
+
+ switch ( mask ) {
+ case 0: // neither is initialized
+ return ( (int) (s1.getInitializationTime() - s2.getInitializationTime()) );
+ case 1: // s1 not initialized, s2 is
+ return -1;
+ case 2: // s2 initialized, s1 not
+ return 1;
+ default: // both initialized, compare investments
+ }
+
+ long i1 = s1.getInvestment();
+ long i2 = s2.getInvestment();
+ if (i1 == i2 ) {
+ // same invesstment, go to depper nodepool first
+ int d1 = s1.getNodepoolDepth();
+ int d2 = s1.getNodepoolDepth();
+
+ if ( d1 == d2 ) {
+ // same nodepool depth, go for largest machine
+ long m1 = s1.getHostMemory();
+ long m2 = s2.getHostMemory();
+
+ if ( m1 == m2 ) {
+ // last resort, sort on youngest first
+ return (int) (s2.getId().getFriendly() - s1.getId().getFriendly());
+ }
+ return (int) (m2 - m1); // largest machine
+ }
+
+ return (d2 - d1); // greatest depth
+
+ } else {
+ return (int) (i1 - i2); // least investment
+ }
+ }
+
//
// Order shares by INCREASING investment
//
@@ -1376,33 +1448,8 @@ public class RmJob
{
public int compare(Share s1, Share s2)
{
- if ( s1.equals(s2) ) return 0;
-
- int mask = 0;
- if ( s1.isInitialized() ) mask |= 0x02;
- if ( s2.isInitialized() ) mask |= 0x01;
-
- switch ( mask ) {
- case 0: // neither is initialized
- return ( (int) (s1.getInitializationTime() - s2.getInitializationTime()) );
- case 1: // s1 not initialized, s2 is
- return -1;
- case 2: // s2 initialized, s1 not
- return 1;
- default: // both initialized, compare investments
- }
-
- long i1 = s1.getInvestment();
- long i2 = s2.getInvestment();
- if (i1 == i2 ) {
- // Maybe splitting hairs. Neither share has investment but both are initialized. Apparently
- // we're waiting for the data to arrive. Choose the most recently assigned as it probably has the least
- // actual progress.
- return (int) (s2.getId().getFriendly() - s1.getId().getFriendly());
- } else {
- return (int) (i1 - i2);
- }
- }
+ return compareInvestment(s1, s2); // UIMA-4275
+ }
}
@Override
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java Mon Apr 27 20:40:28 2015
@@ -115,9 +115,11 @@ public class Scheduler
String schedImplName;
IScheduler[] schedulers;
- long share_quantum = 15; // 15 GB in KB - smallest share size
- long share_free_dram = 0; // 0 GB in KB - minim memory after shares are allocated
- long dramOverride = 0; // if > 0, use this instead of amount reported by agents (modeling and testing)
+ long share_quantum = 15; // 15 GB - smallest share size - converted to KB during init because
+ // hosts report in KB
+
+ long share_free_dram = 0; // 0 GB in KB - minim memory after shares are allocated
+ long dramOverride = 0; // if > 0, use this instead of amount reported by agents (modeling and testing)
int pending_evictions = 0; // for queries
int pending_expansions = 0; // for queries
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java Mon Apr 27 20:40:28 2015
@@ -168,6 +168,12 @@ public class Share
return machine;
}
+ long getHostMemory()
+ {
+ if ( machine != null ) return machine.getMemory();
+ else return 0L; // no clue what it is; don't npe
+ }
+
boolean isPending()
{
return job.isPendingShare(this);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java?rev=1676375&r1=1676374&r2=1676375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/User.java Mon Apr 27 20:40:28 2015
@@ -220,8 +220,9 @@ public class User
share_wealth -= w;
}
- public int calculateCap(int order, int total)
+ public int calculateCap()
{
+ // UIMA-4275, simplify
return Integer.MAX_VALUE; // no cap for users
}