You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/09/11 15:04:10 UTC
svn commit: r1702451 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/
uima-ducc-orchestrator/src/main/java/org/ap...
Author: degenaro
Date: Fri Sep 11 13:04:08 2015
New Revision: 1702451
URL: http://svn.apache.org/r1702451
Log:
UIMA-4569 JD share reservation code needs to use size given by RM, not size requested
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdReservation.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccReservation.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkService.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccReservation.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/JdReservationBean.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/reservations.jsp
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/ReservationFactory.java Fri Sep 11 13:04:08 2015
@@ -66,13 +66,13 @@ public class ReservationFactory {
schedulingInfo.setSchedulingClass(reservationRequestProperties.getProperty(ReservationSpecificationProperties.key_scheduling_class));
String memorySize = reservationRequestProperties.getProperty(ReservationSpecificationProperties.key_memory_size);
MemorySpecification memorySpecification = new MemorySpecification(memorySize);
- schedulingInfo.setMemorySize(memorySpecification.getSize());
+ schedulingInfo.setMemorySizeRequested(memorySpecification.getSize());
schedulingInfo.setMemoryUnits(memorySpecification.getMemoryUnits());
logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("user")+standardInfo.getUser());
logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("description")+standardInfo.getDescription());
logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("class")+schedulingInfo.getSchedulingClass());
logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("priority")+schedulingInfo.getSchedulingPriority());
- logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("memory")+schedulingInfo.getMemorySize()+schedulingInfo.getMemoryUnits());
+ logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("memory")+schedulingInfo.getMemorySizeRequested()+schedulingInfo.getMemoryUnits());
logger.info(methodName, duccWorkReservation.getDuccId(), messages.fetchLabel("instances")+schedulingInfo.getInstancesCount());
logger.trace(methodName, null, messages.fetch("exit"));
return duccWorkReservation;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java Fri Sep 11 13:04:08 2015
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.container.FlagsHelper;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.jd.files.workitem.IRemoteLocation;
@@ -1141,14 +1142,11 @@ public class StateManager {
return changes;
}
- private int KB = 1000;
- private int MB = 1000*KB;
- private int GB = 1000*MB;
-
private int processMapResourcesAdd(DuccWorkJob duccWorkJob, int memoryGbPerProcess, Map<DuccId,IResource> resourceMap) {
String methodName = "processMapResourcesAdd";
logger.trace(methodName, null, messages.fetch("enter"));
int changes = 0;
+ duccWorkJob.getSchedulingInfo().setMemorySizeAllocatedInBytes(memoryGbPerProcess*SizeBytes.GB);
if(resourceMap == null) {
logger.info(methodName, duccWorkJob.getDuccId(), messages.fetch("no map found"));
}
@@ -1172,7 +1170,7 @@ public class StateManager {
break;
}
DuccProcess process = new DuccProcess(duccId, node, processType);
- long process_max_size_in_bytes = memoryGbPerProcess * GB;
+ long process_max_size_in_bytes = memoryGbPerProcess * SizeBytes.GB;
CGroupManager.assign(duccWorkJob.getDuccId(), process, process_max_size_in_bytes);
orchestratorCommonArea.getProcessAccounting().addProcess(duccId, duccWorkJob.getDuccId());
processMap.addProcess(process);
@@ -1286,6 +1284,7 @@ public class StateManager {
String methodName = "reservationMapResourcesAdd";
logger.trace(methodName, null, messages.fetch("enter"));
int changes = 0;
+ duccWorkReservation.getSchedulingInfo().setMemorySizeAllocatedInBytes(memoryGbPerProcess*SizeBytes.GB);
IDuccReservationMap reservationMap = duccWorkReservation.getReservationMap();
if(resourceMap != null) {
Iterator<DuccId> resourceMapIterator = resourceMap.keySet().iterator();
@@ -1295,7 +1294,7 @@ public class StateManager {
Node node = resource.getNode();
NodeIdentity nodeId = node.getNodeIdentity();
if(!reservationMap.containsKey(duccId)) {
- int bytes = memoryGbPerProcess * GB;
+ long bytes = memoryGbPerProcess * SizeBytes.GB;
DuccReservation reservation = new DuccReservation(duccId, node, bytes);
reservationMap.addReservation(reservation);
logger.info(methodName, duccId, messages.fetch("add resource")+" "+messages.fetchLabel("name")+nodeId.getName()+" "+messages.fetchLabel("ip")+nodeId.getIp());
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Fri Sep 11 13:04:08 2015
@@ -519,14 +519,14 @@ public class JobFactory implements IJobF
job.setSchedulingInfo(schedulingInfo);
long jpGB = JobFactoryHelper.getByteSizeJobProcess() / JobFactoryHelper.GB;
if(jpGB > 0) {
- schedulingInfo.setMemorySize(""+jpGB);
+ schedulingInfo.setMemorySizeRequested(""+jpGB);
}
schedulingInfo.setSchedulingClass(jobRequestProperties.getProperty(JobSpecificationProperties.key_scheduling_class));
schedulingInfo.setSchedulingPriority(jobRequestProperties.getProperty(JobSpecificationProperties.key_scheduling_priority));
schedulingInfo.setProcessesMax(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_deployments_max));
schedulingInfo.setProcessesMin(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_deployments_min));
schedulingInfo.setThreadsPerProcess(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_thread_count));
- schedulingInfo.setMemorySize(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_memory_size));
+ schedulingInfo.setMemorySizeRequested(jobRequestProperties.getProperty(JobSpecificationProperties.key_process_memory_size));
schedulingInfo.setMemoryUnits(MemoryUnits.GB);
if (job.getDuccType() == DuccType.Job){
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdHelper.java Fri Sep 11 13:04:08 2015
@@ -21,13 +21,18 @@ package org.apache.uima.ducc.orchestrato
import java.util.Map.Entry;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
+import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccReservation;
import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
public class JdHelper {
-
+
+ private static DuccLogger logger = new DuccLogger(JdHelper.class);
+ private static DuccId jobid = null;
+
public static NodeIdentity getNodeIdentity(IDuccWorkReservation dwr) {
NodeIdentity retVal = null;
if(dwr != null) {
@@ -42,41 +47,7 @@ public class JdHelper {
return retVal;
}
- public static long KB = 1024;
- public static long MB = 1024*KB;
- public static long GB = 1024*MB;
- public static long TB = 1024*GB;
-
- public static long parseSize(String value) {
- long size = 0;
- try {
- String tValue = value.trim();
- if(tValue.endsWith("KB")) {
- tValue = tValue.substring(0, tValue.length() - 2);
- size = new Long(tValue) * KB;
- }
- else if(tValue.endsWith("MB")) {
- tValue = tValue.substring(0, tValue.length() - 2);
- size = new Long(tValue) * MB;
- }
- else if(tValue.endsWith("GB")) {
- tValue = tValue.substring(0, tValue.length() - 2);
- size = new Long(tValue) * GB;
- }
- else if(tValue.endsWith("TB")) {
- tValue = tValue.substring(0, tValue.length() - 2);
- size = new Long(tValue) * TB;
- }
- else {
- size = new Long(tValue);
- }
- }
- catch(Exception e) {
- }
- return size;
- }
-
- public static long parseCount(String value) {
+ private static long parseCount(String value) {
long count = 0;
try {
String tValue = value.trim();
@@ -87,24 +58,26 @@ public class JdHelper {
return count;
}
- public static long getReservationSize(JdHostProperties jdHostProperties) {
- long retVal = 0;
+ public static SizeBytes getReservationSize(IDuccWorkReservation dwr) {
+ String location = "getReservationSize";
+ SizeBytes retVal = null;
try {
- if(jdHostProperties != null) {
- String value = jdHostProperties.getHostMemorySize();
- retVal = parseSize(value);
- }
+ long sizeBytes = dwr.getSchedulingInfo().getMemorySizeAllocatedInBytes();
+ retVal = new SizeBytes(SizeBytes.Type.Bytes, sizeBytes);
+ logger.trace(location, dwr.getDuccId(), retVal.getGBytes()+" GB");
}
catch(Exception e) {
}
return retVal;
}
- public static long getSliceSize(JdHostProperties jdHostProperties) {
- long retVal = 0;
+ public static SizeBytes getSliceSize(JdHostProperties jdHostProperties) {
+ String location = "getSliceSize";
+ SizeBytes retVal = null;
if(jdHostProperties != null) {
String value = jdHostProperties.getJdShareQuantum();
- retVal = parseSize(value+"MB");
+ retVal = new SizeBytes(SizeBytes.Type.MBytes, Long.parseLong(value));
+ logger.trace(location, jobid, retVal.getMBytes()+" MB");
}
return retVal;
}
@@ -112,13 +85,16 @@ public class JdHelper {
public static long SlicesReserveDefault = 2;
public static long getSlicesReserve(JdHostProperties jdHostProperties) {
+ String location = "getSlicesReserve";
long retVal = 0;
if(jdHostProperties != null) {
String value = jdHostProperties.getSlicesReserve();
retVal = parseCount(value);
+ logger.trace(location, jobid, retVal+" "+"specified");
}
if(retVal < 1) {
retVal = SlicesReserveDefault;
+ logger.trace(location, jobid, retVal+" "+"default");
}
return retVal;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdReservation.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdReservation.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdReservation.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdReservation.java Fri Sep 11 13:04:08 2015
@@ -21,6 +21,7 @@ package org.apache.uima.ducc.orchestrato
import java.util.concurrent.ConcurrentHashMap;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
@@ -32,12 +33,13 @@ public class JdReservation extends JdRes
private static final long serialVersionUID = 1L;
private static DuccLogger logger = new DuccLogger(JdReservation.class);
+ private static DuccId jobid = null;
- public JdReservation(IDuccWorkReservation dwr, Long reservationSizeMB, Long sliceSizeMB) {
- initialize(dwr, reservationSizeMB, sliceSizeMB);
+ public JdReservation(IDuccWorkReservation dwr, SizeBytes reservationSize, SizeBytes sliceSize) {
+ initialize(dwr, reservationSize, sliceSize);
}
- private void initialize(IDuccWorkReservation dwr, Long reservationSize, Long sliceSize) {
+ private void initialize(IDuccWorkReservation dwr, SizeBytes reservationSize, SizeBytes sliceSize) {
if(dwr != null) {
DuccId jdReservationId = (DuccId) dwr.getDuccId();
setJdReservationId(jdReservationId);
@@ -77,20 +79,26 @@ public class JdReservation extends JdRes
}
public Long getSlicesTotal() {
- Long reservationSize = getReservationSize();
- Long sliceSize = getSliceSize();
- Long retVal = reservationSize / sliceSize;
+ String location = "getSlicesTotal";
+ SizeBytes reservationSize = getReservationSize();
+ SizeBytes sliceSize = getSliceSize();
+ Long retVal = (long) (reservationSize.getBytes() / (1.0 * sliceSize.getBytes()));
+ logger.trace(location, jobid, retVal);
return retVal;
}
public Long getSlicesInuse() {
- ConcurrentHashMap<DuccId, Long> map = getMap();
+ String location = "getSlicesInuse";
+ ConcurrentHashMap<DuccId, SizeBytes> map = getMap();
long retVal = new Long(map.size());
+ logger.trace(location, jobid, retVal);
return retVal;
}
public Long getSlicesAvailable() {
+ String location = "getSlicesAvailable";
Long retVal = getSlicesTotal() - getSlicesInuse();
+ logger.trace(location, jobid, retVal);
return retVal;
}
@@ -109,10 +117,10 @@ public class JdReservation extends JdRes
return retVal;
}
- protected NodeIdentity allocate(DuccId jdId, DuccId jobId, Long size) {
+ protected NodeIdentity allocate(DuccId jdId, DuccId jobId, SizeBytes size) {
String location = "allocate";
NodeIdentity retVal = null;
- ConcurrentHashMap<DuccId, Long> map = getMap();
+ ConcurrentHashMap<DuccId, SizeBytes> map = getMap();
if(jdId != null) {
synchronized(map) {
if(!map.containsKey(jdId)) {
@@ -132,7 +140,7 @@ public class JdReservation extends JdRes
protected NodeIdentity deallocate(DuccId jdId, DuccId jobId) {
String location = "deallocate";
NodeIdentity retVal = null;
- ConcurrentHashMap<DuccId, Long> map = getMap();
+ ConcurrentHashMap<DuccId, SizeBytes> map = getMap();
if(jdId != null) {
synchronized(map) {
if(map.containsKey(jdId)) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/jd/scheduler/JdScheduler.java Fri Sep 11 13:04:08 2015
@@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.config.CommonConfiguration;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -220,7 +221,7 @@ public class JdScheduler {
return slicesReserveDesired;
}
- private long getSlicesReserveActual(JdHostProperties jdHostProperties) {
+ private long getSlicesReserveActual() {
long slicesReserveActual = countSlicesAvailable();
return slicesReserveActual;
}
@@ -228,7 +229,7 @@ public class JdScheduler {
private boolean isReservationDivestable(IDuccWorkMap dwm, JdHostProperties jdHostProperties) {
boolean retVal = false;
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
+ long slicesReserveActual = getSlicesReserveActual();
if(map.size() > 1) {
if(slicesReserveActual > (2 * slicesReserveDesired)) {
synchronized(map) {
@@ -249,8 +250,8 @@ public class JdScheduler {
String location = "resourceAdjustment";
if(autoManage.get()) {
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
- logger.debug(location, jobid, "desired: "+slicesReserveDesired+" "+"actual: "+slicesReserveActual);
+ long slicesReserveActual = getSlicesReserveActual();
+ logger.debug(location, jobid, "actual: "+slicesReserveActual+" "+"desired: "+slicesReserveDesired);
if(slicesReserveActual < slicesReserveDesired) {
if(requestPending.get()) {
reservationPending(dwm, jdHostProperties);
@@ -276,7 +277,7 @@ public class JdScheduler {
private void reservationPending(IDuccWorkMap dwm, JdHostProperties jdHostProperties) {
String location = "reservationPending";
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
+ long slicesReserveActual = getSlicesReserveActual();
logger.debug(location, jobid, "actual: "+slicesReserveActual+" "+"desired: "+slicesReserveDesired);
}
@@ -316,7 +317,7 @@ public class JdScheduler {
OrchestratorCheckpoint.getInstance().saveState();
//
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
+ long slicesReserveActual = getSlicesReserveActual();
DuccId duccId = dwr.getDuccId();
logger.debug(location, duccId, "actual: "+slicesReserveActual+" "+"desired: "+slicesReserveDesired);
}
@@ -348,14 +349,14 @@ public class JdScheduler {
}
}
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
+ long slicesReserveActual = getSlicesReserveActual();
logger.debug(location, jobid, "actual: "+slicesReserveActual+" "+"desired: "+slicesReserveDesired);
}
private void reservationNoChange(IDuccWorkMap dwm, JdHostProperties jdHostProperties) {
String location = "reservationNoChange";
long slicesReserveDesired = getSlicesReserveDesired(jdHostProperties);
- long slicesReserveActual = getSlicesReserveActual(jdHostProperties);
+ long slicesReserveActual = getSlicesReserveActual();
logger.trace(location, jobid, "actual: "+slicesReserveActual+" "+"desired: "+slicesReserveDesired);
}
@@ -404,16 +405,16 @@ public class JdScheduler {
DuccId duccId = dwr.getDuccId();
DuccId jdReservationDuccId = (DuccId) duccId;
JdReservation jdReservation = null;
- long reservationSizeMB = JdHelper.getReservationSize(jdHostProperties);
- long sliceSizeMB = JdHelper.getSliceSize(jdHostProperties);
+ SizeBytes reservationSize = JdHelper.getReservationSize(dwr);
+ SizeBytes sliceSize = JdHelper.getSliceSize(jdHostProperties);
synchronized(map) {
jdReservation = map.get(jdReservationDuccId);
if(jdReservation == null) {
- jdReservation = new JdReservation(dwr, reservationSizeMB, sliceSizeMB);
+ jdReservation = new JdReservation(dwr, reservationSize, sliceSize);
map.put(jdReservationDuccId, jdReservation);
}
else if(!jdReservation.isUp()) {
- jdReservation = new JdReservation(dwr, reservationSizeMB, sliceSizeMB);
+ jdReservation = new JdReservation(dwr, reservationSize, sliceSize);
map.put(jdReservationDuccId, jdReservation);
}
else {
@@ -449,12 +450,12 @@ public class JdScheduler {
DuccId duccId = dwr.getDuccId();
DuccId jdReservationDuccId = (DuccId) duccId;
JdReservation jdReservation = null;
- long reservationSizeMB = JdHelper.getReservationSize(jdHostProperties);
- long sliceSizeMB = JdHelper.getSliceSize(jdHostProperties);
+ SizeBytes reservationSize = JdHelper.getReservationSize(dwr);
+ SizeBytes sliceSize = JdHelper.getSliceSize(jdHostProperties);
synchronized(map) {
jdReservation = map.get(jdReservationDuccId);
if(jdReservation == null) {
- jdReservation = new JdReservation(dwr, reservationSizeMB, sliceSizeMB);
+ jdReservation = new JdReservation(dwr, reservationSize, sliceSize);
map.put(jdReservationDuccId, jdReservation);
}
}
@@ -547,56 +548,68 @@ public class JdScheduler {
}
public int countSlicesTotal(DuccId duccId) {
+ String location = "countSlicesTotal";
int count = 0;
JdReservation jdReservation = map.get(duccId);
if(jdReservation != null) {
count += jdReservation.getSlicesTotal();
}
+ logger.trace(location, duccId, count);
return count;
}
public int countSlicesInuse(DuccId duccId) {
+ String location = "countSlicesInuse";
int count = 0;
JdReservation jdReservation = map.get(duccId);
if(jdReservation != null) {
count += jdReservation.getSlicesInuse();
}
+ logger.trace(location, duccId, count);
return count;
}
public int countSlicesAvailable(DuccId duccId) {
+ String location = "countSlicesAvailable";
int count = 0;
JdReservation jdReservation = map.get(duccId);
if(jdReservation != null) {
count += jdReservation.getSlicesAvailable();
}
+ logger.trace(location, duccId, count);
return count;
}
public int countSlicesTotal() {
+ String location = "countSlicesTotal";
int count = 0;
for(Entry<DuccId, JdReservation> entry : map.entrySet()) {
JdReservation jdReservation = entry.getValue();
count += jdReservation.getSlicesTotal();
}
+ logger.trace(location, jobid, count);
return count;
}
public int countSlicesInuse() {
+ String location = "countSlicesInuse";
int count = 0;
for(Entry<DuccId, JdReservation> entry : map.entrySet()) {
JdReservation jdReservation = entry.getValue();
count += jdReservation.getSlicesInuse();
}
+ logger.trace(location, jobid, count);
return count;
}
public int countSlicesAvailable() {
+ String location = "countSlicesAvailable";
int count = 0;
for(Entry<DuccId, JdReservation> entry : map.entrySet()) {
JdReservation jdReservation = entry.getValue();
count += jdReservation.getSlicesAvailable();
}
+ logger.trace(location, jobid, count);
return count;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/test/java/org/apache/uima/ducc/orchestrator/jd/scheduler/test/TestSuite.java Fri Sep 11 13:04:08 2015
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
import java.util.Random;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.config.CommonConfiguration;
import org.apache.uima.ducc.common.main.DuccService;
import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -35,8 +36,12 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.orchestrator.OrchestratorCheckpoint;
import org.apache.uima.ducc.orchestrator.OrchestratorCommonArea;
import org.apache.uima.ducc.orchestrator.jd.scheduler.JdHostProperties;
+import org.apache.uima.ducc.orchestrator.jd.scheduler.JdReservation;
import org.apache.uima.ducc.orchestrator.jd.scheduler.JdScheduler;
+import org.apache.uima.ducc.transport.event.common.DuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.DuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkMap;
@@ -106,6 +111,16 @@ public class TestSuite {
private int seed = 0;
private Random random = new Random(seed);
+ private long KB = 1024;
+ private long MB = 1024*KB;
+ private long GB = 1024*MB;
+
+ private void rmAssign(IDuccWorkReservation dwr) {
+ IDuccSchedulingInfo si = dwr.getSchedulingInfo();
+ long size = 15 *GB;
+ si.setMemorySizeAllocatedInBytes(size);
+ }
+
private void assign(IDuccWorkMap dwm) {
for(DuccId reservationId : dwm.getReservationKeySet()) {
IDuccWork dw = dwm.findDuccWork(reservationId);
@@ -114,6 +129,7 @@ public class TestSuite {
case WaitingForResources:
TestHelper.assign(dwr);
// state: WaitingForResources
+ rmAssign(dwr);
dwr.stateChange(ReservationState.Assigned);
OrchestratorCheckpoint.getInstance().saveState();
break;
@@ -142,6 +158,28 @@ public class TestSuite {
}
@Test
+ public void testJdReservation() {
+ try {
+ IDuccWorkReservation dwr = new DuccWorkReservation();
+ DuccId duccId = new DuccId(1);
+ dwr.setDuccId(duccId);
+ dwr.setReservationState(ReservationState.Assigned);
+ IDuccSchedulingInfo si = new DuccSchedulingInfo();
+ dwr.setSchedulingInfo(si);
+ si.setMemorySizeAllocatedInBytes(15*SizeBytes.GB);
+ //
+ SizeBytes reservationSize = new SizeBytes(SizeBytes.Type.GBytes,15);
+ SizeBytes sliceSize = new SizeBytes(SizeBytes.Type.MBytes,300);
+ JdReservation jdReservation = new JdReservation(dwr,reservationSize,sliceSize);
+ assertTrue(jdReservation.getSlicesAvailable() > 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
public void testJdSchedulerStatic() {
try {
JdHostProperties jdHostProperties = new JdHostProperties();
@@ -154,6 +192,7 @@ public class TestSuite {
jdScheduler.handle(dwm);
assertTrue(jdScheduler.countReservationsTotal() == 0);
assertTrue(jdScheduler.countReservationsUp() == 0);
+ assertTrue(jdScheduler.countSlicesTotal() == 0);
//
IDuccWorkReservation dwr = null;
//
@@ -162,15 +201,19 @@ public class TestSuite {
jdScheduler.handle(dwm);
assertTrue(jdScheduler.countReservationsTotal() == 1);
assertTrue(jdScheduler.countReservationsUp() == 0);
+ assertTrue(jdScheduler.countSlicesTotal() == 0);
//
dwr = TestHelper.getDWR(jdHostProperties);
+ rmAssign(dwr);
dwr.setReservationState(ReservationState.Assigned);
dwm.addDuccWork(dwr);
jdScheduler.handle(dwm);
assertTrue(jdScheduler.countReservationsTotal() == 2);
assertTrue(jdScheduler.countReservationsUp() == 1);
+ assertTrue(jdScheduler.countSlicesTotal() > 0);
//
dwr = TestHelper.getDWR(jdHostProperties);
+ rmAssign(dwr);
dwr.setReservationState(ReservationState.Assigned);
dwm.addDuccWork(dwr);
jdScheduler.handle(dwm);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-pm/src/main/java/org/apache/uima/ducc/pm/ProcessManagerComponent.java Fri Sep 11 13:04:08 2015
@@ -155,7 +155,7 @@ implements ProcessManager {
// Create process list for each job
List<IDuccProcess> jobProcessList = new ArrayList<IDuccProcess>();
- long normalizedProcessMemoryRequirements = normalizeMemory(dcj.getSchedulingInfo().getMemorySize(),dcj.getSchedulingInfo().getMemoryUnits());
+ long normalizedProcessMemoryRequirements = normalizeMemory(dcj.getSchedulingInfo().getMemorySizeRequested(),dcj.getSchedulingInfo().getMemoryUnits());
int shares = getShares(normalizedProcessMemoryRequirements);
long processAdjustedMemorySize = shares * shareQuantum * 1024;
ProcessMemoryAssignment pma = new ProcessMemoryAssignment();
@@ -180,7 +180,7 @@ implements ProcessManager {
processAdjustedMemorySize += (processAdjustedMemorySize * ((double)fudgeFactor/100));
pma.setMaxMemoryWithFudge(processAdjustedMemorySize);
- logger.debug(methodName,dcj.getDuccId(),"--------------- User Requested Memory For Process:"+dcj.getSchedulingInfo().getMemorySize()+dcj.getSchedulingInfo().getMemoryUnits()+" PM Calculated Memory Assignment of:"+processAdjustedMemorySize);
+ logger.debug(methodName,dcj.getDuccId(),"--------------- User Requested Memory For Process:"+dcj.getSchedulingInfo().getMemorySizeRequested()+dcj.getSchedulingInfo().getMemoryUnits()+" PM Calculated Memory Assignment of:"+processAdjustedMemorySize);
ICommandLine driverCmdLine = null;
ICommandLine processCmdLine = null;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Fri Sep 11 13:04:08 2015
@@ -525,7 +525,7 @@ public class JobManagerConverter
logger.info(methodName, job.getDuccId(), "total_work", total_work, "completed_work", completed_work,"remaining_work", remaining_work);
- int memory = toInt(si.getMemorySize(), scheduler.getDefaultMemory());
+ int memory = toInt(si.getMemorySizeRequested(), scheduler.getDefaultMemory());
String className = si.getSchedulingClass();
if ( className == null ) {
switch ( job.getDuccType() ) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccReservation.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccReservation.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccReservation.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccReservation.java Fri Sep 11 13:04:08 2015
@@ -31,10 +31,10 @@ public class DuccReservation implements
private DuccId duccId = null;
private Node node = null;
private NodeIdentity nodeIdentity = null;
- private int bytes = 0;
+ private long bytes = 0;
private ITimeWindow timeWindow = null;
- public DuccReservation(DuccId duccId, Node node, int bytes) {
+ public DuccReservation(DuccId duccId, Node node, long bytes) {
setDuccId(duccId);
setNode(node);
NodeIdentity nodeIdentity = node.getNodeIdentity();
@@ -42,7 +42,7 @@ public class DuccReservation implements
setBytes(bytes);
}
- public DuccReservation(DuccId duccId, NodeIdentity nodeIdentity, int bytes) {
+ public DuccReservation(DuccId duccId, NodeIdentity nodeIdentity, long bytes) {
setDuccId(duccId);
setNodeIdentity(nodeIdentity);
setBytes(bytes);
@@ -64,8 +64,8 @@ public class DuccReservation implements
}
- public int getBytes() {
- int retVal = 0;
+ public long getBytes() {
+ long retVal = 0;
try {
retVal = bytes;
}
@@ -75,7 +75,7 @@ public class DuccReservation implements
}
- public void setBytes(int bytes) {
+ public void setBytes(long bytes) {
this.bytes = bytes;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java Fri Sep 11 13:04:08 2015
@@ -35,8 +35,11 @@ public class DuccSchedulingInfo implemen
private static final long serialVersionUID = 1L;
private String schedulingClass = defaultSchedulingClass;
private String schedulingPriority = defaultSchedulingPriority;
+ @Deprecated
private String memorySize = defaultMemorySize;
+ private String memorySizeRequested = defaultMemorySize;
private MemoryUnits memoryUnits = defaultMemoryUnits;
+ private long memorySizeAllocatedInBytes = 0;
private String instancesCount = defaultInstancesCount;
@Deprecated
@@ -102,29 +105,22 @@ public class DuccSchedulingInfo implemen
}
}
- @Deprecated
+ @Deprecated
public String getMachinesCount() {
return machinesCount;
}
- @Deprecated
+ @Deprecated
public void setMachinesCount(String machinesCount) {
if(machinesCount != null) {
this.machinesCount = machinesCount;
}
}
-
-
- public String getMemorySize() {
- return memorySize;
- }
-
- private static long KB = 1000;
- private static long MB = 1000*KB;
- private static long GB = 1000*MB;
+/*
+ @Deprecated
public long getMemorySizeInBytes() {
long retVal = 0;
try {
@@ -147,13 +143,43 @@ public class DuccSchedulingInfo implemen
}
+ @Deprecated
+ public String getMemorySize() {
+ return memorySize;
+ }
+
+
+ @Deprecated
public void setMemorySize(String size) {
if(size != null) {
this.memorySize = size;
}
}
+*/
+
+
+ public String getMemorySizeRequested() {
+ return memorySizeRequested;
+ }
+
+
+ public void setMemorySizeRequested(String size) {
+ if(size != null) {
+ this.memorySizeRequested = size;
+ }
+ }
+
+
+ public long getMemorySizeAllocatedInBytes() {
+ return memorySizeAllocatedInBytes;
+ }
+ public void setMemorySizeAllocatedInBytes(long value) {
+ this.memorySizeAllocatedInBytes = value;
+ }
+
+
public MemoryUnits getMemoryUnits() {
return memoryUnits;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkMap.java Fri Sep 11 13:04:08 2015
@@ -194,7 +194,7 @@ public class DuccWorkMap implements IDuc
Set<DuccId> keys = getJobKeySet();
for(DuccId key : keys) {
IDuccWorkJob job = (IDuccWorkJob)concurrentWorkMap.get(key);
- long bytesPerProcess = job.getSchedulingInfo().getMemorySizeInBytes();
+ long bytesPerProcess = job.getSchedulingInfo().getMemorySizeAllocatedInBytes();
long numberOfProcesses = job.getAliveProcessCount();
size += bytesPerProcess * numberOfProcesses;
}
@@ -206,7 +206,7 @@ public class DuccWorkMap implements IDuc
Set<DuccId> keys = getServiceKeySet();
for(DuccId key : keys) {
IDuccWorkService service = (IDuccWorkService)concurrentWorkMap.get(key);
- long bytesPerProcess = service.getSchedulingInfo().getMemorySizeInBytes();
+ long bytesPerProcess = service.getSchedulingInfo().getMemorySizeAllocatedInBytes();
long numberOfProcesses = 1;
size += bytesPerProcess * numberOfProcesses;
}
@@ -218,7 +218,7 @@ public class DuccWorkMap implements IDuc
Set<DuccId> keys = getReservationKeySet();
for(DuccId key : keys) {
IDuccWorkReservation reservation = (IDuccWorkReservation)concurrentWorkMap.get(key);
- long bytesPerReservation = reservation.getSchedulingInfo().getMemorySizeInBytes();
+ long bytesPerReservation = reservation.getSchedulingInfo().getMemorySizeAllocatedInBytes();
long numberOfReservations = 1;
size += bytesPerReservation * numberOfReservations;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkService.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkService.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkService.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccWorkService.java Fri Sep 11 13:04:08 2015
@@ -20,7 +20,7 @@ package org.apache.uima.ducc.transport.e
import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
-public class DuccWorkService extends ADuccWorkExecutable {
+public class DuccWorkService extends ADuccWorkExecutable implements IDuccWorkService {
/**
* please increment this sUID when removing or modifying a field
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccReservation.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccReservation.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccReservation.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccReservation.java Fri Sep 11 13:04:08 2015
@@ -36,8 +36,8 @@ public interface IDuccReservation extend
public NodeIdentity getNodeIdentity();
public void setNodeIdentity(NodeIdentity nodeIdentity);
- public int getBytes();
- public void setBytes(int bytes);
+ public long getBytes();
+ public void setBytes(long bytes);
public ITimeWindow getTimeWindow();
public void setTimeWindow(ITimeWindow timeWindow);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java Fri Sep 11 13:04:08 2015
@@ -30,7 +30,7 @@ public interface IDuccSchedulingInfo ext
public static final String defaultSchedulingClass = "normal";
public static final String defaultSchedulingPriority = "0";
- public static final String defaultMemorySize = "13";
+ public static final String defaultMemorySize = "1";
public static final MemoryUnits defaultMemoryUnits = MemoryUnits.GB;
public static final String defaultInstancesCount = "1";
@@ -62,10 +62,11 @@ public interface IDuccSchedulingInfo ext
public String getSchedulingPriority();
public void setSchedulingPriority(String schedulingPriority);
- public String getMemorySize();
- public void setMemorySize(String size);
+ public String getMemorySizeRequested();
+ public void setMemorySizeRequested(String size);
- public long getMemorySizeInBytes();
+ public void setMemorySizeAllocatedInBytes(long value);
+ public long getMemorySizeAllocatedInBytes();
public MemoryUnits getMemoryUnits();
public void setMemoryUnits(MemoryUnits units);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/JdReservationBean.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/JdReservationBean.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/JdReservationBean.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/JdReservationBean.java Fri Sep 11 13:04:08 2015
@@ -22,30 +22,27 @@ import java.io.Serializable;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
public class JdReservationBean implements Serializable {
private static final long serialVersionUID = 1L;
- public static long KB = 1024;
- public static long MB = 1024*KB;
- public static long GB = 1024*MB;
- public static long TB = 1024*GB;
private DuccId jdReservationDuccId = null;
private NodeIdentity nodeIdentity;
private ReservationState reservationState = null;
- private Long reservationSize = new Long(30*GB);
- private Long sliceSize = new Long(300*MB);
+ private SizeBytes reservationSize = new SizeBytes(SizeBytes.Type.GBytes,30);
+ private SizeBytes sliceSize = new SizeBytes(SizeBytes.Type.MBytes,300);
- private ConcurrentHashMap<DuccId, Long> map = new ConcurrentHashMap<DuccId, Long>();
+ private ConcurrentHashMap<DuccId, SizeBytes> map = new ConcurrentHashMap<DuccId, SizeBytes>();
- public void setMap(ConcurrentHashMap<DuccId, Long> value) {
+ public void setMap(ConcurrentHashMap<DuccId, SizeBytes> value) {
map = value;
}
- public ConcurrentHashMap<DuccId, Long> getMap() {
+ public ConcurrentHashMap<DuccId, SizeBytes> getMap() {
return map;
}
@@ -73,19 +70,19 @@ public class JdReservationBean implement
return reservationState;
}
- public void setReservationSize(Long value) {
+ public void setReservationSize(SizeBytes value) {
reservationSize = value;
}
- public Long getReservationSize() {
+ public SizeBytes getReservationSize() {
return reservationSize;
}
- public void setSliceSize(Long value) {
+ public void setSliceSize(SizeBytes value) {
sliceSize = value;
}
- public Long getSliceSize() {
+ public SizeBytes getSliceSize() {
return sliceSize;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccAbstractHandler.java Fri Sep 11 13:04:08 2015
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties.DaemonName;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -42,7 +43,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
@@ -325,6 +325,21 @@ public abstract class DuccAbstractHandle
return getUsers(cookie);
}
+ public String getProcessMemorySize(DuccId id, SizeBytes size) {
+ String methodName = "getProcessMemorySize";
+ String retVal = "?";
+ try {
+ double dSize = size.getGBytes();
+ DecimalFormat formatter = new DecimalFormat("###0");
+ retVal = formatter.format(dSize);
+ }
+ catch(Exception e) {
+ duccLogger.trace(methodName, id, e);
+ }
+ return retVal;
+ }
+
+ /*
public String getProcessMemorySize(DuccId id, String type, String size, MemoryUnits units) {
String methodName = "getProcessMemorySize";
String retVal = "?";
@@ -353,6 +368,7 @@ public abstract class DuccAbstractHandle
}
return retVal;
}
+ */
public RequestStateType getStateTypeParameter(HttpServletRequest request) {
RequestStateType requestStateType = requestStateTypeDefault;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java Fri Sep 11 13:04:08 2015
@@ -45,6 +45,7 @@ import javax.servlet.http.HttpServletRes
import org.apache.uima.ducc.common.CancelReasons.CancelReason;
import org.apache.uima.ducc.common.NodeConfiguration;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties.DaemonName;
import org.apache.uima.ducc.common.internationalization.Messages;
@@ -3627,10 +3628,6 @@ public class DuccHandler extends DuccAbs
duccLogger.trace(methodName, null, messages.fetch("exit"));
}
- private long KB = 1000;
- private long MB = 1000*KB;
- private long GB = 1000*MB;
-
private void handleDuccServletClusterUtilization(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
@@ -3644,7 +3641,7 @@ public class DuccHandler extends DuccAbs
String utilization = "0%";
- long memFree = msi.memFree * GB;
+ long memFree = msi.memFree * SizeBytes.GB;
long memInuse = DuccData.getInstance().getLive().getMemoryInuse();
if(memFree > 0) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerClassic.java Fri Sep 11 13:04:08 2015
@@ -41,6 +41,7 @@ import org.apache.uima.ducc.cli.ws.json.
import org.apache.uima.ducc.cli.ws.json.MachineFactsList;
import org.apache.uima.ducc.common.IDuccEnv;
import org.apache.uima.ducc.common.NodeConfiguration;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties.DaemonName;
import org.apache.uima.ducc.common.internationalization.Messages;
@@ -61,7 +62,7 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccReservation;
import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
-import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
+import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IRationale;
@@ -110,7 +111,6 @@ public class DuccHandlerClassic extends
}
private void buildJobsListEntry(HttpServletRequest request, StringBuffer sb, DuccId duccId, IDuccWorkJob job, DuccData duccData, long now, ServicesRegistry servicesRegistry) {
- String type="Job";
String id = normalize(duccId);
// Terminate
sb.append("<td valign=\"bottom\" class=\"ducc-col-terminate\">");
@@ -309,9 +309,8 @@ public class DuccHandlerClassic extends
sb.append("</td>");
// Size
sb.append("<td valign=\"bottom\" align=\"right\">");
- String size = job.getSchedulingInfo().getMemorySize();
- MemoryUnits units = job.getSchedulingInfo().getMemoryUnits();
- sb.append(getProcessMemorySize(duccId,type,size,units));
+ SizeBytes size = new SizeBytes(SizeBytes.Type.Bytes,job.getSchedulingInfo().getMemorySizeAllocatedInBytes());
+ sb.append(getProcessMemorySize(duccId,size));
sb.append("</td>");
// Total
sb.append("<td valign=\"bottom\" align=\"right\">");
@@ -443,7 +442,6 @@ public class DuccHandlerClassic extends
}
private void buildReservationsListEntry(HttpServletRequest request, StringBuffer sb, DuccId duccId, IDuccWork duccwork, DuccData duccData, long now) {
- String type="Reservation";
String id = normalize(duccId);
String reservationType = "Unmanaged";
if(duccwork instanceof DuccWorkJob) {
@@ -742,11 +740,18 @@ public class DuccHandlerClassic extends
}
}
sb.append("</td>");
- // Size
+ //
+ IDuccSchedulingInfo si = duccwork.getSchedulingInfo();
+ SizeBytes size;
+ // Size (given)
+ sb.append("<td align=\"right\">");
+ size = new SizeBytes(SizeBytes.Type.Bytes,si.getMemorySizeAllocatedInBytes());
+ sb.append(getProcessMemorySize(duccId,size));
+ sb.append("</td>");
+ // Size (requested)
sb.append("<td align=\"right\">");
- String size = duccwork.getSchedulingInfo().getMemorySize();
- MemoryUnits units = duccwork.getSchedulingInfo().getMemoryUnits();
- sb.append(getProcessMemorySize(duccId,type,size,units));
+ size = new SizeBytes(si.getMemoryUnits().name(),Long.parseLong(si.getMemorySizeRequested()));
+ sb.append(getProcessMemorySize(duccId,size));
sb.append("</td>");
// Host Names
sb.append("<td>");
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Fri Sep 11 13:04:08 2015
@@ -44,6 +44,7 @@ import org.apache.uima.ducc.cli.ws.json.
import org.apache.uima.ducc.cli.ws.json.ReservationFactsList;
import org.apache.uima.ducc.common.IDuccEnv;
import org.apache.uima.ducc.common.NodeConfiguration;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties.DaemonName;
import org.apache.uima.ducc.common.internationalization.Messages;
@@ -64,8 +65,8 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccReservation;
import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
+import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
import org.apache.uima.ducc.transport.event.common.IDuccState.ReservationState;
-import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IRationale;
@@ -126,7 +127,6 @@ public class DuccHandlerJsonFormat exten
}
private JsonArray buildJobRow(HttpServletRequest request, IDuccWorkJob job, DuccData duccData, long now, ServicesRegistry servicesRegistry) {
- String type="Job";
JsonArray row = new JsonArray();
StringBuffer sb;
DuccId duccId = job.getDuccId();
@@ -344,9 +344,8 @@ public class DuccHandlerJsonFormat exten
// Size
sb = new StringBuffer();
sb.append("<span>");
- String size = job.getSchedulingInfo().getMemorySize();
- MemoryUnits units = job.getSchedulingInfo().getMemoryUnits();
- sb.append(getProcessMemorySize(duccId,type,size,units));
+ SizeBytes size = new SizeBytes(SizeBytes.Type.Bytes,job.getSchedulingInfo().getMemorySizeAllocatedInBytes());
+ sb.append(getProcessMemorySize(duccId,size));
sb.append("</span>");
row.add(new JsonPrimitive(sb.toString()));
// Total
@@ -540,7 +539,6 @@ public class DuccHandlerJsonFormat exten
private JsonArray buildReservationRow(HttpServletRequest request, IDuccWork duccwork, DuccData duccData, long now) {
- String type="Reservation";
JsonArray row = new JsonArray();
String reservationType = "Unmanaged";
if(duccwork instanceof DuccWorkJob) {
@@ -840,12 +838,21 @@ public class DuccHandlerJsonFormat exten
}
}
row.add(new JsonPrimitive(sb.toString()));
- // Size
+ //
+ IDuccSchedulingInfo si = duccwork.getSchedulingInfo();
+ SizeBytes size;
+ // Size (given)
+ sb = new StringBuffer();
+ sb.append("<span>");
+ size = new SizeBytes(SizeBytes.Type.Bytes,duccwork.getSchedulingInfo().getMemorySizeAllocatedInBytes());
+ sb.append(getProcessMemorySize(duccId,size));
+ sb.append("</span>");
+ row.add(new JsonPrimitive(sb.toString()));
+ // Size (requested)
sb = new StringBuffer();
sb.append("<span>");
- String size = duccwork.getSchedulingInfo().getMemorySize();
- MemoryUnits units = duccwork.getSchedulingInfo().getMemoryUnits();
- sb.append(getProcessMemorySize(duccId,type,size,units));
+ size = new SizeBytes(si.getMemoryUnits().name(),Long.parseLong(si.getMemorySizeRequested()));
+ sb.append(getProcessMemorySize(duccId,size));
sb.append("</span>");
row.add(new JsonPrimitive(sb.toString()));
// List
@@ -2065,7 +2072,8 @@ public class DuccHandlerJsonFormat exten
userProcesses = DuccMachinesData.getInstance().getUserProcesses(reservation.getUniqueNodes(),user);
list = reservation.getNodes();
}
- String size = getProcessMemorySize(reservation.getDuccId(),"Reservation",reservation.getSchedulingInfo().getMemorySize(),reservation.getSchedulingInfo().getMemoryUnits());
+ SizeBytes resSize = new SizeBytes(SizeBytes.Type.Bytes,reservation.getSchedulingInfo().getMemorySizeAllocatedInBytes());
+ String size = ""+getProcessMemorySize(reservation.getDuccId(),resSize);
String description = reservation.getStandardInfo().getDescription();
ReservationFacts facts = new ReservationFacts(id,start,end,user,rclass,state,reason,allocation,userProcesses,size,list,description);
factsList.add(facts);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java Fri Sep 11 13:04:08 2015
@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentSk
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeConfiguration;
+import org.apache.uima.ducc.common.SizeBytes;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
@@ -190,7 +191,7 @@ public class NodeViz
String user = si.getUser();
String duccid = service_id == null ? Long.toString(w.getDuccId().getFriendly()) : service_id; // UIMA-4209
- int jobmem = Integer.parseInt(sti.getMemorySize());
+ int jobmem = (int) (sti.getMemorySizeAllocatedInBytes()/SizeBytes.GB);
String sclass = sti.getSchedulingClass();
int quantum = default_quantum;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/reservations.jsp
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/reservations.jsp?rev=1702451&r1=1702450&r2=1702451&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/reservations.jsp (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/reservations.jsp Fri Sep 11 13:04:08 2015
@@ -47,6 +47,7 @@ if (table_style.equals("scroll")) {
$('td:eq(9)', nRow).css( "text-align", "right" );
$('td:eq(10)', nRow).css( "text-align", "right" );
$('td:eq(11)', nRow).css( "text-align", "right" );
+ $('td:eq(12)', nRow).css( "text-align", "right" );
return nRow;
},
} );
@@ -115,7 +116,8 @@ if (table_style.equals("scroll")) {
<th title="The current state of this reservation">State</th>
<th title="The reason for the final state of this reservation, normally CanceledByUser">Reason</th>
<th title="The combined number of PIDs on the assigned resource(s) for user">User<br>Processes</th>
- <th title="The memory allocation per resource for this reservation, in GB">Size</th>
+ <th title="The memory allocation for this reservation, in GB">Size<br/><small>(given)</small></th>
+ <th title="The memory request for this reservation, in GB">Size<br/><small>(requested)</small></th>
<th title="The machine(s) assigned to this reservation">Host<br>Names</th>
<th title="The user specified description of this reservation">Description</th>
</tr>
@@ -146,7 +148,8 @@ if (table_style.equals("classic")) {
<th title="The current state of this reservation">State</th>
<th title="The reason for the final state of this reservation, normally CanceledByUser">Reason</th>
<th title="The combined number of PIDs on the assigned resource(s) for user">User<br>Processes</th>
- <th title="The memory allocation per resource for this reservation, in GB">Size</th>
+ <th title="The memory allocation for this reservation, in GB">Size<br/><small>(given)</small></th>
+ <th title="The memory request for this reservation, in GB">Size<br/><small>(requested)</small></th>
<th title="The machine(s) assigned to this reservation">Host<br>Names</th>
<th title="The user specified description of this reservation">Description</th>
</tr>