You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/04/10 22:17:39 UTC
svn commit: r1466657 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/
uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/
Author: degenaro
Date: Wed Apr 10 20:17:38 2013
New Revision: 1466657
URL: http://svn.apache.org/r1466657
Log:
UIMA-2804 DUCC orchestrator (OR) CGroups support for JD and JPs
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java?rev=1466657&r1=1466656&r2=1466657&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/CGroupManager.java Wed Apr 10 20:17:38 2013
@@ -18,15 +18,13 @@
*/
package org.apache.uima.ducc.orchestrator;
-import org.apache.uima.ducc.common.utils.id.IDuccId;
import org.apache.uima.ducc.transport.event.common.CGroup;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
public class CGroupManager {
- public static void assign(IDuccProcess process) {
- IDuccId duccId = process.getDuccId();
- CGroup cgroup = new CGroup(duccId);
+ public static void assign(IDuccProcess process, long max_size_in_bytes) {
+ CGroup cgroup = new CGroup(max_size_in_bytes);
process.setCGroup(cgroup);
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java?rev=1466657&r1=1466656&r2=1466657&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/JobFactory.java Wed Apr 10 20:17:38 2013
@@ -69,6 +69,15 @@ public class JobFactory {
return jobFactory;
}
+ private long driver_max_size_in_bytes = 0;
+
+ public JobFactory() {
+ String ducc_jd_share_quantum = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_jd_share_quantum);
+ long oneKB = 1024;
+ long oneMB = 1024*oneKB;
+ driver_max_size_in_bytes = Long.parseLong(ducc_jd_share_quantum) * oneMB;
+ }
+
private OrchestratorCommonArea orchestratorCommonArea = OrchestratorCommonArea.getInstance();
private IDuccIdFactory duccIdFactory = orchestratorCommonArea.getDuccIdFactory();
private JobDriverHostManager hostManager = orchestratorCommonArea.getHostManager();
@@ -332,7 +341,7 @@ public class JobFactory {
DuccId duccId = jdIdFactory.next();
duccId.setFriendly(0);
DuccProcess driverProcess = new DuccProcess(duccId,nodeIdentity,ProcessType.Pop);
- CGroupManager.assign(driverProcess);
+ CGroupManager.assign(driverProcess, driver_max_size_in_bytes);
driverProcess.setResourceState(ResourceState.Allocated);
driverProcess.setNodeIdentity(nodeIdentity);
driver.getProcessMap().put(driverProcess.getDuccId(), driverProcess);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1466657&r1=1466656&r2=1466657&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java Wed Apr 10 20:17:38 2013
@@ -30,6 +30,7 @@ import org.apache.uima.ducc.common.NodeI
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
+import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.TimeStamp;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent;
@@ -78,6 +79,16 @@ public class StateManager {
return stateManager;
}
+ private long quantum_size_in_bytes = 0;
+
+ public StateManager() {
+ String ducc_rm_share_quantum = DuccPropertiesResolver.getInstance().getFileProperty(DuccPropertiesResolver.ducc_rm_share_quantum);
+ long oneKB = 1024;
+ long oneMB = 1024*oneKB;
+ long oneGB = 1024*oneMB;
+ quantum_size_in_bytes = Long.parseLong(ducc_rm_share_quantum) * oneGB;
+ }
+
private OrchestratorCommonArea orchestratorCommonArea = OrchestratorCommonArea.getInstance();
private Messages messages = orchestratorCommonArea.getSystemMessages();
private DuccWorkMap workMap = orchestratorCommonArea.getWorkMap();
@@ -799,6 +810,7 @@ public class StateManager {
Iterator<DuccId> resourceMapIterator = resourceMap.keySet().iterator();
while(resourceMapIterator.hasNext()) {
DuccId duccId = resourceMapIterator.next();
+ IResource resource = resourceMap.get(duccId);
Node node = resourceMap.get(duccId).getNode();
NodeIdentity nodeId = node.getNodeIdentity();
if(!processMap.containsKey(duccId)) {
@@ -814,7 +826,8 @@ public class StateManager {
break;
}
DuccProcess process = new DuccProcess(duccId, node, processType);
- CGroupManager.assign(process);
+ long process_max_size_in_bytes = quantum_size_in_bytes * resource.countShares();
+ CGroupManager.assign(process, process_max_size_in_bytes);
orchestratorCommonArea.getProcessAccounting().addProcess(duccId, duccWorkJob.getDuccId());
processMap.addProcess(process);
process.setResourceState(ResourceState.Allocated);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java?rev=1466657&r1=1466656&r2=1466657&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/CGroup.java Wed Apr 10 20:17:38 2013
@@ -28,12 +28,12 @@ public class CGroup implements Serializa
private static final long serialVersionUID = 1L;
private IDuccId id;
- private long maxMemoryLimit;
+ private long maxMemoryLimit; // in bytes
private boolean reservation;
private int shares;
- public CGroup(IDuccId duccId) {
- setId(id);
+ public CGroup(long max_size_in_bytes) {
+ setMaxMemoryLimit(max_size_in_bytes);
}
public int getShares() {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java?rev=1466657&r1=1466656&r2=1466657&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java Wed Apr 10 20:17:38 2013
@@ -25,7 +25,6 @@ import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.node.metrics.ProcessGarbageCollectionStats;
import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.common.utils.id.IDuccId;
import org.apache.uima.ducc.transport.agent.IUimaPipelineAEComponent;
import org.apache.uima.ducc.transport.event.common.IDuccProcessType.ProcessType;
import org.apache.uima.ducc.transport.event.common.IProcessState.ProcessState;