You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/09/23 19:50:44 UTC
svn commit: r1704906 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/
uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/
uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/
Author: challngr
Date: Wed Sep 23 17:50:44 2015
New Revision: 1704906
URL: http://svn.apache.org/viewvc?rev=1704906&view=rev
Log:
UIMA-4569 Supply per-node quantum from NodeConfiguration.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1704906&r1=1704905&r2=1704906&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java Wed Sep 23 17:50:44 2015
@@ -65,14 +65,16 @@ public class NodeConfiguration
Map<String, DuccProperties> usermap = new HashMap<String, DuccProperties>(); // all users, by name UIMA-4275
ArrayList<String> independentClasses = new ArrayList<String>(); // all classes that don't derive from something
- Map<String, String> allNodes = new HashMap<String, String>(); // map node -> nodepool name, map for dup checking
+ Map<String, String> allNodes = new HashMap<String, String>(); // map node -> nodefile name, map for dup checking
Map<String, DuccProperties> poolsByNodefile = new HashMap<String, DuccProperties>(); // nodepool node file -> nodepool props
+ Map<String, DuccProperties> poolsByNodeName = new HashMap<String, DuccProperties>(); // Nodepools, by node
Map<String, String> allImports = new HashMap<String, String>(); // map nodefile -> importer, map for dup checking
Map<String, String> referrers = new HashMap<String, String>(); // map nodefile -> referring nodepool, for dup checking
DuccLogger logger;
String defaultDomain = null;
+ int defaultQuantum = 15;
String firstNodepool = null;
boolean fairShareExists = false;
@@ -651,7 +653,7 @@ public class NodeConfiguration
for (DuccProperties props : independentNodepools ) {
String q = props.getProperty("share-quantum");
if ( q == null ) {
- props.setProperty("share-quantum", ""+SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15));
+ props.setProperty("share-quantum", "" + defaultQuantum);
} else {
try {
Integer.parseInt(q); // insure it's a number
@@ -797,6 +799,7 @@ public class NodeConfiguration
}
allNodes.put(node, nodefile); // for dup checking - we only get to read a node once
nodes.put(node, nodefile); // UIMA-4142 map host -> domain
+ poolsByNodeName.put(node, p); // So we can find pool-related things for the node
// include fully and non-fully qualified names to allow sloppiness of config
ndx = node.indexOf(".");
@@ -998,7 +1001,18 @@ public class NodeConfiguration
return reserveDefault;
}
- public int getShareQuantum(String classname)
+ public int getQuantumForNode(String node)
+ {
+ DuccProperties np = poolsByNodeName.get(node);
+ if ( np == null ) {
+ // it has to be the default np, so use the default quantum
+ return defaultQuantum;
+ }
+ // otherwise it's required that the quantum for the nodepool is set to something
+ return Integer.parseInt(np.getProperty("share-quantum"));
+ }
+
+ public int getQuantumForClass(String classname)
throws IllegalConfigurationException
{
// to find the quantum for a class -
@@ -1051,6 +1065,7 @@ public class NodeConfiguration
throw new IllegalConfigurationException("DUCC_HOME must be defined as a system property.");
}
defaultDomain = getDomainName();
+ defaultQuantum = SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", 15);
try {
String tconfig_file_name = resolve(config_file_name);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1704906&r1=1704905&r2=1704906&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Wed Sep 23 17:50:44 2015
@@ -18,10 +18,7 @@
*/
package org.apache.uima.ducc.ws;
-import java.util.ArrayList;
import java.util.List;
-import java.util.ListIterator;
-import java.util.TreeMap;
import org.apache.uima.ducc.common.IDuccEnv;
import org.apache.uima.ducc.common.config.CommonConfiguration;
@@ -87,23 +84,6 @@ public class DuccBoot extends Thread {
return limit;
}
- private static TreeMap<Integer,String> sort(ArrayList<String> list) {
- String location = "sort";
- TreeMap<Integer,String> map = new TreeMap<Integer,String>();
- ListIterator<String> listIterator = list.listIterator();
- while(listIterator.hasNext()) {
- try {
- String value = listIterator.next().trim();
- int index = value.indexOf('.');
- Integer key = Integer.parseInt(value.substring(0,index));
- map.put(key, value);
- }
- catch(Throwable t) {
- logger.warn(location, jobid, t);
- }
- }
- return map;
- }
// private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) {
// String location = "restoreReservations";
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1704906&r1=1704905&r2=1704906&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java Wed Sep 23 17:50:44 2015
@@ -19,8 +19,6 @@
package org.apache.uima.ducc.ws.server.nodeviz;
-import java.io.FileNotFoundException;
-import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
@@ -33,7 +31,6 @@ import org.apache.uima.ducc.common.SizeB
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
-import org.apache.uima.ducc.common.utils.IllegalConfigurationException;
import org.apache.uima.ducc.common.utils.SystemPropertyResolver;
import org.apache.uima.ducc.common.utils.Version;
import org.apache.uima.ducc.transport.event.OrchestratorStateDuccEvent;
@@ -125,11 +122,6 @@ public class NodeViz
IDuccWorkMap jobmap = ev.getWorkMap();
- int job_shares = 0;
- int service_shares = 0;
- int pop_shares = 0;
- int reservation_shares = 0;
-
int job_gb = 0;
int service_gb = 0;
int pop_gb = 0;
@@ -189,7 +181,7 @@ public class NodeViz
String sclass = sti.getSchedulingClass();
int quantum = default_quantum;
try {
- quantum = nc.getShareQuantum(sclass);
+ quantum = nc.getQuantumForClass(sclass);
} catch ( Exception e ) {
// this most likely caused by a reconfigure so that a job's class no longer exists. nothing to do about it
// but punt and try not to crash.
@@ -219,15 +211,12 @@ public class NodeViz
switch ( type ) {
case Job:
- job_shares += qshares;
job_gb += jobmem;
break;
- case Pop:
- pop_shares += qshares;
+ case Pop:
pop_gb += jobmem;
break;
- case Service:
- service_shares += qshares;
+ case Service:
service_gb += jobmem;
break;
}
@@ -253,7 +242,6 @@ public class NodeViz
IDuccReservationMap rm = de.getReservationMap();
logger.debug(methodName, w.getDuccId(), "Receive:", type, w.getStateObject(), "processes[", rm.size(), "] Completed:", w.isCompleted());
- reservation_shares += qshares;
reservation_gb += jobmem;
for ( IDuccReservation r: rm.values()) {
@@ -298,9 +286,7 @@ public class NodeViz
String key = strip(s); // our key, possibly with domain stripped
if ( ! hosts.containsKey(key) ) {
- // System.out.println("Set host from MachineInfo with key :" + key + ":");
-
- VisualizedHost vh = new VisualizedHost(mi, default_quantum);
+ VisualizedHost vh = new VisualizedHost(mi, nc.getQuantumForNode(s));
hosts.put(key, vh);
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java?rev=1704906&r1=1704905&r2=1704906&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java Wed Sep 23 17:50:44 2015
@@ -77,18 +77,9 @@ class VisualizedHost
this.name = NodeViz.strip(info.getName());
this.ip = info.getIp();
- //TODO
- String ns = "";
- if ( false && ns == "" || ns == null ) {
- this.mem = 0;
- this.shares = 0;
- this.mem_reservable = 0;
- } else {
- this.mem = Integer.parseInt(info.getMemFree());
- this.shares = (mem / quantum);
- this.mem_reservable = shares * quantum;
- }
-
+ this.mem = Integer.parseInt(info.getMemFree());
+ this.shares = (mem / quantum);
+ this.mem_reservable = shares * quantum;
this.shares_free = shares;
}