You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2013/09/11 21:17:12 UTC
svn commit: r1521995 [1/2] - in /uima/sandbox/uima-ducc/trunk:
src/main/admin/ uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/ u...
Author: challngr
Date: Wed Sep 11 19:17:11 2013
New Revision: 1521995
URL: http://svn.apache.org/r1521995
Log:
UIMA-3260 Revamped RM class config
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/IllegalConfigurationException.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccSchedulerClasses.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IScheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/JobManagerUpdate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/check_ducc Wed Sep 11 19:17:11 2013
@@ -92,13 +92,16 @@ class CheckDucc(DuccUtil):
print " Reap user processes. This uses kill -9 and ducc_ling to forcibly terrrminate user processes."
print " Only processes specified by '-u' or '--userid' are targeted."
print ""
+ print " -v --verbose"
+ print " If specified, print the validated configuration to the console."
+ print ""
print " -? prints this message."
sys.exit(1)
def main(self, argv):
try:
- opts, args = getopt.getopt(argv, 'cikn:pqrs:u:h?v', ['--configuration', '--nodelist=', '--user=', '--int', '--quit', '--kill', '--pids', '--reap', '--version'])
+ opts, args = getopt.getopt(argv, 'cikn:opqrs:u:h?v', ['--configuration', '--nodelist=', '--user=', '--int', '--quit', '--kill', '--pids', '--reap', '--verbose' ])
except:
self.usage("Invalid arguments " + ' '.join(argv))
@@ -112,6 +115,7 @@ class CheckDucc(DuccUtil):
do_validate = False
checkdate = 0
config_only = False
+ verbose = False
for ( o, a ) in opts:
if o in ('-c', '--configuration'):
@@ -143,10 +147,10 @@ class CheckDucc(DuccUtil):
# intended to be called recursively from check_ducc, NOT from the command line
do_validate = True
checkdate = float(a)
+ elif o in ('-v', '--verbose'):
+ verbose = True
elif o in ('-h', '-?', '--help'):
self.usage(None)
- elif o in ('-v', '--version'):
- self.version(None)
else:
print 'badarg', a
usage('bad arg: ' + a)
@@ -210,10 +214,17 @@ class CheckDucc(DuccUtil):
nodes['local'] = localnodes
self.verify_jvm()
- if self.verify_class_configuration(nodes, check_nodepools):
- print "OK: Class configuration checked"
+
if ( config_only ):
+ if ( len(nodefiles) > 1 ):
+ print "NOTOK: Config check: specify a single master nodefile only."
+ return
+ if self.verify_class_configuration(nodefiles[0], verbose):
+ print "OK: Class configuration checked"
+ else:
+ print "NOTOK: Errors in class or node configuration."
+
return
# checking starts here
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_util.py Wed Sep 11 19:17:11 2013
@@ -609,126 +609,31 @@ class DuccUtil(DuccBase):
return True
return False
- #
- # Make sure all the nodes in the configured nodepools are also in the startup list
- #
- def check_nodepools(self, classprops, allnodes):
- #
- # First make sure that all the nodepools that are declared have definition files
- # and that the defined nodes are in some nodelist.
- #
- nodepools_ok = True
-
- nplist = classprops.get('scheduling.nodepool')
- if ( nplist == None ):
- return nodepools_ok
-
- nodepools = nplist.split()
- for np in nodepools:
- npkey = 'scheduling.nodepool.' + np
- npfilename = classprops.get(npkey)
- if ( npfilename == None ):
- print 'NOTOK: Missing nodepool definition file for Nodepool "' + np + '"'
- nodepools_ok = False
- continue
-
- npfile = self.DUCC_HOME + '/resources/' + npfilename
- if ( not os.path.exists(npfile) ):
- print 'NOTOK: Cannot find nodepool file "' + npfile + '"'
- continue
-
- npnodes = {}
- npnodes = self.read_nodefile(npfile, npnodes)
- found = False
- for ( impfile, nodes ) in npnodes.items():
- if len(nodes) == 0:
- continue
- for node in nodes:
- for (nodefile, nodelist) in allnodes.items():
- for n in nodelist:
- if ( self.compare_nodes(n, node)):
- found = True
- break
- if ( not found ):
- print 'NOTOK: Cannot find node defined in pool "' +np+'" in any nodefile:', node
- nodepools_ok = False
+ def verify_class_configuration(self, allnodes, verbose):
- if ( nodepools_ok ):
- print 'OK: All nodepools are verified'
- else:
- print 'NOTOK: some nodepools are not correctly defined.'
-
- return nodepools_ok
-
- def verify_class_configuration(self, allnodes, must_verify_nodepools):
+ print 'allnodes', allnodes
answer = True
# first, find the class definition
classfile = self.ducc_properties.get('ducc.rm.class.definitions')
- classfile = self.resolve(classfile, self.propsfile) # resolve the classfile relative to ducc.properties
print 'Class definition file is', classfile
- classprops = DuccProperties()
- try:
- classprops.load(classfile)
- except:
- print 'NOTOK: Cannot read properties file', classfile
- return False
-
- # Verify nodepool definitions.
- if ( must_verify_nodepools and (not self.check_nodepools(classprops, allnodes)) ):
- # this check will emit necessary messages
- answer = False
-
- nodepools = classprops.get('scheduling.nodepool')
- if ( nodepools == None ):
- nodepools = [] # avoid NPE if none
+ CMD = self.jvm
+ CMD = CMD + " -DDUCC_HOME=" + self.DUCC_HOME
+ CMD = CMD + " org.apache.uima.ducc.common.NodeConfiguration "
+ CMD = CMD + " -v " + allnodes
+ if ( verbose ):
+ CMD = CMD + " -p "
else:
- nodepools = nodepools.split()
-
- class_set = classprops.get('scheduling.class_set').split()
- # first, make sure every class that is defined exists, has a policy, and a priority
- # FAIR_SHARE classes, they must also have a weight
- # if a nodeppol is assigned, it must also be one of the defined, and now verified, nodepools
- for cl in class_set:
- po = classprops.get('scheduling.class.' + cl +'.policy')
- if ( po == None ):
- print 'NOTOK: Missing policy definition for class "' + cl + '"'
- answer = False
- else:
- we = classprops.get('scheduling.class.' + cl +'.share_weight')
- if ( po == 'FAIR_SHARE' and we == None ):
- print 'NOTOK: Missing "weight" definition for class: "' + cl + '"'
- answer = False
-
- pr = classprops.get('scheduling.class.' + cl +'.priority')
- if ( pr == None ):
- print 'NOTOK: Missing priority definition for class: "' + cl + '"'
- answer = False
-
- clnp = classprops.get('scheduling.class.' + cl +'.nodepool')
- if ( clnp != None ):
- if ( not clnp in nodepools ):
- print 'NOTOK: Nodepool "' + clnp + '" is configured for class "' + cl + '" but has no definition.'
- answer = False
-
- # Dig out the jobdriver class and insure it exists.
- jdclass = self.ducc_properties.get('ducc.jd.host.class')
- if ( not jdclass in class_set ):
- print 'NOTOK: Job Driver class "' + jdclass + '" is not defined (see ducc.properties: ducc.jd.host.class).'
- answer = False
-
- # if a default.name and/or default.name.reserve class is defined, make sure they exist
- default_class = classprops.get('scheduling.default.name')
- if ( (default_class != None) and (not default_class in class_set) ):
- print 'NOTOK: Default class "' + default_class + '" is not defined.'
- answer = False
-
- default_reserve_class = classprops.get('scheduling.default.name.reserve')
- if ( (default_reserve_class != None) and (not default_reserve_class in class_set) ):
- print 'NOTOK: Default reserve class "' + default_reserve_class + '" is not defined.'
- answer = False
+ CMD = CMD + " "
+ CMD = CMD + classfile
+ print CMD
+ rc = os.system(CMD)
+ if ( rc == 0 ):
+ print "OK: Class and node definitions validated."
+ else:
+ print "NOTOK: Cannot validate class and/or node definitions."
- return answer
+ return (rc == 0)
def __init__(self):
DuccBase.__init__(self)
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/start_ducc Wed Sep 11 19:17:11 2013
@@ -308,8 +308,8 @@ class StartDucc(DuccUtil):
print "Can't read nodefile", nf
ok = False
- if ok and (len(nodefiles) > 0):
- if self.verify_class_configuration(nodes, must_verify_nodepools):
+ if ok and (len(nodefiles) == 1):
+ if self.verify_class_configuration(nodefiles[0], False):
print "OK: Class configuration checked"
else:
print "NOTOK: Bad configuration, cannot start."
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java Wed Sep 11 19:17:11 2013
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Properties;
import org.apache.commons.cli.MissingArgumentException;
-import org.apache.uima.ducc.cli.IUiOptions.UiOption;
import org.apache.uima.ducc.cli.aio.AllInOneLauncher;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
import org.apache.uima.ducc.common.utils.DuccSchedulerClasses;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/aio/AllInOneLauncher.java Wed Sep 11 19:17:11 2013
@@ -391,7 +391,7 @@ public class AllInOneLauncher extends Cl
mh.frameworkTrace(cid, mid, exit);
}
- private void examine_scheduling_class() throws MissingArgumentException {
+ private void examine_scheduling_class() throws Exception {
String mid = "examine_scheduling_class";
mh.frameworkTrace(cid, mid, enter);
String pname = UiOption.SchedulingClass.pname();
@@ -752,7 +752,7 @@ public class AllInOneLauncher extends Cl
}
}
- private void examine() throws MissingArgumentException, IllegalArgumentException, IOException {
+ private void examine() throws Exception {
String mid = "examine";
mh.frameworkTrace(cid, mid, "enter");
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java?rev=1521995&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/NodeConfiguration.java Wed Sep 11 19:17:11 2013
@@ -0,0 +1,995 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.StringTokenizer;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccProperties;
+import org.apache.uima.ducc.common.utils.IllegalConfigurationException;
+
+
+
+
+/**
+ * This class reads and parses a node configuration file. It is used primarily by RM for scheduling
+ * and by the web server to present the configuration.
+ */
+public class NodeConfiguration
+{
+ String config_file_name = null;
+ BufferedReader in;
+ int lineno = 0;
+ DuccProperties defaultFairShareClass = new DuccProperties();
+ DuccProperties defaultFixedShareClass = new DuccProperties();
+ DuccProperties defaultReserveClass = new DuccProperties();
+ DuccProperties defaultNodepool = new DuccProperties();
+
+ Map<String, DuccProperties> np_set;
+ Map<String, DuccProperties> class_set;
+
+ Map<String, String> allNodefiles;
+
+ DuccLogger logger;
+ String defaultDomain = null;
+ String firstNodepool = null;
+
+ DuccProperties fairShareDefault = null;
+ DuccProperties reserveDefault = null;
+ String ducc_home = null;
+ String default_domain = null;
+
+ public NodeConfiguration(String config_file_name, DuccLogger logger)
+ {
+ this.config_file_name = config_file_name;
+ this.logger = logger;
+ this.allNodefiles = new HashMap<String, String>();
+
+ ducc_home = System.getProperty("DUCC_HOME");
+
+ defaultFairShareClass.put("type", "class");
+ defaultFairShareClass.put("name", "defaultFairShareClass");
+ defaultFairShareClass.put("policy", "FAIR_SHARE");
+ defaultFairShareClass.put("weight", "100");
+ defaultFairShareClass.put("priority", "10");
+ defaultFairShareClass.put("cap", Integer.toString(Integer.MAX_VALUE));
+ defaultFairShareClass.put("expand-by-doubling", "true");
+ defaultFairShareClass.put("initialization-cap", "2");
+ defaultFairShareClass.put("use-prediction", "true");
+ defaultFairShareClass.put("max-processes", Integer.toString(Integer.MAX_VALUE));
+ defaultFairShareClass.put("prediction-fudge", "60000");
+ defaultFairShareClass.put("nodepool", "<required>");
+ defaultFairShareClass.put("debug", "fixed");
+ defaultFairShareClass.put("abstract", "<optional>");
+ defaultFairShareClass.put("children", "<optional>");
+ defaultFairShareClass.put("parent", "<optional>");
+ defaultFairShareClass.put("debug", "<optional>");
+ defaultFairShareClass.put("default", "<optional>");
+ defaultFairShareClass.put("name", "<required>"); // required, but always filled in by the parser. needed here for validation.
+
+ defaultFixedShareClass.put("type", "class");
+ defaultFixedShareClass.put("name", "defaultFixedShareClass");
+ defaultFixedShareClass.put("abstract", "<optional>");
+ defaultFixedShareClass.put("children", "<optional>");
+ defaultFixedShareClass.put("parent", "<optional>");
+ defaultFixedShareClass.put("policy", "FIXED_SHARE");
+ defaultFixedShareClass.put("priority", "5");
+ defaultFixedShareClass.put("default", "<optional>");
+ defaultFixedShareClass.put("max-processes", Integer.toString(Integer.MAX_VALUE));
+ defaultFixedShareClass.put("cap", Integer.toString(Integer.MAX_VALUE));
+ defaultFixedShareClass.put("nodepool", "<required>");
+
+ defaultReserveClass.put("type", "class");
+ defaultReserveClass.put("name", "defaultReserveClass");
+ defaultReserveClass.put("abstract", "<optional>");
+ defaultReserveClass.put("children", "<optional>");
+ defaultReserveClass.put("parent", "<optional>");
+ defaultReserveClass.put("policy", "RESERVE");
+ defaultReserveClass.put("priority", "1");
+ defaultReserveClass.put("default", "<optional>");
+ defaultReserveClass.put("max-machines", Integer.toString(Integer.MAX_VALUE));
+ defaultReserveClass.put("cap", Integer.toString(Integer.MAX_VALUE));
+ defaultReserveClass.put("nodepool", "<required>");
+ defaultReserveClass.put("enforce", "true");
+
+ defaultNodepool.put("type", "nodepool");
+ defaultNodepool.put("name", "<optional>");
+ defaultNodepool.put("nodefile", "<optional>");
+ defaultNodepool.put("parent", "<optional>");
+ defaultNodepool.put("domain", "<optional>");
+ }
+
+ /**
+ * Resolve a filename relative to DUCC_HOME
+ */
+ String resolve(String file)
+ throws IllegalConfigurationException
+ {
+ if ( !file.startsWith("/") ) {
+ file = ducc_home + "/resources/" + file;
+ }
+ File f = new File(file);
+ if ( ! f.exists() ) {
+ throw new IllegalConfigurationException("File " + file + " does not exist or cannot be read.");
+ }
+ return file;
+ }
+
+ void logInfo(String methodName, String message)
+ {
+ if ( logger == null ) {
+ System.out.println(message);
+ } else {
+ logger.info(methodName, null, message);
+ }
+ }
+
+ void logWarn(String methodName, String message)
+ {
+ if ( logger == null ) {
+ System.out.println(message);
+ } else {
+ logger.warn(methodName, null, message);
+ }
+ }
+
+ void logError(String methodName, String message)
+ {
+ if ( logger == null ) {
+ System.out.println(message);
+ } else {
+ logger.error(methodName, null, message);
+ }
+ }
+
+ /**
+ * Use the NodeIdentity to infer my the domain name.
+ *
+ * Itertate through the possible names - if one of them has a '.'
+ * the we have to assume the following stuff is the domain name.
+ * We only get one such name, so we give up the search if we find
+ * it.
+ */
+ private String getDomainName()
+ {
+ // String methodName = "getDomainName";
+
+ if ( defaultDomain != null ) return defaultDomain;
+
+ InetAddress me = null;
+ try {
+ me = InetAddress.getLocalHost();
+ } catch (UnknownHostException e1) {
+ // TODO Auto-generated catch block
+ e1.printStackTrace();
+ }
+ String my_happy_name = me.getHostName();
+ String my_canonical_name = me.getCanonicalHostName();
+
+ if ( my_canonical_name.startsWith(my_happy_name) ) {
+ int ndx = my_canonical_name.indexOf(".");
+ return my_canonical_name.substring(ndx+1);
+ }
+ return null;
+ }
+
+ Map<String, String> allNodes = new HashMap<String, String>(); // To insure no duplicates
+ Map<String, String> readNodepoolFile(String npfile, String domain, boolean skip)
+ throws IllegalConfigurationException
+ {
+ //String methodName = "readNodepoolFile";
+ allNodefiles.put(npfile, npfile);
+ String ducc_home = System.getProperty("DUCC_HOME");
+ npfile = resolve(npfile);
+
+ Map<String, String> response = new HashMap<String, String>();
+
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(npfile));
+ String node = "";
+ while ( (node = br.readLine()) != null ) {
+ int ndx = node.indexOf("#");
+ if ( ndx >= 0 ) {
+ node = node.substring(0, ndx);
+ }
+ node = node.trim();
+ if (node.equals("") ) {
+ continue;
+ }
+
+ if ( node.startsWith("domain") ) {
+ String[] tmp = node.split("\\s");
+ if ( tmp.length == 2 ) {
+ domain = tmp[1];
+ continue;
+ } else {
+ throw new IllegalConfigurationException("Invalid domain specification in node file " + npfile + ": " + node);
+ }
+ }
+
+ if ( node.startsWith("import") ) {
+ String[] tmp = node.split("\\s");
+ if ( allNodefiles.containsKey(tmp[1]) ) {
+ if ( skip ) continue;
+ throw new IllegalConfigurationException("Duplicate imported nodefile " + tmp[1] + " found in " + npfile + ", not allowed.");
+ }
+ response.putAll(readNodepoolFile(tmp[1], domain, skip));
+ continue;
+ }
+
+ if ( allNodes.containsKey(node) ) {
+ throw new IllegalConfigurationException("Duplicate node found in " + npfile + ": " + node + "; first occurance in " + allNodes.get(node));
+ }
+ allNodes.put(node, npfile);
+ response.put(node, node);
+
+ // include fully and non-fully qualified names to allow sloppiness of config
+
+ ndx = node.indexOf(".");
+ String dnode = null;
+ if ( ndx >= 0 ) {
+ dnode = node.substring(0, ndx);
+ response.put(dnode, dnode);
+ } else if ( domain != null ) {
+ dnode = node + "." + domain;
+ response.put(dnode, dnode);
+ }
+ if( dnode != null ) {
+ if ( allNodes.containsKey(dnode) ) {
+ throw new IllegalConfigurationException("Duplicate node found in " + npfile + ": " + dnode + "; first occurance in " + allNodes.get(dnode));
+ }
+ allNodes.put(dnode, npfile);
+ }
+
+ }
+ br.close();
+
+ } catch (FileNotFoundException e) {
+ throw new IllegalConfigurationException("Cannot open NodePool file \"" + npfile + "\": file not found.");
+ } catch (IOException e) {
+ throw new IllegalConfigurationException("Cannot read NodePool file \"" + npfile + "\": I/O Error.");
+ }
+
+// for (String s : response.keySet() ) {
+// System.out.println(npfile + ": " + s);
+// }
+ return response;
+ }
+
+ /**
+ * Provide a continual stream of lines, removing empty lines and comment lines.
+ */
+ String readLine()
+ throws IOException
+ {
+ String line = null;
+ while ( (line = in.readLine()) != null ) {
+ lineno++;
+ // System.out.println("Line[" + lineno + "]: " + line);
+ line = line.trim();
+ if ( line.equals("") ) continue;
+ if ( line.startsWith("#") ) continue;
+ return line;
+ }
+ return null;
+ }
+
+ /**
+ * Fill up the token buffer.
+ */
+ StringTokenizer buf = null;
+ boolean fillBuf()
+ throws IOException
+ {
+ while ( (buf == null) || !buf.hasMoreTokens() ) {
+ String line = readLine();
+ if ( line == null ) return false;
+ buf = new StringTokenizer(line, "\n\t\r\f{} =;", true);
+ }
+ return true;
+ }
+
+ /**
+ * Provide a continual stream of tokens, throwing out whitespace and semocolons
+ */
+ String nextToken()
+ throws IOException
+ {
+ while ( fillBuf() ) {
+ String tok = null;
+ while ( buf.hasMoreTokens() ) {
+ tok = buf.nextToken();
+ if ( tok.equals(" ") ) continue;
+ if ( tok.equals("\t") ) continue;
+ if ( tok.equals(";") ) continue; // optional semicolon, ignored
+ return tok;
+ }
+ }
+ return null;
+ }
+
+ void parseInternal(DuccProperties props)
+ throws IOException,
+ IllegalConfigurationException
+ {
+// System.out.println("Parsing nodepool " + name);
+// if ( parent == null ) {
+// System.out.println(" <base>");
+// } else {
+// System.out.println(" Inherits from " + parent);
+// }
+
+ String tok = null;
+ while ( (tok = nextToken() ) != null ) {
+ if ( tok.equals("}") ) return;
+
+ String k = tok;
+ if ( k.equals("{") ) {
+ throw new IllegalConfigurationException("Missing '}' near line " + lineno + " in " + config_file_name);
+ }
+ String v = nextToken();
+ if ( v.equals("=") ) v = nextToken(); // (optionally allow k v or k=v)
+
+ if ( v.equals("}") ) {
+ throw new IllegalConfigurationException("Missing value near line " + lineno + " in " + config_file_name);
+ }
+ if ( v.equals("{") ) {
+ throw new IllegalConfigurationException("Missing '}' near line " + lineno + " in " + config_file_name);
+ }
+
+ // note we allow duplicate entries, which turn into a list
+ if ( props.getProperty(k) == null ) {
+ props.put(k, v);
+ } else {
+ throw new IllegalConfigurationException("Duplicate property near line " + lineno + " in " + config_file_name + ": " + k);
+ }
+ }
+ return;
+ }
+
+ DuccProperties parseNodepool(String name, String parent)
+ throws IOException,
+ IllegalConfigurationException
+ {
+// System.out.println("Parsing nodepool " + name);
+// if ( parent == null ) {
+// System.out.println(" <base>");
+// } else {
+// System.out.println(" Inherits from " + parent);
+// }
+
+ if ( firstNodepool == null ) {
+ firstNodepool = name;
+ }
+
+ DuccProperties ret = new DuccProperties();
+ ret.put("type", "nodepool");
+ ret.put("name", name);
+ if ( parent != null ) {
+ throw new IllegalConfigurationException("Illegal inheritance (inheritance not supported for nodepools) near line " + lineno + " in " + config_file_name);
+ }
+
+ parseInternal(ret);
+ String dd = ret.getProperty("domain");
+ if ( name.equals(firstNodepool) ) {
+ defaultDomain = dd;
+ } else {
+ if ( dd != null ) {
+ throw new IllegalConfigurationException("Default domain specified in nodepool other than first nodepool \"" + firstNodepool + "\", not allowed, near line " + lineno + " in " + config_file_name);
+ }
+ }
+
+ supplyDefaults(ret, defaultNodepool);
+
+ return ret;
+ }
+
+ DuccProperties parseClass(String name, String parent)
+ throws IOException,
+ IllegalConfigurationException
+ {
+ DuccProperties ret = new DuccProperties();
+ ret.put("type", "class");
+ ret.put("name", name);
+ if ( parent != null ) {
+ ret.put("parent", parent);
+ }
+
+ parseInternal(ret);
+
+ return ret;
+ }
+
+ ArrayList<DuccProperties> nodepools = new ArrayList<DuccProperties>();
+ ArrayList<DuccProperties> classes = new ArrayList<DuccProperties>();
+ DuccProperties parseStanzas()
+ throws IOException,
+ IllegalConfigurationException
+ {
+ String tok;
+ while ( (tok = nextToken()) != null ) {
+ String type = tok; // stanza type
+
+ String name = nextToken(); // stanza name
+ if ( name == null ) {
+ throw new IllegalConfigurationException("Missing stanza name near line " + lineno + " in " + config_file_name);
+ }
+
+ String parent = nextToken(); // who to inherit from, or "{"
+ String start = null;
+ if ( parent.equals("{") ) {
+ start = parent;
+ parent = null;
+ } else {
+ start = nextToken();
+ }
+ if ( ! start.equals("{") ) {
+ throw new IllegalConfigurationException("Missing '{' near line " + lineno + " in " + config_file_name);
+ }
+
+ if ( type.equals("Nodepool") ) nodepools.add(parseNodepool(name, parent));
+ if ( type.equals("Class") ) classes.add(parseClass(name, parent));
+ }
+ return null;
+ }
+
+ /**
+ * Given the 'in' properties, look through the 'model' propertis and make sure there are
+ * no unsupported properties, and that all required properties are filled in. Works for both
+ * Class and Nodepool properties.
+ */
+ void supplyDefaults(DuccProperties in, DuccProperties model)
+ throws IllegalConfigurationException
+ {
+ // first make sure all properties for the input object are valid
+ String name = in.getProperty("name");
+ String type = in.getProperty("type");
+ for (Object o : in.keySet()) {
+ String k = (String) o;
+ if ( model.get(k) == null ) { // key not in model is illegal
+ throw new IllegalConfigurationException("Illegal property \"" + k + "\" in " + type + " " + name);
+ }
+ }
+
+ // now make sure all required fields are supplied and fill in defaults
+ for ( Object o : model.keySet() ) {
+ String k = (String) o;
+ String vm = model.getProperty(k);
+ String vi = in.getProperty(k);
+
+ if ( vi == null) {
+ if ( vm.equals("<required>" ) ) {
+ throw new IllegalConfigurationException("Missing required property " + k + " in " + type + " " + name);
+ }
+
+ if ( vm.equals("<optional>") ) { // its optional but there is no meaningful default
+ continue;
+ }
+
+ // System.out.println("Object " + name + " inherits " + k + "," + vm + " from " + model.get("name"));
+ in.put(k, vm); // fill in the default
+ }
+ }
+ }
+
+ Map<String, DuccProperties> npmap = new HashMap<String, DuccProperties>();
+ Map<String, DuccProperties> clmap = new HashMap<String, DuccProperties>();
+ ArrayList<DuccProperties> independentNodepools = new ArrayList<DuccProperties>();
+ ArrayList<String> independentClasses = new ArrayList<String>();
+
+ /**
+ * Propogate my properties to all my kids and their kids, etc. Classes only, Nodepools don't
+ * do any sort of inheritance.
+ */
+ void propogateDown(String clname)
+ {
+ DuccProperties cl = clmap.get(clname);
+ String children = cl.getStringProperty("children", null);
+ if ( children == null ) return;
+
+ String[] kids = children.split("\\s+");
+ for ( String kid : kids ) {
+ DuccProperties kp = clmap.get(kid);
+
+ for ( Object o : cl.keySet() ) {
+ if ( ! kp.containsKey(o) ) {
+ String k = (String) o;
+ if ( k.equals("abstract" ) ) continue; // don't propogate down abstractness
+ if ( k.equals("children" ) ) continue; // don't propogate down children
+ if ( k.equals("default" ) ) continue; // don't propogate down default
+ String v = cl.getStringProperty(k);
+ // System.out.println("Object " + kp.get("name") + " inherits " + k + "," + v + " from " + cl.get("name"));
+ kp.put(k, v);
+ }
+ }
+ propogateDown(kid);
+ }
+ }
+
+ void handleDefault(DuccProperties p, DuccProperties def, String policy)
+ throws IllegalConfigurationException
+ {
+ String dflt = p.getProperty("default");
+ if ( dflt != null ) {
+ if ( def != null ) {
+ throw new IllegalConfigurationException("Class " + p.getProperty("name")
+ + ": Only one " + policy + " default allowed. Already defined in class \""
+ + def.getProperty("name")
+ + "\"");
+ } else {
+ if ( policy.equals("FAIR_SHARE" ) ) {
+ fairShareDefault = p;
+ } else {
+ reserveDefault = p;
+ }
+ }
+ }
+ }
+
+ /**
+ * Map all the classes and do inheritance.
+ * Check values for correctness.
+ */
+ void doClassInheritance()
+ throws IllegalConfigurationException
+ {
+ // map the clases, crash on duplicates
+ for ( DuccProperties p : classes ) {
+ String name = p.getStringProperty("name");
+ if ( clmap.containsKey(name) ) {
+ throw new IllegalConfigurationException("Duplicate class: " + name);
+ }
+ clmap.put(name, p);
+ }
+
+ // now establish the parent -> child relationships
+ for ( DuccProperties p : clmap.values() ) {
+ String parent = p.getProperty("parent");
+ String name = p.getProperty("name");
+
+ if ( (p.getProperty("abstract") != null) &&
+ (p.getProperty("default") != null ) ) {
+ throw new IllegalConfigurationException("Class " + name + ": Abstract class is not allowed to specify \"default\"");
+ }
+
+
+ if ( parent == null ) {
+ independentClasses.add(name);
+ } else {
+ DuccProperties par_cl = clmap.get(parent);
+ if ( par_cl == null ) {
+ throw new IllegalConfigurationException("Class " + name + " parent pool " + parent + " cannot be found.");
+ }
+ String children = par_cl.getStringProperty("children", null);
+ if ( children == null ) {
+ children = name;
+ } else {
+ children = children + " " + name;
+ }
+ par_cl.put("children", children);
+ }
+ }
+
+ // now starting at every root, propogate stuff down
+ for ( String s : independentClasses ) {
+ propogateDown(s);
+ }
+
+ // must fill in defaults, which we couldn't do until we finished inheritance
+ for ( DuccProperties p : clmap.values() ) {
+ String policy = p.getStringProperty("policy", null);
+ String name = p.getProperty("name");
+
+ if ( policy == null ) {
+ throw new IllegalConfigurationException("Class " + name + " is missing scheduling policy ");
+ }
+ if ( policy.equals("FAIR_SHARE") ) {
+ handleDefault(p, fairShareDefault, policy);
+ supplyDefaults(p, defaultFairShareClass);
+ } else
+ if ( policy.equals("FIXED_SHARE") ) {
+ handleDefault(p, reserveDefault, policy);
+ supplyDefaults(p, defaultFixedShareClass);
+ } else
+ if ( policy.equals("RESERVE") ) {
+ handleDefault(p, reserveDefault, policy);
+ supplyDefaults(p, defaultReserveClass);
+ } else {
+ throw new IllegalConfigurationException("Unknown scheduling policy \"" + policy + "\" in class " + name);
+ }
+ }
+
+ // remove the abstract classes as they are no longer needed and we don't want to let them leak out
+ // where somebody might think they're ok to use
+ Iterator<String> iter = clmap.keySet().iterator();
+ while ( iter.hasNext() ) {
+ String k = iter.next();
+ DuccProperties p = clmap.get(k);
+ if ( p.containsKey("abstract") ) {
+ // System.out.println("---------------- Remove " + p.get("name"));
+ iter.remove();
+ }
+ }
+
+ }
+
+ /**
+ * Find all the top-level nodepools.
+ * Make sure every parent nodepool exists.
+ * Set the names of the child nodepools into each parent.
+ * Set the names of the classes managed in each nodepool into the nodepools.
+ */
+ void connectNodepools()
+ throws IllegalConfigurationException
+ {
+ // map the nodepools, crashing on duplicates
+ for ( DuccProperties p : nodepools ) {
+ String name = p.getStringProperty("name");
+ if ( npmap.containsKey(name) ) {
+ throw new IllegalConfigurationException("Duplicate nodepool: " + name);
+ }
+
+ npmap.put(name, p);
+ }
+
+ // map the child nodepools into their parents
+ for ( DuccProperties p : npmap.values() ) {
+ String parent = p.getStringProperty("parent", null);
+ String name = p.getStringProperty("name");
+ if ( parent == null ) {
+ independentNodepools.add(p);
+ } else {
+ DuccProperties par_pool = npmap.get(parent);
+ if ( par_pool == null ) {
+ throw new IllegalConfigurationException("Nodepool " + name+ " parent pool " + parent + " cannot be found.");
+ }
+ @SuppressWarnings("unchecked")
+ List<DuccProperties> children = (List<DuccProperties>) par_pool.get("children");
+ if ( children == null ) {
+ children = new ArrayList<DuccProperties>();
+ par_pool.put("children", children);
+ }
+ children.add(p);
+ }
+ }
+
+ // connect the classes into their nodepools
+ for ( DuccProperties p : classes ) {
+ if ( p.containsKey("abstract") ) continue; // don't propogate these out
+
+ String name = p.getStringProperty("name");
+
+ String npname = p.getStringProperty("nodepool", null);
+ if ( npname == null ) {
+ throw new IllegalConfigurationException("Class " + name + " is not assigned to a nodepool.");
+ }
+ DuccProperties np = npmap.get(npname);
+ if ( np == null ) {
+ throw new IllegalConfigurationException("Class " + name + " assigned to nodepool " + npname + " but nodepool does not exist.");
+ }
+
+ @SuppressWarnings("unchecked")
+ List<DuccProperties> class_set = (List<DuccProperties>) np.get("classes");
+ if ( class_set == null ) {
+ class_set = new ArrayList<DuccProperties>();
+ np.put("classes", class_set);
+ }
+ class_set.add(p);
+
+ }
+ }
+
+ void readNodefile(DuccProperties p, String domain)
+ throws IllegalConfigurationException
+ {
+ String npfile = p.getProperty("nodefile");
+ if ( npfile != null ) {
+ p.put("nodes", readNodepoolFile(npfile, domain, false));
+ }
+
+ @SuppressWarnings("unchecked")
+ List<DuccProperties> children = (List<DuccProperties>) p.get("children");
+ if ( children != null ) {
+ for ( DuccProperties pc : children ) {
+ readNodefile(pc, domain);
+ }
+ }
+ }
+
+ /**
+ * Read the complete node configuration as defined in. Intended for use from command line, not
+ * usually elsewhere.
+ */
+ void fullValidation(String global_nodefile)
+ throws IllegalConfigurationException
+ {
+ global_nodefile = resolve(global_nodefile);
+ if ( allNodefiles.containsKey(global_nodefile) ) return; // already passed if is's there and we're here
+ readNodepoolFile(global_nodefile, default_domain, true); // will throw if there's an issue
+ }
+
+ public DuccProperties getDefaultFairShareClass()
+ {
+ return fairShareDefault;
+ }
+
+ public DuccProperties getDefaultReserveClass()
+ {
+ return reserveDefault;
+ }
+
+ public DuccProperties[] getToplevelNodepools()
+ {
+ return independentNodepools.toArray(new DuccProperties[independentNodepools.size()]);
+ }
+
+ public DuccProperties getClass(String name)
+ {
+ if ( clmap == null ) return null;
+ return clmap.get(name);
+ }
+
+ public Map<String, DuccProperties> getClasses()
+ {
+ return clmap;
+ }
+
+ public void readConfiguration()
+ throws FileNotFoundException,
+ IOException,
+ IllegalConfigurationException
+ {
+ if ( ducc_home == null ) {
+ throw new IllegalConfigurationException("DUCC_HOME must be defined as a system property.");
+ }
+
+ config_file_name = resolve(config_file_name);
+ in = new BufferedReader(new FileReader(config_file_name));
+ default_domain = getDomainName();
+
+ parseStanzas();
+ doClassInheritance();
+ connectNodepools();
+
+ for (DuccProperties p : independentNodepools) { // walk the tree and read the node files
+ readNodefile(p, default_domain);
+ }
+
+ try {
+ in.close();
+ } catch (IOException e) {
+ // nothing ... who cares if we got this far, and what can we do anyway ?
+ }
+ }
+
+ void printNodepool(DuccProperties p, String indent)
+ {
+ String methodName = "printNodepool";
+
+ logInfo(methodName, indent + "Nodepool " + p.getProperty("name"));
+ String nodefile = p.getProperty("nodeile");
+ logInfo(methodName, indent + " Node File: " + (nodefile == null ? "None" : nodefile));
+
+ @SuppressWarnings("unchecked")
+ List<DuccProperties> class_set = (List<DuccProperties>) p.get("classes");
+ if ( class_set == null ) {
+ logInfo(methodName, indent + " No classes defined.");
+ } else {
+ StringBuffer buf = new StringBuffer();
+ buf.append(indent + " Classes:");
+ for (DuccProperties cp : class_set ) {
+ buf.append(" " + cp.get("name"));
+ }
+ logInfo(methodName, buf.toString());
+ }
+
+ @SuppressWarnings("unchecked")
+ List<DuccProperties> children = (List<DuccProperties>) p.get("children");
+
+ if ( children == null ) {
+ logInfo(methodName, indent + " No subpools.\n");
+ } else {
+ StringBuffer buf = new StringBuffer();
+ buf.append(indent + " Subpools:");
+ for ( DuccProperties cp : children ) {
+ buf.append(" " + cp.get("name"));
+ }
+ logInfo(methodName, buf.toString());
+ for ( DuccProperties cp : children ) {
+ printNodepool(cp, indent + indent);
+ }
+ }
+ }
+
+ void printProperty(String k, Object v)
+ {
+ String methodName = "printProperty";
+ if ( v == null ) return; // ignore non-existant properties
+ logInfo(methodName, String.format(" %-20s: %s", k, v));
+ }
+
+ /**
+ * Print class values in controlled order and format, not to be confused
+ * with what you'd get just iterating the map.
+ */
+ void printClass(DuccProperties cl)
+ {
+ String methodName = "printClass";
+ logInfo(methodName, "Class " + cl.get("name"));
+ printProperty("Policy", cl.get("policy"));
+ printProperty("Nodepool", cl.get("nodepool"));
+ printProperty("Priority", cl.get("priority"));
+ printProperty("Weight", cl.get("weight"));
+ printProperty("Debug", cl.get("debug"));
+ printProperty("Cap", cl.get("cap"));
+ printProperty("Expand By Doubling", cl.get("expand-by-doubling"));
+ printProperty("Initialization Cap", cl.get("initialization-cap"));
+ printProperty("Use Prediction", cl.get("use-prediction"));
+ printProperty("Prediction Fudge", cl.get("prediction-fudge"));
+ printProperty("Max Processes", cl.get("max-processes"));
+ printProperty("Max Machines", cl.get("max-machines"));
+ printProperty("Enforce Memory Size",cl.get("enforce"));
+
+ logInfo(methodName, "");
+ }
+
+ public void printConfiguration()
+ {
+ String methodName = "printConfiguration";
+
+ if ( independentNodepools == null || clmap == null ) {
+ logError(methodName, "Configuration has not been generated. (Run readConfiguration first.)");
+ }
+
+ // First iterate nodepools
+ for ( DuccProperties p : independentNodepools ) { // it's a tree, use recursion
+ printNodepool(p, " ");
+ }
+
+ DuccProperties[] class_set = clmap.values().toArray(new DuccProperties[clmap.size()]);
+ Arrays.sort(class_set, new ClassSorter());
+
+ for ( DuccProperties p : class_set ) {
+ printClass(p);
+ }
+ }
+
+ static void usage(String msg)
+ {
+ if ( msg != null ) {
+ System.out.println(msg);
+ System.out.println("");
+ }
+ System.out.println("Usage:");
+ System.out.println(" NodeConfiguration [-p] [-v nodefile] <configfile>");
+ System.out.println("Where:");
+ System.out.println(" -p does validation and prints the configuration with all defaults filled in");
+ System.out.println(" If omitted, validation only is performed.");
+ System.out.println(" -v <nodefile> does full node validation against the startup nodefile. This is useful");
+ System.out.println(" when there are configured nodes not explicitly defined to a nodepool.");
+ System.out.println(" <configfile is the node configuration in ducc_runtime/resources.");
+ System.out.println(" -? show this help.");
+ System.out.println("");
+ System.exit(1);
+ }
+
+ /**
+ * Testing and verification of the file
+ */
+ public static void main(String[] args)
+ {
+
+ boolean doprint = false;
+ boolean validate_nodes = false;
+ String global_nodefile = null;
+ String config = null;
+
+ int i = 0;
+
+ // (simplistic, expects to be called from a script that does rigorous argument checking
+ // and setup)
+ for ( i = 0; i < args.length; i++ ) {
+ if ( args[i].equals("-p") ) {
+ doprint = true;
+ continue;
+ }
+
+ if ( args[i].equals("-v") ) {
+ validate_nodes = true;
+ global_nodefile = args[i+1];
+ i++;
+ continue;
+ }
+
+ if ( args[i].equals("-?") ) {
+ usage(null);
+ }
+
+ config = args[i];
+ }
+
+ NodeConfiguration nc = new NodeConfiguration(config, null);
+
+ int rc = 0;
+ try {
+ nc.readConfiguration(); // if it doesn't crash it must have worked
+
+ if ( validate_nodes ) {
+ nc.fullValidation(global_nodefile); // this too, gonna throw if there's an issue
+ }
+
+ if ( doprint ) {
+ nc.printConfiguration();
+ }
+ } catch (FileNotFoundException e) {
+ System.out.println("Configuration file " + config + " does not exist or cannot be read.");
+ rc = 1;
+ } catch (IOException e) {
+ System.out.println("IOError reading configuration file " + config + ": " + e.toString());
+ rc = 1;
+ } catch (IllegalConfigurationException e) {
+ System.out.println(e.getMessage());
+ rc = 1;
+ }
+
+ System.exit(rc);
+ }
+
+ static public class ClassSorter
+ implements Comparator<DuccProperties>
+ {
+ public int compare(DuccProperties pr1, DuccProperties pr2)
+ {
+ // primary sort thus: FAIR_SHARE, FIXED_SHARE, RESERVE
+ // seconddary sort on class name
+
+
+ if ( pr1.equals(pr2) ) return 0; // deal with the 'equals' contract
+
+ String p1 = pr1.getProperty("policy");
+ String p2 = pr2.getProperty("policy");
+
+ // happily, the string values sort the way we want so we can eliminate a lot of stuff
+ // by just checking equality.
+ if ( ! p1.equals(p2) ) {
+ return p1.compareTo(p2);
+ }
+
+ // and if they're the same, just tiebreak on the name
+ String n1 = pr1.getProperty("name");
+ String n2 = pr2.getProperty("name");
+ return n1.compareTo(n2);
+ }
+ }
+
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccSchedulerClasses.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccSchedulerClasses.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccSchedulerClasses.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccSchedulerClasses.java Wed Sep 11 19:17:11 2013
@@ -19,12 +19,11 @@
package org.apache.uima.ducc.common.utils;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
import java.util.ArrayList;
+import java.util.Map;
import java.util.Properties;
-import org.apache.commons.lang.StringUtils;
+import org.apache.uima.ducc.common.NodeConfiguration;
public class DuccSchedulerClasses {
@@ -34,7 +33,11 @@ public class DuccSchedulerClasses {
public static final String JobDriver = "JobDriver";
private static DuccSchedulerClasses instance = null;
-
+
+ private long lastModified = 0;
+ NodeConfiguration nodeConfiguration = null;
+ DuccLogger logger = null;
+
private String fileName = null;
public static DuccSchedulerClasses getInstance() {
@@ -52,7 +55,18 @@ public class DuccSchedulerClasses {
fileName = dir_home+File.separator+dir_resources+File.separator+file_classes;
}
+ /**
+ * Pass in a logger, usefull in the daemons. If no logger the NodeConfiguration will
+ * write to stdout/stderr.
+ */
+ public DuccSchedulerClasses(DuccLogger logger) {
+ super();
+ this.logger = logger;
+ }
+
public String getProperty(Properties properties, String name) {
+ if ( properties == null ) return null;
+
String retVal = "";
String property = properties.getProperty(name);
if(property != null) {
@@ -61,103 +75,102 @@ public class DuccSchedulerClasses {
return retVal;
}
- public DuccProperties getClasses() {
- DuccProperties properties = new DuccProperties();
- try {
- File file = new File(fileName);
- FileInputStream fis = new FileInputStream(file);
- properties.load(fis);
- fis.close();
- }
- catch(IOException e) {
- e.printStackTrace();
- }
- return properties;
+ public NodeConfiguration readConfiguration()
+ throws Exception
+ {
+ instance = getInstance();
+
+ File file = new File(fileName);
+ if ( lastModified != file.lastModified() ) { // reread if it looks like it changed
+ lastModified = file.lastModified();
+ nodeConfiguration = new NodeConfiguration(fileName, logger);
+ lastModified = file.lastModified();
+ nodeConfiguration.readConfiguration();
+ }
+
+ return nodeConfiguration;
}
-
- public boolean isPreemptable(String class_name) {
+
+ public Map<String, DuccProperties> getClasses()
+ throws Exception
+ {
+ readConfiguration();
+ return nodeConfiguration.getClasses();
+ }
+
+ public boolean isPreemptable(String class_name)
+ throws Exception
+ {
boolean retVal = false;
- Properties properties = getClasses();
- String policy = getProperty(properties,"scheduling.class."+class_name+".policy");
+ readConfiguration();
+
+ DuccProperties properties = nodeConfiguration.getClass(class_name);
+ String policy = getProperty(properties, "policy");
if(policy.equals(FAIR_SHARE)) {
retVal = true;
}
return retVal;
}
- public String getDefaultClassName() {
+ public String getDefaultClassName()
+ throws Exception
+ {
String retVal = null;
- Properties properties = getClasses();
- String key = "scheduling.default.name";
- String value = properties.getProperty(key);
- if(value != null) {
- value = value.trim();
- if(value.length() > 0) {
- retVal = value;
- }
- }
+ readConfiguration();
+ DuccProperties properties = nodeConfiguration.getDefaultFairShareClass();
+ if ( properties != null ) {
+ retVal = properties.getProperty("name");
+ }
return retVal;
}
+ /**
+ * In the brave new world there is no default debug class.
+ */
public String getDebugClassDefaultName() {
- String retVal = null;
- Properties properties = getClasses();
- String key = "scheduling.default.name.debug";
- String value = properties.getProperty(key);
- if(value != null) {
- value = value.trim();
- if(value.length() > 0) {
- retVal = value;
- }
- }
- return retVal;
+ return null;
}
- public String getDebugClassSpecificName(String class_name) {
+ public String getDebugClassSpecificName(String class_name)
+ throws Exception
+ {
String retVal = null;
- if(class_name != null) {
- Properties properties = getClasses();
- String key = "scheduling.class."+class_name+".debug";
- String value = properties.getProperty(key);
- if(value != null) {
- value = value.trim();
- if(value.length() > 0) {
- retVal = value;
- }
- }
- }
+ readConfiguration();
+ DuccProperties properties = nodeConfiguration.getClass(class_name);
+ if ( properties != null ) {
+ retVal = properties.getProperty("debug");
+ }
return retVal;
}
- public String[] getReserveClasses() {
- ArrayList<String> classList = new ArrayList<String>();
- Properties properties = getClasses();
- String class_set = properties.getProperty("scheduling.class_set");
- class_set.trim();
- if(class_set != null) {
- String[] class_array = StringUtils.split(class_set);
- for(int i=0; i<class_array.length; i++) {
- String class_name = class_array[i].trim();
- String policy = getProperty(properties,"scheduling.class."+class_name+".policy");
- if(policy.equals(FIXED_SHARE)) {
- classList.add(class_name);
- }
- else if(policy.equals(RESERVE) && !class_name.equals(JobDriver)) {
- classList.add(class_name);
- }
- }
- }
+ public String[] getReserveClasses()
+ throws Exception
+ {
+ readConfiguration();
+ Map<String, DuccProperties> allClasses = nodeConfiguration.getClasses();
+ ArrayList<String> classList = new ArrayList<String>();
+ for ( DuccProperties p : allClasses.values() ) {
+ String pol = p.getProperty("policy");
+ String name = p.getProperty("name");
+ if ( (pol.equals(RESERVE) || pol.equals(FIXED_SHARE)) && ( !name.equals(JobDriver) ) ) {
+ classList.add(p.getProperty("name"));
+ }
+ }
+
String[] retVal = classList.toArray(new String[0]);
return retVal;
}
- public String getReserveClassDefaultName() {
- String name = "";
- Properties properties = getClasses();
- String value = properties.getProperty("scheduling.default.name.reserve");
- if(value != null) {
- name = value.trim();
- }
- return name;
+ public String getReserveClassDefaultName()
+ throws Exception
+ {
+ String retVal = "";
+ readConfiguration();
+ DuccProperties properties = nodeConfiguration.getDefaultReserveClass();
+ if ( properties != null ) {
+ retVal = properties.getProperty("name");
+ }
+ return retVal;
+
}
}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/IllegalConfigurationException.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/IllegalConfigurationException.java?rev=1521995&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/IllegalConfigurationException.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/IllegalConfigurationException.java Wed Sep 11 19:17:11 2013
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.utils;
+
+public class IllegalConfigurationException
+ extends Exception
+{
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public IllegalConfigurationException(String s)
+ {
+ super(s);
+ }
+
+ public IllegalConfigurationException(String s, Throwable cause)
+ {
+ super(s, cause);
+ }
+
+ public IllegalConfigurationException(Throwable cause)
+ {
+ super(cause);
+ }
+
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IScheduler.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/IScheduler.java Wed Sep 11 19:17:11 2013
@@ -18,7 +18,7 @@
*/
package org.apache.uima.ducc.rm.scheduler;
-import java.util.HashMap;
+import java.util.Map;
import org.apache.uima.ducc.rm.scheduler.SchedConstants.EvictionPolicy;
@@ -34,7 +34,7 @@ public interface IScheduler
{
public void schedule(SchedulingUpdate upd);
- public void setClasses(HashMap<ResourceClass, ResourceClass> classes); // classes
+ public void setClasses(Map<ResourceClass, ResourceClass> classes); // classes
public void setNodePool(NodePool nodepool);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/JobManagerUpdate.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/JobManagerUpdate.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/JobManagerUpdate.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/JobManagerUpdate.java Wed Sep 11 19:17:11 2013
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.rm.scheduler;
import java.util.HashMap;
+import java.util.Map;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -30,11 +31,11 @@ import org.apache.uima.ducc.common.utils
public class JobManagerUpdate
{
- private HashMap<DuccId, HashMap<Share, Share>> expandedShares = new HashMap<DuccId, HashMap<Share, Share>>(); // by job, which shares to add
+ private Map<DuccId, HashMap<Share, Share>> expandedShares = new HashMap<DuccId, HashMap<Share, Share>>(); // by job, which shares to add
- private HashMap<DuccId, HashMap<Share, Share>> shrunkenShares = new HashMap<DuccId, HashMap<Share, Share>>(); // by job, which shares to remove
+ private Map<DuccId, HashMap<Share, Share>> shrunkenShares = new HashMap<DuccId, HashMap<Share, Share>>(); // by job, which shares to remove
- private HashMap<DuccId, IRmJob> allJobs; // current state of all jobs
+ private Map<DuccId, IRmJob> allJobs; // current state of all jobs
@SuppressWarnings("unchecked")
public void setAllJobs(HashMap<DuccId, IRmJob> jobs)
@@ -42,7 +43,7 @@ public class JobManagerUpdate
allJobs = (HashMap<DuccId, IRmJob>) jobs.clone(); // shallow copy intentional
}
- public HashMap<DuccId, IRmJob> getAllJobs()
+ public Map<DuccId, IRmJob> getAllJobs()
{
return allJobs;
}
@@ -58,11 +59,11 @@ public class JobManagerUpdate
expandedShares.put(job.getId(), (HashMap<Share, Share>) shares.clone()); // shallow copy intentional
}
- public HashMap<DuccId, HashMap<Share, Share>> getExpandedShares() {
+ public Map<DuccId, HashMap<Share, Share>> getExpandedShares() {
return expandedShares;
}
- public HashMap<DuccId, HashMap<Share, Share>> getShrunkenShares() {
+ public Map<DuccId, HashMap<Share, Share>> getShrunkenShares() {
return shrunkenShares;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodePool.java Wed Sep 11 19:17:11 2013
@@ -37,7 +37,6 @@ class NodePool
{
DuccLogger logger = DuccLogger.getLogger(NodePool.class, COMPONENT_NAME);
String id;
- static final String globalName = "--global--";
NodePool parent = null;
int depth;
@@ -90,20 +89,20 @@ class NodePool
HashMap<Integer, HashMap<Node, Machine>> virtualMachinesByOrder;
static int maxorder = 0;
- NodePool(NodePool parent, EvictionPolicy ep, int order)
- {
- this.parent = parent;
- this.id = globalName;
- this.evictionPolicy = ep;
- this.depth = 0;
- this.order = order;
- }
+// NodePool(NodePool parent, String id, EvictionPolicy ep, int order)
+// {
+// this.parent = parent;
+// this.id = id;
+// this.evictionPolicy = ep;
+// this.depth = 0;
+// this.order = order;
+// }
- NodePool(NodePool parent, String id, Map<String, String> subpool, EvictionPolicy ep, int depth, int order)
+ NodePool(NodePool parent, String id, Map<String, String> nodes, EvictionPolicy ep, int depth, int order)
{
this.parent = parent;
this.id = id;
- this.subpoolNames = subpool;
+ this.subpoolNames = nodes;
this.evictionPolicy = ep;
this.depth = depth;
this.order = order;
@@ -590,7 +589,8 @@ class NodePool
np.reset(order);
}
- if ( id.equals(globalName) && ( updated > 0 ) ) {
+ if ( (parent == null) && ( updated > 0 ) ) {
+ // top-level nodepool will recurse
logger.info(methodName, null, "Scheduling Tables:\n", toString());
updated = 0;
}
@@ -665,10 +665,11 @@ class NodePool
* We can assume that all node updates are refused until all subpools are created
* so we don't have to worry about updating the pools until nodeArrives(), below.
*/
- void createSubpool(String className, Map<String, String> names, int order)
+ NodePool createSubpool(String className, Map<String, String> names, int order)
{
NodePool np = new NodePool(this, className, names, evictionPolicy, depth + 1, order);
children.put(className, np);
+ return np;
}
/**
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java Wed Sep 11 19:17:11 2013
@@ -42,7 +42,7 @@ public class NodepoolScheduler
{
DuccLogger logger = DuccLogger.getLogger(NodepoolScheduler.class, COMPONENT_NAME);
- HashMap<ResourceClass, ResourceClass> resourceClasses;
+ Map<ResourceClass, ResourceClass> resourceClasses;
Map<IRmJob, IRmJob> needyJobs = new TreeMap<IRmJob, IRmJob>(new JobByTimeSorter());
NodePool globalNodepool;
@@ -62,7 +62,7 @@ public class NodepoolScheduler
do_defragmentation = SystemPropertyResolver.getBooleanProperty("ducc.rm.defragmentation", do_defragmentation);
}
- public void setClasses(HashMap<ResourceClass, ResourceClass> prclasses)
+ public void setClasses(Map<ResourceClass, ResourceClass> prclasses)
{
this.resourceClasses = prclasses;
@@ -73,6 +73,7 @@ public class NodepoolScheduler
// i.e. we don't support mixed-share at this point.
HashMap<Integer, ArrayList<ResourceClass>> sorter = new HashMap<Integer, ArrayList<ResourceClass>>();
for ( ResourceClass pcl : prclasses.values() ) {
+
ArrayList<ResourceClass> cl = sorter.get(pcl.getPriority());
if ( cl == null ) {
cl = new ArrayList<ResourceClass>();
@@ -742,7 +743,7 @@ public class NodepoolScheduler
if ( np == globalNodepool ) {
ArrayList<ResourceClass> tmp = new ArrayList<ResourceClass>();
for ( ResourceClass rc : rcs ) {
- if (rc.getNodepoolName().equals(NodePool.globalName) ) {
+ if ( rc.getNodepoolName().equals(globalNodepool.getId()) ) {
tmp.add(rc);
}
}
@@ -2165,9 +2166,6 @@ public class NodepoolScheduler
public void schedule(SchedulingUpdate upd)
{
String methodName = "schedule";
-
- logger.info(methodName, null, "Machine occupancy before schedule");
- globalNodepool.queryMachines();
int jobcount = 0;
for ( ResourceClass rc : resourceClasses.values() ) {
@@ -2180,10 +2178,13 @@ public class NodepoolScheduler
}
if ( jobcount == 0 ) {
- logger.info(methodName, null, "No jobs to schedule");
+ logger.info(methodName, null, "No jobs to schedule under nodepool", globalNodepool.getId());
return;
}
+ logger.info(methodName, null, "Machine occupancy before schedule");
+ globalNodepool.queryMachines();
+
this.schedulingUpdate = upd;
//
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java?rev=1521995&r1=1521994&r2=1521995&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ResourceClass.java Wed Sep 11 19:17:11 2013
@@ -23,7 +23,6 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
import org.apache.uima.ducc.common.utils.SystemPropertyResolver;
@@ -35,7 +34,7 @@ public class ResourceClass
implements SchedConstants,
IEntity
{
- private DuccLogger logger = DuccLogger.getLogger(this.getClass(), COMPONENT_NAME);
+ //private DuccLogger logger = DuccLogger.getLogger(this.getClass(), COMPONENT_NAME);
private String id;
private Policy policy;
@@ -44,11 +43,11 @@ public class ResourceClass
private int share_weight; // for fair-share, the share weight to use
private int min_shares; // fixed-shre: min shares to hand out
- private int max_processes; // fixed-share: max shares to hand out regardless of
+ private int max_processes = 0; // fixed-share: max shares to hand out regardless of
// what is requested or what fair-share turns out to be
- private int max_machines; // reservation: max machines that can be reserved by a single user - global across
- // all this user's requests.
+ private int max_machines = 0; // reservation: max machines that can be reserved by a single user - global across
+ // all this user's requests.
// for reservation, this caps machines.
// for shares, this caps shares
@@ -80,7 +79,7 @@ public class ResourceClass
private boolean expand_by_doubling = true;
private int initialization_cap = 2;
- private int prediction_fudge = 10000;
+ private long prediction_fudge = 60000;
private boolean use_prediction = true;
private int[] given_by_order = null;
@@ -88,93 +87,134 @@ public class ResourceClass
private static Comparator<IEntity> apportionmentSorter = new ApportionmentSorterCl();
- public ResourceClass(String id) //, HashMap<String, Machine> machinesByName, HashMap<String, Machine> machinesByIp)
+ public ResourceClass(DuccProperties props)
{
- this.id = id;
- this.policy = Policy.FAIR_SHARE;
- this.share_weight = 100;
- this.priority = 10;
- this.max_processes = Integer.MAX_VALUE;
+ //
+ // We can assume everything useful is here because the parser insured it
+ //
+ this.id = props.getStringProperty("name");
+ this.policy = Policy.valueOf(props.getStringProperty("policy"));
+ this.priority = props.getIntProperty("priority");
this.min_shares = 0;
- this.max_machines = Integer.MAX_VALUE;
-
- this.absolute_cap = Integer.MAX_VALUE;
- this.percent_cap = 1.0;
-
- this.expand_by_doubling = SystemPropertyResolver.getBooleanProperty("ducc.rm.expand.by.doubling", true);
- this.initialization_cap = SystemPropertyResolver.getIntProperty("ducc.rm.initialization.cap", 2);
-
- this.use_prediction = SystemPropertyResolver.getBooleanProperty("ducc.rm.prediction", true);
- this.prediction_fudge = SystemPropertyResolver.getIntProperty("ducc.rm.prediction.fudge", 10000); // extra fudge factor to add in to the
- // projection, in miliseconds.
-
- //this.machinesByName = machinesByName;
- //this.machinesByIp = machinesByIp;
- }
-
- // TODO: sanity check
- // - emit warnings if shares are specified in reservations
- // if machines are sprcified for fair or fixed-share
- // etc.
- void init(DuccProperties props)
- {
- //String methodName = "init";
- String k = "scheduling.class." + id + ".";
- String s;
- s = props.getProperty(k + "policy");
-
- if ( s == null ) {
- throw new SchedulingException(null, "Configuration problem: no policy for class " + id + ".");
+ if ( policy == Policy.RESERVE ) {
+ this.max_machines = props.getIntProperty("max-machines");
+ this.enforce_memory = props.getBooleanProperty("enforce-memory", true);
}
- policy = Policy.valueOf(s);
-
- share_weight = props.getIntProperty(k + "share_weight" , DEFAULT_SHARE_WEIGHT);
- priority = props.getIntProperty(k + "priority", DEFAULT_PRIORITY);
- min_shares = props.getIntProperty(k + "min_shares", 0); // default no min
-
- switch ( policy ) {
- case FAIR_SHARE:
- max_processes = props.getIntProperty(k + "max_processes", DEFAULT_MAX_PROCESSES); // default no max
- max_machines = 0;
- break;
-
- case FIXED_SHARE:
- max_processes = props.getIntProperty(k + "max_processes", DEFAULT_MAX_PROCESSES); // default no max
- max_machines = 0;
- break;
-
- case RESERVE:
- max_processes = 0;
- max_machines = props.getIntProperty(k + "max_machines", DEFAULT_MAX_INSTANCES); // default max 1
- break;
+ if ( policy != Policy.RESERVE ) {
+ this.max_processes = props.getIntProperty("max-processes");
}
+
if ( max_processes <= 0 ) max_processes = Integer.MAX_VALUE;
if ( max_machines <= 0 ) max_machines = Integer.MAX_VALUE;
- enforce_memory = props.getBooleanProperty(k + "enforce.memory", true);
-
- initialization_cap = props.getIntProperty(k + "initialization.cap", initialization_cap);
- expand_by_doubling = props.getBooleanProperty(k + "expand.by.doubling", expand_by_doubling);
- use_prediction = props.getBooleanProperty(k + "prediction", use_prediction);
- prediction_fudge = props.getIntProperty(k + "prediction.fudge", prediction_fudge);
-
- s = props.getStringProperty(k + "cap", ""+Integer.MAX_VALUE); // default no cap
- if ( s.endsWith("%") ) {
- int t = Integer.parseInt(s.substring(0, s.length()-1));
- percent_cap = (t * 1.0 ) / 100.0;
+ this.absolute_cap = Integer.MAX_VALUE;
+ this.percent_cap = 1.0;
+
+ String cap = props.getStringProperty("cap");
+ if ( cap.endsWith("%") ) {
+ int t = Integer.parseInt(cap.substring(0, cap.length()-1));
+ this.percent_cap = (t * 1.0 ) / 100.0;
} else {
- absolute_cap = Integer.parseInt(s);
+ absolute_cap = Integer.parseInt(cap);
if (absolute_cap == 0) absolute_cap = Integer.MAX_VALUE;
}
- nodepoolName = props.getStringProperty(k + "nodepool", null); // optional nodepool
- if (nodepoolName == null) {
- nodepoolName = NodePool.globalName;
- }
+ if ( this.policy == Policy.FAIR_SHARE ) {
+ this.share_weight = props.getIntProperty("weight");
+ if ( props.containsKey("expand-by-doubling") ) {
+ this.expand_by_doubling = props.getBooleanProperty("expand-by-doubling", true);
+ } else {
+ this.expand_by_doubling = SystemPropertyResolver.getBooleanProperty("ducc.rm.expand.by.doubling", true);
+ }
+
+ if ( props.containsKey("initialization-cap") ) {
+ this.initialization_cap = props.getIntProperty("initialization-cap");
+ } else {
+ this.initialization_cap = SystemPropertyResolver.getIntProperty("ducc.rm.initialization.cap", 2);
+ }
+
+ if ( props.containsKey("use-prediction") ) {
+ this.use_prediction = props.getBooleanProperty("use-prediction", true);
+ } else {
+ this.use_prediction = SystemPropertyResolver.getBooleanProperty("ducc.rm.prediction", true);
+ }
+
+ if ( props.containsKey("prediction-fudge") ) {
+ this.prediction_fudge = props.getLongProperty("prediction-fudge");
+ } else {
+ this.prediction_fudge = SystemPropertyResolver.getLongProperty("ducc.rm.prediction.fudge", 60000);
+ }
+ }
+
+ this.nodepoolName = props.getStringProperty("nodepool");
+
+
}
+// // TODO: sanity check
+// // - emit warnings if shares are specified in reservations
+// // if machines are sprcified for fair or fixed-share
+// // etc.
+// void init(DuccProperties props)
+// {
+// //String methodName = "init";
+// String k = "scheduling.class." + id + ".";
+// String s;
+// s = props.getProperty(k + "policy");
+
+// if ( s == null ) {
+// throw new SchedulingException(null, "Configuration problem: no policy for class " + id + ".");
+// }
+// policy = Policy.valueOf(s);
+
+// share_weight = props.getIntProperty(k + "share_weight" , DEFAULT_SHARE_WEIGHT);
+// priority = props.getIntProperty(k + "priority", DEFAULT_PRIORITY);
+// min_shares = props.getIntProperty(k + "min_shares", 0); // default no min
+
+// switch ( policy ) {
+// case FAIR_SHARE:
+// max_processes = props.getIntProperty(k + "max_processes", DEFAULT_MAX_PROCESSES); // default no max
+// max_machines = 0;
+// break;
+
+// case FIXED_SHARE:
+// max_processes = props.getIntProperty(k + "max_processes", DEFAULT_MAX_PROCESSES); // default no max
+// max_machines = 0;
+// break;
+
+// case RESERVE:
+// max_processes = 0;
+// max_machines = props.getIntProperty(k + "max_machines", DEFAULT_MAX_INSTANCES); // default max 1
+// break;
+
+// }
+// if ( max_processes <= 0 ) max_processes = Integer.MAX_VALUE;
+// if ( max_machines <= 0 ) max_machines = Integer.MAX_VALUE;
+
+// enforce_memory = props.getBooleanProperty(k + "enforce.memory", true);
+
+// initialization_cap = props.getIntProperty(k + "initialization.cap", initialization_cap);
+// expand_by_doubling = props.getBooleanProperty(k + "expand.by.doubling", expand_by_doubling);
+// use_prediction = props.getBooleanProperty(k + "prediction", use_prediction);
+// prediction_fudge = props.getIntProperty(k + "prediction.fudge", prediction_fudge);
+
+// s = props.getStringProperty(k + "cap", ""+Integer.MAX_VALUE); // default no cap
+// if ( s.endsWith("%") ) {
+// int t = Integer.parseInt(s.substring(0, s.length()-1));
+// percent_cap = (t * 1.0 ) / 100.0;
+// } else {
+// absolute_cap = Integer.parseInt(s);
+// if (absolute_cap == 0) absolute_cap = Integer.MAX_VALUE;
+// }
+
+// nodepoolName = props.getStringProperty(k + "nodepool", null); // optional nodepool
+// if (nodepoolName == null) {
+// nodepoolName = NodePool.globalName;
+// }
+// }
+
public long getTimestamp()
{
return 0;
@@ -220,7 +260,7 @@ public class ResourceClass
return use_prediction;
}
- public int getPredictionFudge()
+ public long getPredictionFudge()
{
return prediction_fudge;
}
@@ -418,11 +458,10 @@ public class ResourceClass
void removeJob(IRmJob j)
{
- String methodName = "removeJob";
if ( ! allJobs.containsKey(j) ) {
if ( j.isRefused() ) return;
- logger.error(methodName, j.getId(), "Priority class", getName(), "cannot find job to remove.");
+ throw new SchedulingException(j.getId(), "Priority class " + getName() + " cannot find job to remove.");
}
allJobs.remove(j);