You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2014/05/20 21:26:52 UTC
svn commit: r1596375 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/
uima-ducc-rm/src/main/java/org/apache/uima...
Author: challngr
Date: Tue May 20 19:26:52 2014
New Revision: 1596375
URL: http://svn.apache.org/r1596375
Log:
UIMA-3824 Initial query occupancy.
Added:
uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancy.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
Added: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy?rev=1596375&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy (added)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy Tue May 20 19:26:52 2014
@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -----------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# -----------------------------------------------------------------------
+
+
+import os
+import sys
+import getopt
+
+from ducc_util import DuccUtil
+
+class DuccRmQOccupancy(DuccUtil):
+
+ def usage(self, msg):
+ if ( msg != None ):
+ print msg
+
+ print 'rm_qoccupancy --console | --compact'
+ print ' This returns details of each DUCC node.'
+ print ' --console formats the output in human-readable form'
+ print ' --compact formats the output for scripting'
+ print ''
+ print 'Example:'
+ print ' rm_qoccupancy --console'
+ print ''
+ print ' rm_qoccupancy --compact'
+ sys.exit(1)
+
+ def main(self, argv):
+
+ format = None
+
+ try:
+ opts, args = getopt.getopt(argv, 'h?', ['console', 'compact'])
+ except:
+ self.usage("Invalid arguments " + ' '.join(argv))
+
+ for ( o, a ) in opts:
+ if o in ('--console'):
+ format = o
+ if o in ('--compact'):
+ format = o
+ elif o in ('-h', '?'):
+ self.usage(None)
+
+ if ( format == None ):
+ self.usage("Must specify --console or --compact")
+
+ DUCC_JVM_OPTS = ' -Dducc.deploy.configuration=' + self.DUCC_HOME + "/resources/ducc.properties "
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -DDUCC_HOME=' + self.DUCC_HOME
+ DUCC_JVM_OPTS = DUCC_JVM_OPTS + ' -Dducc.head=' + self.ducc_properties.get('ducc.head')
+
+ self.spawn(self.java(), DUCC_JVM_OPTS, 'org.apache.uima.ducc.common.main.DuccRmAdmin', '--qoccupancy', format)
+
+ return
+
+if __name__ == "__main__":
+ stopper = DuccRmQOccupancy()
+ stopper.main(sys.argv[1:])
+
+
Propchange: uima/sandbox/uima-ducc/trunk/src/main/admin/rm_qoccupancy
------------------------------------------------------------------------------
svn:executable = *
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancy.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancy.java?rev=1596375&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancy.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancy.java Tue May 20 19:26:52 2014
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.admin.event;
+
+public class RmAdminQOccupancy
+ extends DuccAdminEvent
+{
+ private static final long serialVersionUID = -8101741014979144426L;
+
+ public RmAdminQOccupancy()
+ {
+
+ }
+
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java?rev=1596375&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmAdminQOccupancyReply.java Tue May 20 19:26:52 2014
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.admin.event;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class RmAdminQOccupancyReply
+ extends RmAdminReply
+{
+ private static final long serialVersionUID = -8101741014979144426L;
+
+
+ List<RmQueriedMachine> machines = new ArrayList<RmQueriedMachine>();
+
+ public RmAdminQOccupancyReply()
+ {
+ super(null);
+ }
+
+ public void addMachine(RmQueriedMachine m)
+ {
+ machines.add(m);
+ }
+
+ public List<RmQueriedMachine> getMachines()
+ {
+ return machines;
+ }
+
+
+ public String toConsole()
+ {
+ Collections.sort(machines, new MachineByOrderSorter());
+ StringBuffer sb = new StringBuffer();
+ sb.append(RmQueriedMachine.header());
+ sb.append("\n");
+ for ( RmQueriedMachine m : machines ) {
+ sb.append(m.toConsole());
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ public String toCompact()
+ {
+ // ( no need to sort, this is intended for script scraping so we let the script order things as it wants )
+ StringBuffer sb = new StringBuffer();
+ for ( RmQueriedMachine m : machines ) {
+ sb.append(m.toCompact());
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+
+ class MachineByOrderSorter
+ implements Comparator<RmQueriedMachine>
+ {
+ public int compare(RmQueriedMachine m1, RmQueriedMachine m2)
+ {
+ int o1 = m1.getShareOrder();
+ int o2 = m2.getShareOrder();
+ if ( o1 == o2 ) {
+ return m1.getId().compareTo(m2.getId());
+ }
+ return o2 - o1;
+ }
+ }
+
+
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java?rev=1596375&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedMachine.java Tue May 20 19:26:52 2014
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.admin.event;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RmQueriedMachine
+ implements Serializable
+{
+ private static final long serialVersionUID = -8101741014979144426L;
+
+ String name;
+ String nodepoolId;
+ long memory;
+ int order;
+ List<RmQueriedShare> shares = null;
+
+ public RmQueriedMachine(String name, String nodepoolId, long memory, int order)
+ {
+ this.name = name;
+ this.nodepoolId = nodepoolId;
+ this.memory = memory;
+ this.order = order;
+ }
+
+ public void addShare(RmQueriedShare rqs)
+ {
+ if ( shares == null ) shares = new ArrayList<RmQueriedShare>();
+ shares.add(rqs);
+ }
+
+ public String getId() { return name; }
+ public long getMemory() { return memory; }
+ public int getShareOrder() { return order; }
+
+ static String fmt_s = "%12s %10s %10s %5s %4s %s";
+ String fmt_d = "%12s %10s %10d %5d %4d" ;
+ public static String header()
+ {
+ return String.format(fmt_s, "Node", "Nodepool", "Memory", "Order", "Free", "Shares\n");
+ }
+
+ public String toConsole()
+ {
+ StringBuffer sb = new StringBuffer();
+
+ if ( shares == null ) {
+ sb.append(String.format(fmt_d, name, nodepoolId, memory, order, 0));
+ sb.append(" [none]");
+ } else {
+ int used = 0;
+ for ( RmQueriedShare s : shares ) {
+ used += s.getShareOrder();
+ }
+ sb.append(String.format(fmt_d, name, nodepoolId, memory, order, order - used));
+
+ String spacer = " ";
+ String altSpacer = "\n" + String.format(fmt_s, " ", " ", " ", " ", " ", " "); // yes, blank, of exactly the right size
+ for ( RmQueriedShare s : shares ) {
+ sb.append(spacer);
+ sb.append(s.toConsole());
+ spacer = altSpacer;
+ }
+ }
+ return sb.toString();
+
+ }
+
+ public String toCompact()
+ {
+ // name memory order(nqshares) unused-shares share-details...
+ StringBuffer sb = new StringBuffer();
+ sb.append(name);
+ sb.append(" ");
+ sb.append(nodepoolId);
+ sb.append(" ");
+ sb.append(Long.toString(memory));
+ sb.append(" ");
+ sb.append(Integer.toString(order));
+
+ if ( shares == null ) {
+ sb.append(Integer.toString(order));
+ sb.append(", None");
+ } else {
+ int used = 0;
+ for ( RmQueriedShare s : shares ) {
+ used += s.getShareOrder();
+ }
+
+ sb.append(Integer.toString(order - used));
+ for ( RmQueriedShare s : shares ) {
+ sb.append(",");
+ sb.append(s.toCompact());
+ }
+ }
+ return sb.toString();
+ }
+
+ public String toString()
+ {
+ return String.format("%10s %10d %3d %3d", name, memory, order, shares == null ? 0 : shares.size());
+ }
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java?rev=1596375&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/admin/event/RmQueriedShare.java Tue May 20 19:26:52 2014
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.admin.event;
+
+import java.io.Serializable;
+
+public class RmQueriedShare
+ implements Serializable
+{
+ private static final long serialVersionUID = -8101741014979144426L;
+
+ long jobId; // DuccID of job
+ long shareId; // DuccID of share
+ long investmentInit;
+ long investmentRt;
+ int order;
+
+ boolean evicted = false;
+ boolean purged = false;
+ boolean fixed = false;
+ boolean initialized = false;
+ public RmQueriedShare(long job_id, long share_id, int order, long investment_init, long investment_rt)
+ {
+ this.jobId = job_id;
+ this.shareId = share_id;
+ this.order = order;
+ this.investmentInit = investment_init;
+ this.investmentRt = investment_rt;
+ }
+
+ public void setEvicted(boolean v){ this.evicted = v; }
+ public void setPurged(boolean v){ this.purged = v; }
+ public void setFixed(boolean v) { this.fixed = v; }
+ public void setInitialized(boolean v) {this.initialized = v; }
+
+ public long getJobId() { return this.jobId; }
+ public long getId() { return this.shareId; }
+ public long getInvestmentInit() { return this.investmentInit; }
+ public long getInvestmentRt() { return this.investmentRt; }
+ public int getShareOrder() { return order; }
+
+ public boolean isEvicted() { return evicted; }
+ public boolean isPurged() { return purged; }
+ public boolean isFixed() { return fixed; }
+ public boolean isInitialized() { return initialized; }
+
+
+ public String toCompact()
+ {
+ return String.format("%d %d %d %d %d %s %s %s", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ }
+
+ public String toConsole()
+ {
+ return String.format("J[%8d] S[%8d] O[%d] II[%8d] IR[%8d] E[%5s] P[%5s] F[%5s] I[%5s]", jobId, shareId, order, investmentInit, investmentRt, evicted, purged, fixed, initialized);
+ }
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/main/DuccRmAdmin.java Tue May 20 19:26:52 2014
@@ -30,6 +30,8 @@ import org.apache.camel.impl.DefaultCame
import org.apache.camel.impl.DefaultClassResolver;
import org.apache.uima.ducc.common.admin.event.DuccAdminEvent;
import org.apache.uima.ducc.common.admin.event.RmAdminQLoad;
+import org.apache.uima.ducc.common.admin.event.RmAdminQOccupancy;
+import org.apache.uima.ducc.common.admin.event.RmAdminQOccupancyReply;
import org.apache.uima.ducc.common.admin.event.RmAdminReply;
import org.apache.uima.ducc.common.admin.event.RmAdminVaryOff;
import org.apache.uima.ducc.common.admin.event.RmAdminVaryOn;
@@ -210,13 +212,24 @@ public class DuccRmAdmin
*
* @throws Exception
*/
- public void qload()
+ public RmAdminReply qload()
throws Exception
{
RmAdminQLoad ql = new RmAdminQLoad();
- RmAdminReply reply = dispatchAndWaitForReply(ql);
- System.out.println(reply.toString());
+ return dispatchAndWaitForReply(ql);
+ }
+
+ /**
+ * Query occupancy.
+ *
+ * @throws Exception
+ */
+ public RmAdminQOccupancyReply qoccupancy()
+ throws Exception
+ {
+ RmAdminQOccupancy qo = new RmAdminQOccupancy();
+ return (RmAdminQOccupancyReply) dispatchAndWaitForReply(qo);
}
@@ -237,7 +250,23 @@ public class DuccRmAdmin
if ( args[0].equals("--qload")) {
if ( args.length != 1 ) usage("Query load takes no arguments.");
- qload();
+ RmAdminReply ret = qload();
+ System.out.println(ret.toString());
+ return;
+ }
+
+ if ( args[0].equals("--qoccupancy")) {
+ if ( args.length != 2 ) usage("Query occupancy: specify --console or --compact");
+ if ( !args[1].equals("--console") && !args[1].equals("--compact") ) {
+ usage("Invalid argument: " + args[1] + " - specify --console or --compact");
+ }
+ RmAdminQOccupancyReply ret = qoccupancy();
+ if ( args[1].equals("--console") ) {
+ System.out.println(ret.toConsole());
+ } else {
+ System.out.println(ret.toCompact());
+ }
+
return;
}
}
@@ -252,6 +281,7 @@ public class DuccRmAdmin
System.out.println(" --varyoff string-delimeted-nodes");
System.out.println(" --varyon string-delimeted-nodes");
System.out.println(" --qload");
+ System.out.println(" --qoccupancy");
System.exit(1);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Tue May 20 19:26:52 2014
@@ -241,7 +241,7 @@ public class JobManagerConverter
long investment = proc.getWiMillisInvestment();
logger.info(methodName, j.getId(), "Assigning share in state", state, "pid", pid, "for recovery", s.toString());
j.recoverShare(s);
- s.update(j.getId(), mem, investment, state, proc.getTimeWindowInit(), proc.getTimeWindowRun(), pid);
+ s.update(j.getId(), mem, investment, state, proc.getTimeWindowInit(), pid);
}
}
logger.info(methodName, j.getId(), "Scheduling for recovery.");
@@ -517,7 +517,7 @@ public class JobManagerConverter
String pid = p.getPID();
logger.info(methodName, jobid, "New process ", s.toString(), mem, state, pid);
- if ( ! s.update(jobid, mem, investment, state, p.getTimeWindowInit(), p.getTimeWindowRun(), pid) ) {
+ if ( ! s.update(jobid, mem, investment, state, p.getTimeWindowInit(), pid) ) {
// TODO: probably change to just a warning and cancel the job - for now I want an attention-getter
throw new SchedulingException(jobid, "Process assignemnt arrives for share " + s.toString() +
" but jobid " + jobid + " does not match share " + s.getJob().getId());
@@ -612,7 +612,7 @@ public class JobManagerConverter
logger.info(methodName, jobid, "Process", pl.getPID(), "marked complete because it is purged. State:", state);
}
- if ( ! s.update(jobid, mem, investment, state, pl.getTimeWindowInit(), pl.getTimeWindowRun(), pid) ) {
+ if ( ! s.update(jobid, mem, investment, state, pl.getTimeWindowInit(), pid) ) {
// TODO: probably change to just a warning and cancel the job - for now I want an attention-getter
throw new SchedulingException(jobid, "Process update arrives for share " + s.toString() +
" but jobid " + jobid + " does not match job in share " + s.getJob().getId());
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java Tue May 20 19:26:52 2014
@@ -27,6 +27,7 @@ import org.apache.camel.Processor;
import org.apache.camel.builder.RouteBuilder;
import org.apache.uima.ducc.common.admin.event.DuccAdminEvent;
import org.apache.uima.ducc.common.admin.event.RmAdminQLoad;
+import org.apache.uima.ducc.common.admin.event.RmAdminQOccupancy;
import org.apache.uima.ducc.common.admin.event.RmAdminReply;
import org.apache.uima.ducc.common.admin.event.RmAdminVaryOff;
import org.apache.uima.ducc.common.admin.event.RmAdminVaryOn;
@@ -147,6 +148,9 @@ public class ResourceManagerComponent
} else
if (body instanceof RmAdminQLoad) {
reply = scheduler.queryLoad();
+ } else
+ if (body instanceof RmAdminQOccupancy) {
+ reply = scheduler.queryOccupancy();
}
exchange.getIn().setBody(reply);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/ISchedulerMain.java Tue May 20 19:26:52 2014
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.admin.event.RmAdminQLoadReply;
+import org.apache.uima.ducc.common.admin.event.RmAdminQOccupancyReply;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -75,4 +76,5 @@ public interface ISchedulerMain
String varyoff(String[] nodes);
String varyon(String[] nodes);
RmAdminQLoadReply queryLoad();
+ RmAdminQOccupancyReply queryOccupancy();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Machine.java Tue May 20 19:26:52 2014
@@ -22,6 +22,8 @@ import java.util.HashMap;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
+import org.apache.uima.ducc.common.admin.event.RmQueriedMachine;
+import org.apache.uima.ducc.common.admin.event.RmQueriedShare;
@@ -246,6 +248,24 @@ public class Machine
return cnt;
}
+ RmQueriedMachine queryMachine()
+ {
+ RmQueriedMachine ret = new RmQueriedMachine(id, nodepool.getId(), memory, share_order);
+ for ( Share s : activeShares.values() ) {
+ RmQueriedShare rqs = new RmQueriedShare(s.getJob().getId().getFriendly(),
+ s.getId().getFriendly(),
+ s.getShareOrder(),
+ s.getInitializationTime(),
+ s.getInvestment());
+ rqs.setFixed(s.isFixed());
+ rqs.setPurged(s.isPurged());
+ rqs.setEvicted(s.isEvicted());
+ rqs.setInitialized(s.isInitialized());
+ ret.addShare(rqs);
+ }
+ return ret;
+ }
+
/**
* A machine's investment is the sum of it's share's investments.
*/
@@ -257,7 +277,7 @@ public class Machine
}
return answer;
}
-
+
public static String getDashes()
{
return String.format("%20s %5s %13s %13s %11s %s", "--------------------", "-----", "-------------", "-------------", "-----------", "------ ...");
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Scheduler.java Tue May 20 19:26:52 2014
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.rm.scheduler;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
@@ -30,6 +31,7 @@ import org.apache.uima.ducc.common.NodeC
import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.Pair;
import org.apache.uima.ducc.common.admin.event.RmAdminQLoadReply;
+import org.apache.uima.ducc.common.admin.event.RmAdminQOccupancyReply;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccProperties;
import org.apache.uima.ducc.common.utils.DuccPropertiesResolver;
@@ -1145,6 +1147,48 @@ public class Scheduler
return reply;
}
+ private void collectMachinesForQuery(NodePool np, RmAdminQOccupancyReply ret)
+ {
+ Collection<Machine> machs = np.getAllMachines().values();
+ for ( Machine m : machs ) {
+ ret.addMachine(m.queryMachine());
+ }
+ }
+
+ public synchronized RmAdminQOccupancyReply queryOccupancy()
+ {
+ RmAdminQOccupancyReply ret = new RmAdminQOccupancyReply();
+
+ //
+ // iterate top-level nodepools to get all their subpools
+ // iterate the subpools to get all their machines
+ // iterage the machines and request a query object
+ // add query object to ret
+ // return ret
+
+ // We want to be dependent on common project, not the other way around, so
+ // we keep the query objects in common and put knowledge of how to construc
+ // them into rm's Machine class.
+ //
+ // The alternative, passing RM's Machine to the query object creates a circular
+ // dependency with RM depending on common and common depending on RM.
+ //
+
+ //
+ // Not a cheap query, by the way.
+ //
+ for ( NodePool np : nodepools ) {
+ collectMachinesForQuery(np, ret);
+
+ Collection<NodePool> pools = np.getChildren().values();
+ for ( NodePool npp : pools ) {
+ collectMachinesForQuery(npp, ret);
+ }
+ }
+
+ return ret;
+ }
+
/**
* Callback from job manager, need shares for a new fair-share job.
*/
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java?rev=1596375&r1=1596374&r2=1596375&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/Share.java Tue May 20 19:26:52 2014
@@ -42,9 +42,6 @@ public class Share
private int share_order; // may not be same as machine's order
private ITimeWindow init_time = null; // how much time this process spends initializing
- @SuppressWarnings("unused")
- private ITimeWindow run_time = null; // how much time this process spends initializing
-
// private HashMap<Integer, Long> activeQuestions = new HashMap<Integer, Long>();
@@ -213,7 +210,7 @@ public class Share
// so we don't end up counting this job more than it deserves.
}
- public boolean update(DuccId jobid, long mem, long investment, ProcessState state, ITimeWindow init_time, ITimeWindow run_time, String pid)
+ public boolean update(DuccId jobid, long mem, long investment, ProcessState state, ITimeWindow init_time, String pid)
{
if ( ! jobid.equals(job.getId()) ) return false; // something has gone horribly wrong
@@ -222,7 +219,6 @@ public class Share
this.state = state;
this.pid = pid;
this.init_time = init_time;
- this.run_time = run_time;
return true;
}