You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2013/10/29 15:45:34 UTC
svn commit: r1536740 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm:
JobManagerConverter.java ResourceManagerComponent.java
config/ResourceManagerConfiguration.java scheduler/NodepoolScheduler.java
Author: challngr
Date: Tue Oct 29 14:45:34 2013
New Revision: 1536740
URL: http://svn.apache.org/r1536740
Log:
UIMA-3398 Tie nodes found in recovery into scheduler using NodeStability, not using Scheduler directly.
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/config/ResourceManagerConfiguration.java
uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java?rev=1536740&r1=1536739&r2=1536740&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/JobManagerConverter.java Tue Oct 29 14:45:34 2013
@@ -73,7 +73,8 @@ public class JobManagerConverter
{
DuccLogger logger = DuccLogger.getLogger(JobManagerConverter.class, COMPONENT_NAME);
ISchedulerMain scheduler;
-
+ NodeStability nodeStability = null;
+
DuccWorkMap localMap = null;
JobManagerUpdate lastJobManagerUpdate = new JobManagerUpdate();
@@ -81,10 +82,12 @@ public class JobManagerConverter
boolean recovery = false;
- public JobManagerConverter(ISchedulerMain scheduler)
+ public JobManagerConverter(ISchedulerMain scheduler, NodeStability ns)
{
this.scheduler = scheduler;
this.localMap = new DuccWorkMap();
+ this.nodeStability = ns;
+
DuccLogger.setUnthreaded();
recovery = SystemPropertyResolver.getBooleanProperty("ducc.rm.fast.recovery", true);
@@ -1024,7 +1027,7 @@ public class JobManagerConverter
}
logger.info(methodName, null, "Recovered[", nodes.size(), "] nodes from OR state.");
for (Node n : nodes.values() ) {
- scheduler.nodeArrives(n);
+ nodeStability.nodeArrives(n);
}
return (nodes.size() != 0);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java?rev=1536740&r1=1536739&r2=1536740&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/ResourceManagerComponent.java Tue Oct 29 14:45:34 2013
@@ -59,6 +59,8 @@ public class ResourceManagerComponent
DuccEventDispatcher eventDispatcher;
String stateEndpoint;
+ NodeStability stabilityManager = null;
+
public ResourceManagerComponent(CamelContext context) {
super("ResourceManager", context);
this.scheduler = new Scheduler();
@@ -74,13 +76,18 @@ public class ResourceManagerComponent
return schedulerReady;
}
+ public void setNodeStability(NodeStability ns)
+ {
+ this.stabilityManager = ns;
+ }
+
public void start(DuccService service, String[] args)
throws Exception
{
super.start(service, args);
DuccDaemonRuntimeProperties.getInstance().boot(DaemonName.ResourceManager,getProcessJmxUrl());
- converter = new JobManagerConverter(scheduler);
+ converter = new JobManagerConverter(scheduler, stabilityManager);
initStability = SystemPropertyResolver.getIntProperty("ducc.rm.init.stability", DEFAULT_INIT_STABILITY_COUNT);
nodeStability = SystemPropertyResolver.getIntProperty("ducc.rm.node.stability", DEFAULT_STABILITY_COUNT);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/config/ResourceManagerConfiguration.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/config/ResourceManagerConfiguration.java?rev=1536740&r1=1536739&r2=1536740&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/config/ResourceManagerConfiguration.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/config/ResourceManagerConfiguration.java Tue Oct 29 14:45:34 2013
@@ -52,6 +52,7 @@ import org.springframework.context.annot
int nodeStability = SystemPropertyResolver.getIntProperty("ducc.rm.node.stability", DEFAULT_STABILITY_COUNT);
int agentMetricsRate = SystemPropertyResolver.getIntProperty("ducc.agent.node.metrics.publish.rate", DEFAULT_NODE_METRICS_RATE);
NodeStability ns = new NodeStability(rm, nodeStability, agentMetricsRate);
+ rm.setNodeStability(ns);
jmel.setEndpoint(common.rmStateUpdateEndpoint);
jmel.setNodeStability(ns);
ns.start();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java?rev=1536740&r1=1536739&r2=1536740&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-rm/src/main/java/org/apache/uima/ducc/rm/scheduler/NodepoolScheduler.java Tue Oct 29 14:45:34 2013
@@ -1120,8 +1120,12 @@ public class NodepoolScheduler
// Don't schedule non-preemptable shares over subpools
if ( np.countLocalShares() < n_instances ) {
- schedulingUpdate.refuse(j, "Job refused because insufficient resources are availble.");
- logger.warn(methodName, j.getId(), "Cannot accept Fixed Share job nodepool " + np.getId()
+ schedulingUpdate.refuse(j, "1 Job refused because insufficient resources are availble. Available for class "
+ + rc.getName() + ": "
+ + np.countLocalShares()
+ + "requested:" + n_instances);
+
+ logger.warn(methodName, j.getId(), "1 Cannot accept Fixed Share job nodepool " + np.getId()
+ " has insufficient nodes left. Available["
+ np.countLocalShares()
+ "] requested[" + n_instances + "]");
@@ -1132,8 +1136,8 @@ public class NodepoolScheduler
// Now see if we have sufficient shares in the system for this allocation. Note that pool nodes are accounted for here as well.
//
if ( np.countNSharesByOrder(order) < n_instances ) { // countSharesByOrder is N shares, as is minshares
- schedulingUpdate.refuse(j, "Job refused because insufficient resources are availble.");
- logger.warn(methodName, j.getId(), "Cannot accept Fixed Share job, insufficient shares available. Available[" + np.countNSharesByOrder(order) + "] requested[" + n_instances + "]");
+ schedulingUpdate.refuse(j, "2 Job refused because insufficient resources are availble.");
+ logger.warn(methodName, j.getId(), "2 Cannot accept Fixed Share job, insufficient shares available. Available[" + np.countNSharesByOrder(order) + "] requested[" + n_instances + "]");
continue;
}
@@ -1142,12 +1146,12 @@ public class NodepoolScheduler
//
shares_given_out += (n_instances * order);
if ( shares_given_out > classcap ) { // to q-shares before comparing
- schedulingUpdate.refuse(j, "Job refused because class cap of " + classcap + " is exceeded.");
+ schedulingUpdate.refuse(j, "3 Job refused because class cap of " + classcap + " is exceeded.");
continue;
}
if ( rc.getMaxProcesses() < n_instances ) { // Does it blow the configured limit for this class?
- schedulingUpdate.refuse(j, "Job refused because class max of " + rc.getMaxProcesses() + " is exceeded.");
+ schedulingUpdate.refuse(j, "3 Job refused because class max of " + rc.getMaxProcesses() + " is exceeded.");
continue;
}