You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2015/09/30 20:35:40 UTC
svn commit: r1706120 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator:
Validate.java factory/JobFactory.java
Author: degenaro
Date: Wed Sep 30 18:35:40 2015
New Revision: 1706120
URL: http://svn.apache.org/viewvc?rev=1706120&view=rev
Log:
UIMA-4612 DUCC Orchestrator (OR) externalize max threads per job & process limits to ducc.properties
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/Validate.java
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/Validate.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/Validate.java?rev=1706120&r1=1706119&r2=1706120&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/Validate.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/Validate.java Wed Sep 30 18:35:40 2015
@@ -80,7 +80,7 @@ public class Validate {
return retVal;
}
- public static boolean integer(boolean retVal, Properties properties, String key, String defaultValue, String minValue, String maxValue) {
+ public static boolean integer(boolean retVal, Properties properties, String key, String defaultValue, String minValue) {
String value = (String)properties.get(key);
if(value == null) {
String reason = createReason("default",key,defaultValue);
@@ -90,17 +90,11 @@ public class Validate {
try {
int specified_value = Integer.parseInt(value);
int min_value = Integer.parseInt(minValue);
- int max_value = Integer.parseInt(maxValue);
if(specified_value < min_value) {
String reason = createReason("invalid, under "+minValue,key,value);
addError(properties,reason);
retVal = false;
}
- if(specified_value > max_value) {
- String reason = createReason("invalid, above "+maxValue,key,value);
- addError(properties,reason);
- retVal = false;
- }
}
catch(Exception e) {
String reason = createReason("invalid, non-integer",key,value);
@@ -123,8 +117,7 @@ public class Validate {
properties,
JobSpecificationProperties.key_process_thread_count,
IDuccSchedulingInfo.defaultThreadsPerProcess,
- IDuccSchedulingInfo.minThreadsPerProcess,
- IDuccSchedulingInfo.maxThreadsPerProcess);
+ IDuccSchedulingInfo.minThreadsPerProcess);
// scheduling class
key = JobRequestProperties.key_scheduling_class;
value = (String) properties.get(key);
@@ -222,8 +215,7 @@ public class Validate {
properties,
JobSpecificationProperties.key_process_thread_count,
IDuccSchedulingInfo.defaultThreadsPerProcess,
- IDuccSchedulingInfo.minThreadsPerProcess,
- IDuccSchedulingInfo.maxThreadsPerProcess);
+ IDuccSchedulingInfo.minThreadsPerProcess);
// scheduling class
key = ServiceRequestProperties.key_scheduling_class;
value = (String) properties.get(key);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1706120&r1=1706119&r2=1706120&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Wed Sep 30 18:35:40 2015
@@ -424,25 +424,64 @@ public class JobFactory implements IJobF
job.setDriver(driver);
}
+ private void check_max_job_pipelines(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
+ String methodName = "check_max_job_pipelines";
+ long ducc_limit = 0;
+ String p_limit;
+ p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_job_max_pipelines_count);
+ if(p_limit == null) {
+ p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_threads_limit);
+ }
+ if (p_limit != null) {
+ if (!p_limit.equals("unlimited")) {
+ try {
+ ducc_limit = Long.parseLong(p_limit);
+ }
+ catch(Exception e) {
+ logger.error(methodName, job.getDuccId(), e);
+ }
+ }
+ }
+ if (ducc_limit <= 0) {
+ return;
+ }
+ long user_limit = schedulingInfo.getLongProcessesMax();
+ logger.trace(methodName, job.getDuccId(), "user_limit"+"="+user_limit+" "+"ducc_limit"+"="+ducc_limit);
+ if(user_limit > ducc_limit) {
+ logger.info(methodName, job.getDuccId(), "change max job pipelines from "+user_limit+" to " + ducc_limit);
+ schedulingInfo.setLongProcessesMax(ducc_limit);
+ }
+ }
+
+ private void check_max_process_pipelines(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
+ String methodName = "check_max_process_pipelines";
+ int ducc_limit = 0;
+ String p_limit;
+ p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_process_max_pipelines_count);
+ if (p_limit != null) {
+ if (!p_limit.equals("unlimited")) {
+ try {
+ ducc_limit = Integer.parseInt(p_limit);
+ }
+ catch(Exception e) {
+ logger.error(methodName, job.getDuccId(), e);
+ }
+ }
+ }
+ if (ducc_limit <= 0) {
+ return;
+ }
+ int user_limit = schedulingInfo.getIntThreadsPerProcess();
+ logger.trace(methodName, job.getDuccId(), "user_limit"+"="+user_limit+" "+"ducc_limit"+"="+ducc_limit);
+ if(user_limit > ducc_limit) {
+ logger.info(methodName, job.getDuccId(), "change max process pipelines from "+user_limit+" to " + ducc_limit);
+ schedulingInfo.setIntThreadsPerProcess(ducc_limit);
+ }
+ }
+
private void checkSchedulingLimits(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
- String methodName = "checkSchedulingLimits";
- long limit = 0;
- String p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_threads_limit);
- if (p_limit != null) {
- if (!p_limit.equals("unlimited")) {
- limit = Long.parseLong(p_limit);
- }
- }
- if (limit <= 0) {
- return;
- }
- int threads_per_process = schedulingInfo.getIntThreadsPerProcess();
- long processesLimit = limit / threads_per_process;
- long maxProcesses = schedulingInfo.getLongProcessesMax();
- if (maxProcesses == 0 || maxProcesses > processesLimit) {
- logger.info(methodName, job.getDuccId(), "change max-processes from "+maxProcesses+" to " + processesLimit);
- schedulingInfo.setProcessesMax(String.valueOf(processesLimit));
- }
+ check_max_job_pipelines(job, schedulingInfo);
+ check_max_process_pipelines(job, schedulingInfo);
}
public DuccWorkJob createJob(CommonConfiguration common, JobRequestProperties jobRequestProperties) throws ResourceUnavailableForJobDriverException {