You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2016/04/29 17:21:12 UTC
svn commit: r1741647 - in /uima/sandbox/uima-ducc/trunk: src/main/admin/
src/main/resources/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/
uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/
uima-ducc-orchestrator/src/main/java/or...
Author: burn
Date: Fri Apr 29 15:21:12 2016
New Revision: 1741647
URL: http://svn.apache.org/viewvc?rev=1741647&view=rev
Log:
UIMA-4861 Rename old property when merging the site properties. Drop ducc limit for pipelines-per-process. Make pipelines-per-job limit the max-processes
Modified:
uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager
uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager?rev=1741647&r1=1741646&r2=1741647&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/admin/ducc_props_manager Fri Apr 29 15:21:12 2016
@@ -50,6 +50,16 @@ class DuccPropManager:
file3_props = Properties()
+ # Check for a site specification override of the property renamed in 2.1.0
+ oname = 'ducc.threads.limit'
+ nname = 'ducc.job.max.pipelines.count'
+ prop = file2_props.get_property(oname)
+ if prop != None:
+ prop.c.append('# RENAMED by merger ' + time.strftime('%c') + ' old name: ' + prop.k)
+ prop.k = nname
+ file2_props.put_property(prop)
+ file2_props.delete(oname)
+
# first pass, create merged props with stuff in base props file updated with delta
for k in file1_props.get_keys():
base = file1_props.get_property(k)
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties?rev=1741647&r1=1741646&r2=1741647&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/default.ducc.properties Fri Apr 29 15:21:12 2016
@@ -163,9 +163,6 @@ ducc.ws.jvm.args = -Xmx2G -Dja
# Note: formerly known as ducc.threads.limit
ducc.job.max.pipelines.count = 5000
-# Max number of work-item CASes for each process (default is "unlimited")
-ducc.process.max.pipelines.count = 100
-
# ========== CLI Configuration block ==========
# These environment values are included on job/service/AP submissions
ducc.environment.propagated = USER HOME LANG DUCC_SERVICE_INSTANCE
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java?rev=1741647&r1=1741646&r2=1741647&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/DuccPropertiesResolver.java Fri Apr 29 15:21:12 2016
@@ -90,7 +90,6 @@ public class DuccPropertiesResolver {
//<UIMA-4612>
public static final String ducc_job_max_pipelines_count = "ducc.job.max.pipelines.count";
- public static final String ducc_process_max_pipelines_count = "ducc.process.max.pipelines.count";
//</UIMA-4612>
public static final String ducc_driver_jvm_args = "ducc.driver.jvm.args";
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex?rev=1741647&r1=1741646&r2=1741647&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-duccdocs/src/site/tex/duccbook/part4/admin/ducc-properties.tex Fri Apr 29 15:21:12 2016
@@ -491,15 +491,6 @@
\item[Type] Local
\end{description}
- \item[ducc.process.max.pipelines.count] \hfill \\
- This enforces a maximum number of pipelines per process.
-
- \begin{description}
- \item[Default] 100
- \item[Type] Local
- \end{description}
-
-
\item[ducc.environment.propagated] \hfill \\
This specifies the environmental variables whose values will be merged with the
user-specified environment option on job, process and service submissions.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1741647&r1=1741646&r2=1741647&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Fri Apr 29 15:21:12 2016
@@ -406,66 +406,40 @@ public class JobFactory implements IJobF
job.setDriver(driver);
}
- private void check_max_job_pipelines(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
+ private void checkSchedulingLimits(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
String methodName = "check_max_job_pipelines";
long ducc_limit = 0;
String p_limit;
- p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_job_max_pipelines_count);
+ // Check the old name first in case it is in site.ducc.properties ... new name is in ducc.default.properties
+ p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_threads_limit);
if(p_limit == null) {
- p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_threads_limit);
+ p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_job_max_pipelines_count);
}
- if (p_limit != null) {
- if (!p_limit.equals("unlimited")) {
- try {
- ducc_limit = Long.parseLong(p_limit);
- }
- catch(Exception e) {
- logger.error(methodName, job.getDuccId(), e);
- }
- }
+ if (p_limit != null && !p_limit.equals("unlimited")) {
+ try {
+ ducc_limit = Long.parseLong(p_limit);
+ }
+ catch(Exception e) {
+ logger.error(methodName, job.getDuccId(), e);
+ }
}
if (ducc_limit <= 0) {
return;
}
- long user_limit = schedulingInfo.getLongProcessesMax();
- logger.trace(methodName, job.getDuccId(), "user_limit"+"="+user_limit+" "+"ducc_limit"+"="+ducc_limit);
- if(user_limit > ducc_limit) {
- logger.info(methodName, job.getDuccId(), "change max job pipelines from "+user_limit+" to " + ducc_limit);
- schedulingInfo.setLongProcessesMax(ducc_limit);
- }
- }
-
- private void check_max_process_pipelines(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
- String methodName = "check_max_process_pipelines";
- int ducc_limit = 0;
- String p_limit;
- p_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_process_max_pipelines_count);
- if (p_limit != null) {
- if (!p_limit.equals("unlimited")) {
- try {
- ducc_limit = Integer.parseInt(p_limit);
- }
- catch(Exception e) {
- logger.error(methodName, job.getDuccId(), e);
- }
- }
- }
- if (ducc_limit <= 0) {
- return;
+ // Don't round up as that could exceed the ducc limit ... also restrict pipelines-per-process if too large !!
+ int pipelines_per_process = schedulingInfo.getIntThreadsPerProcess();
+ if (pipelines_per_process > ducc_limit) {
+ schedulingInfo.setIntThreadsPerProcess((int) ducc_limit);
}
- int user_limit = schedulingInfo.getIntThreadsPerProcess();
- logger.trace(methodName, job.getDuccId(), "user_limit"+"="+user_limit+" "+"ducc_limit"+"="+ducc_limit);
- if(user_limit > ducc_limit) {
- logger.info(methodName, job.getDuccId(), "change max process pipelines from "+user_limit+" to " + ducc_limit);
- schedulingInfo.setIntThreadsPerProcess(ducc_limit);
+ long processes_limit = ducc_limit / schedulingInfo.getIntThreadsPerProcess();
+ long user_limit = schedulingInfo.getLongProcessesMax();
+ logger.trace(methodName, job.getDuccId(), "user_limit"+"="+user_limit+" "+"ducc_processes_limit"+"="+processes_limit);
+ if(user_limit > processes_limit) {
+ logger.info(methodName, job.getDuccId(), "change max job processes from "+user_limit+" to "+ducc_limit+"/"+schedulingInfo.getIntThreadsPerProcess());
+ schedulingInfo.setLongProcessesMax(processes_limit);
}
}
-
- private void checkSchedulingLimits(DuccWorkJob job, DuccSchedulingInfo schedulingInfo) {
- check_max_job_pipelines(job, schedulingInfo);
- check_max_process_pipelines(job, schedulingInfo);
- }
-
+
public DuccWorkJob createJob(CommonConfiguration common, JobRequestProperties jobRequestProperties) throws ResourceUnavailableForJobDriverException {
DuccWorkJob job = new DuccWorkJob();
job.setDuccType(DuccType.Job);