You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by cw...@apache.org on 2018/07/30 14:43:05 UTC
svn commit: r1837063 - in /uima/uima-ducc/trunk:
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/...
Author: cwiklik
Date: Mon Jul 30 14:43:05 2018
New Revision: 1837063
URL: http://svn.apache.org/viewvc?rev=1837063&view=rev
Log:
UIMA-5843 added error window support
Modified:
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
Modified: uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccJobSubmit.java Mon Jul 30 14:43:05 2018
@@ -80,6 +80,8 @@ public class DuccJobSubmit
UiOption.ProcessInitializationFailuresCap,
UiOption.ProcessFailuresLimit,
UiOption.ProcessPipelineCount,
+ UiOption.ProcessErrorWindowThreshold,
+ UiOption.ProcessErrorWindowSize,
UiOption.ProcessPerItemTimeMax,
UiOption.ProcessInitializationTimeMax,
@@ -163,6 +165,7 @@ public class DuccJobSubmit
{
init (this.getClass().getName(), opts, args, jobRequestProperties, consoleCb);
check_descriptor_options();
+ check_error_window_definition();
if (isAllInOne()) {
allInOneLauncher = new AllInOneLauncher(userSpecifiedProperties, consoleCb); // Pass the already fixed-up user properties
}
@@ -182,12 +185,43 @@ public class DuccJobSubmit
throws Exception
{
init (this.getClass().getName(), opts, props, jobRequestProperties, consoleCb);
+
check_descriptor_options();
+ check_error_window_definition();
if (isAllInOne()) {
allInOneLauncher = new AllInOneLauncher(userSpecifiedProperties, consoleCb); // Pass the already fixed-up user properties
}
}
-
+ private void check_error_window_definition() throws IllegalArgumentException {
+ int windowSize=0;
+ int maxThreshold=0;
+
+ boolean errorWindowSizeDefined = jobRequestProperties.containsKey(UiOption.ProcessErrorWindowSize.pname());
+ boolean errorWindowThresholdDefined = jobRequestProperties.containsKey(UiOption.ProcessErrorWindowThreshold.pname());
+
+ if ( errorWindowSizeDefined ) {
+ try {
+ windowSize = jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname());
+ } catch( NumberFormatException e) {
+ throw new IllegalArgumentException("process_error_window_size must be a postive number - you've provided:"+
+ jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname()));
+ }
+ }
+ if ( errorWindowThresholdDefined ) {
+ try {
+ maxThreshold = jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname());
+ } catch( NumberFormatException e) {
+ throw new IllegalArgumentException("process_error_window_threshold must be a postive number - you've provided:"+
+ jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname()));
+ }
+ }
+ if ( maxThreshold > windowSize ) {
+ throw new IllegalArgumentException("process_error_window_threshold must be smaller/equal than/to process_error_window_size - you've provided:"+
+ " process_error_window_threshold="+jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowThreshold.pname()) +
+ " process_error_window_size="+jobRequestProperties.getIntProperty(UiOption.ProcessErrorWindowSize.pname()));
+
+ }
+ }
/*
* If preemptable change to a non-preemptable scheduling class.
* If none provided use the default fixed class
Modified: uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java Mon Jul 30 14:43:05 2018
@@ -517,7 +517,20 @@ public interface IUiOptions
public String description() { return "Number of pipelines per process (i.e. simultaneously processed work-items)."; }
public String argname() { return "integer"; }
public String deflt() { return "4"; }
- },
+ },
+ ProcessErrorWindowThreshold {
+ public String pname() { return JobSpecificationProperties.key_process_error_threshold; }
+ public String description() { return "Maximum number of process errors per process."; }
+ public String argname() { return "integer"; }
+ public String deflt() { return "1"; }
+ },
+
+ ProcessErrorWindowSize {
+ public String pname() { return JobSpecificationProperties.key_process_error_window; }
+ public String description() { return "Size of error window used in conjuntion with maximum number of process errors."; }
+ public String argname() { return "integer"; }
+ public String deflt() { return "1"; }
+ },
ProcessPerItemTimeMax {
public String pname() { return JobSpecificationProperties.key_process_per_item_time_max; }
Modified: uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/container/FlagsHelper.java Mon Jul 30 14:43:05 2018
@@ -48,6 +48,8 @@ public class FlagsHelper {
JpCmOverrides,
JpDd,
JpThreadCount,
+ JpErrorThreshold,
+ JpErrorWindowSize,
JpType, // { uima, uima-as }
JpProcessorClass("processor.class"),
UserClasspath, // path1.class:path2.jar:path3/*:...
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/factory/JobFactory.java Mon Jul 30 14:43:05 2018
@@ -641,6 +641,15 @@ public class JobFactory implements IJobF
// </flags for JP to build Aggregate>
}
+ String process_error_threshold = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_error_threshold);
+ if(process_error_threshold != null) {
+ addDashD(pipelineCommandLine, FlagsHelper.Name.JpErrorThreshold, process_error_threshold);
+ }
+ String process_error_window = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_error_window);
+ if(process_error_window != null) {
+ addDashD(pipelineCommandLine, FlagsHelper.Name.JpErrorWindowSize, process_error_window);
+ }
+
String process_thread_count = jobRequestProperties.getProperty(JobSpecificationProperties.key_process_pipeline_count);
if(process_thread_count != null) {
addDashD(pipelineCommandLine, FlagsHelper.Name.JpThreadCount, process_thread_count);
Modified: uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/ServiceConfiguration.java Mon Jul 30 14:43:05 2018
@@ -20,6 +20,7 @@
package org.apache.uima.ducc.ps.service;
import org.apache.uima.ducc.ps.service.errors.ServiceInitializationException;
+import org.apache.uima.ducc.ps.service.jmx.JMXAgent;
public class ServiceConfiguration {
private String clientURL;
@@ -47,7 +48,14 @@ public class ServiceConfiguration {
private String maxErrors;
private String errorWindowSize;
private ClassLoader sysCL=null;
-
+ private String processType;
+
+ public String getProcessType() {
+ return processType;
+ }
+ public void setProcessType(String processType) {
+ this.processType = processType;
+ }
public String getMaxErrors() {
return maxErrors;
}
@@ -240,7 +248,7 @@ public class ServiceConfiguration {
serviceJmxConnectURL = System.getProperty("ducc.jmx.port");
customRegistryClass = System.getProperty("ducc.deploy.registry.class");
customProcessorClass = System.getProperty("ducc.deploy.custom.processor.class");
-
+ processType = System.getProperty("ducc.deploy.components");//=job-process
duccHome = System.getenv("DUCC_HOME");
jobId = System.getenv("DUCC_JOBID");
duccProcessId = System.getenv("DUCC_PROCESSID");
Modified: uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/AbstractServiceProcessor.java Mon Jul 30 14:43:05 2018
@@ -37,8 +37,10 @@ public class AbstractServiceProcessor {
protected int maxErrors=1; // default is to fail on 1st error
protected int windowSize=1;
protected int DEFAULT_INIT_DELAY=30000;
-
- protected IServiceErrorHandler getErrorHandler() {
+
+
+ protected IServiceErrorHandler getErrorHandler(Logger logger) {
+ logger.log(Level.INFO, "Error Handler Configuration: threshold="+maxErrors+" windowSize="+windowSize);
// concrete implementation of this abstract class should
// provide a way to set maxErrors and windowSize. The IServiceProcessor
// provides a setter method for overriding default values
@@ -66,4 +68,7 @@ public class AbstractServiceProcessor {
return serializerMap.get(Thread.currentThread().getId());
}
+ protected void stop() throws Exception {
+
+ }
}
Modified: uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaAsServiceProcessor.java Mon Jul 30 14:43:05 2018
@@ -107,6 +107,7 @@ public class UimaAsServiceProcessor exte
public UimaAsServiceProcessor(String[] args, ServiceConfiguration serviceConfiguration) {
this.args = args;
this.serviceConfiguration = serviceConfiguration;
+
// start a thread which will collect AE initialization state
launchStateInitializationCollector();
}
@@ -162,7 +163,7 @@ public class UimaAsServiceProcessor exte
}
// Needed to resolve ${queue.name} placeholder in DD generated by DUCC
System.setProperty(queuePropertyName, endpointName);
- errorHandler = getErrorHandler();
+ errorHandler = getErrorHandler(logger);
// generate Spring context file once
synchronized (UimaAsServiceProcessor.class) {
@@ -462,6 +463,7 @@ public class UimaAsServiceProcessor exte
Method waitMethod = classToLaunch.getMethod("waitUntilStopped");
waitMethod.invoke(brokerInstance);
brokerRunning = false;
+ super.stop();
} catch (Exception e) {
logger.log(Level.WARNING, "stop", e);
Modified: uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaProcessResult.java Mon Jul 30 14:43:05 2018
@@ -55,5 +55,9 @@ public class UimaProcessResult implement
exception.printStackTrace(new PrintWriter(sw));
return sw.toString();
}
+ @Override
+ public Exception getExceptionObject() {
+ return exception;
+ }
}
Modified: uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-pullservice/src/main/java/org/apache/uima/ducc/ps/service/processor/uima/UimaServiceProcessor.java Mon Jul 30 14:43:05 2018
@@ -143,7 +143,7 @@ public class UimaServiceProcessor extend
logger.log(Level.FINE, "Process Thread:"+ Thread.currentThread().getName()+" Initializing AE");
}
- errorHandler = getErrorHandler();
+ errorHandler = getErrorHandler(logger);
try {
// multiple threads may call this method. Send initializing state once
@@ -282,6 +282,8 @@ public class UimaServiceProcessor extend
if ( ae != null ) {
ae.destroy();
}
+ super.stop();
+
} catch( Exception e) {
logger.log(Level.WARNING, "stop", e);
}
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java?rev=1837063&r1=1837062&r2=1837063&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/cli/JobSpecificationProperties.java Mon Jul 30 14:43:05 2018
@@ -65,6 +65,8 @@ public class JobSpecificationProperties
public static String key_process_initialization_failures_cap = "process_initialization_failures_cap";
public static String key_process_failures_limit = "process_failures_limit";
public static String key_process_pipeline_count = "process_pipeline_count";
+ public static String key_process_error_threshold = "process_error_window_threshold";
+ public static String key_process_error_window = "process_error_window_size";
/*
public static String key_process_get_meta_time_max = "process_get_meta_time_max";