You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2014/02/23 17:25:56 UTC
svn commit: r1571023 - in /uima/sandbox/uima-ducc/trunk: src/main/resources/
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/
Author: challngr
Date: Sun Feb 23 16:25:56 2014
New Revision: 1571023
URL: http://svn.apache.org/r1571023
Log:
UIMA-3405 CLI support for failure management.
Modified:
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties Sun Feb 23 16:25:56 2014
@@ -246,10 +246,14 @@ ducc.sm.state.update.endpoint=ducc.sm.st
ducc.sm.state.update.endpoint.type=topic
# Default pinger
ducc.sm.default.monitor.class = org.apache.uima.ducc.cli.UimaAsPing
-# Max consecutive init failures or run failures within the failure window
+# max consecutive init failures or ping failures for ping-only services before we stop starting things
ducc.sm.instance.failure.max 5
-# Failure window size in MINUTES.
+# backward compatibility
+ducc.sm.instance.failure.limit ${ducc.sm.instance.failure.max}
+# failure window size in MINUTES.
ducc.sm.instance.failure.window 30
+# max consecutive instance init failures before we stop trying to start things
+ducc.sm.init.failure.limit 1
# How often services are monitored to insure they're responding, in milliseconds
ducc.sm.meta.ping.rate = 60000
# How many consecutive missed pings are required to mark a service unresponive
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceApi.java Sun Feb 23 16:25:56 2014
@@ -66,12 +66,12 @@ public class DuccServiceApi
UiOption.ProcessDD,
UiOption.ProcessExecutable,
UiOption.ProcessExecutableArgs,
- UiOption.InstanceFailuresLimit,
UiOption.ClasspathOrder,
// UiOption.Specification // not used for registration
UiOption.ServiceDependency,
UiOption.ServiceRequestEndpoint,
UiOption.ServiceLinger,
+
UiOption.ServicePingArguments,
UiOption.ServicePingClass,
UiOption.ServicePingClasspath,
@@ -79,6 +79,10 @@ public class DuccServiceApi
UiOption.ServicePingTimeout,
UiOption.ServicePingDoLog,
+ UiOption.InstanceFailureWindow,
+ UiOption.InstanceFailureLimit,
+ UiOption.InstanceInitFailureLimit,
+
UiOption.Register,
UiOption.Autostart,
UiOption.Instances,
@@ -117,12 +121,17 @@ public class DuccServiceApi
UiOption.Instances,
UiOption.Autostart,
UiOption.Activate,
+
UiOption.ServicePingArguments,
UiOption.ServicePingClass,
UiOption.ServicePingClasspath,
UiOption.ServicePingJvmArgs,
UiOption.ServicePingTimeout,
UiOption.ServicePingDoLog,
+
+ UiOption.InstanceFailureWindow,
+ UiOption.InstanceFailureLimit,
+ UiOption.InstanceInitFailureLimit,
};
UiOption[] query_options = {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccServiceSubmit.java Sun Feb 23 16:25:56 2014
@@ -57,18 +57,23 @@ public class DuccServiceSubmit
UiOption.ProcessDD,
UiOption.ProcessExecutable,
UiOption.ProcessExecutableArgs,
- UiOption.InstanceFailuresLimit,
+ UiOption.InstanceFailureLimit,
UiOption.ClasspathOrder,
UiOption.Specification,
UiOption.ServiceDependency,
UiOption.ServiceRequestEndpoint,
UiOption.ServiceLinger,
+
UiOption.ServicePingArguments,
UiOption.ServicePingClass,
UiOption.ServicePingClasspath,
UiOption.ServicePingJvmArgs,
UiOption.ServicePingTimeout,
UiOption.ServicePingDoLog,
+
+ UiOption.InstanceFailureWindow,
+ UiOption.InstanceFailureLimit,
+ UiOption.InstanceInitFailureLimit,
};
/**
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/IUiOptions.java Sun Feb 23 16:25:56 2014
@@ -194,14 +194,30 @@ public interface IUiOptions
public String label() { return name(); }
},
- InstanceFailuresLimit {
+ InstanceFailureLimit {
public String pname() { return "instance_failures_limit"; }
public String description() { return "Number of consecutive instance failures that will cause the service to be stopped."; }
public String argname() { return "integer"; }
public String example() { return null; }
- public String deflt() { return "2"; }
- public String label() { return "InstanceFailuresLimit"; }
+ public String label() { return name(); }
+ },
+
+ InstanceInitFailureLimit {
+ public String pname() { return "instance_init_failures_limit"; }
+ public String description() { return "Number of consecutive instance initialization failures that will cause SM to cease starting the service."; }
+ public String argname() { return "integer"; }
+ public String example() { return null; }
+ public String label() { return name(); }
},
+
+ InstanceFailureWindow {
+ public String pname() { return "instance_failures_window"; }
+ public String argname() { return "integer"; }
+ public String description() { return "Size of the window (in minutes) used to manage service instance runtime failures."; }
+ public String example() { return "30"; }
+ public String label() { return name(); }
+ },
+
JobId {
public String pname() { return JobRequestProperties.key_id; }
public String argname() { return "string"; }
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/PingDriver.java Sun Feb 23 16:25:56 2014
@@ -153,8 +153,8 @@ class PingDriver
this.working_directory = resolveStringProperty ("working_directory" , ping_props, job_props, null); // cli always puts this int job props, no default
this.log_directory = resolveStringProperty ("log_directory" , ping_props, job_props, null); // cli always puts this int job props, no default
- this.failure_window = resolveIntProperty ("service_failure_window", ping_props, job_props, ServiceManagerComponent.failure_window);
- this.failure_max = resolveIntProperty ("service_failure_max" , ping_props, job_props, ServiceManagerComponent.failure_max);
+ this.failure_window = resolveIntProperty ("instance_failures_window", ping_props, job_props, ServiceManagerComponent.failure_window);
+ this.failure_max = resolveIntProperty ("instance_failures_limit" , ping_props, job_props, ServiceManagerComponent.failure_max);
jvm_args_str = jvm_args_str.trim();
if ( jvm_args_str.equals("") ) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceManagerComponent.java Sun Feb 23 16:25:56 2014
@@ -97,8 +97,9 @@ public class ServiceManagerComponent
static int meta_ping_timeout = 500; // timeout on ping
static String default_ping_class;
- static int failure_max = 5;
- static int failure_window = 30;
+ static int init_failure_max = 1; // total
+ static int failure_max = 5; // total in window
+ static int failure_window = 30; // window size in minutes
private String state_dir = null;
private String state_file = null;
@@ -112,7 +113,7 @@ public class ServiceManagerComponent
private boolean testmode = false;
Map<String, String> administrators = new HashMap<String, String>();
- String version = "1.1.1";
+ String version = "1.1.2";
public ServiceManagerComponent(CamelContext context)
{
@@ -253,16 +254,18 @@ public class ServiceManagerComponent
super.start(service, args);
DuccDaemonRuntimeProperties.getInstance().boot(DaemonName.ServiceManager,getProcessJmxUrl());
- failure_max = SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.max", failure_max);
- failure_window = SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.window", failure_window);
- meta_ping_rate = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.rate", meta_ping_rate);
- meta_ping_timeout = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.timeout", meta_ping_timeout);
- meta_ping_stability = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.stability", meta_ping_stability);
- default_ping_class = SystemPropertyResolver.getStringProperty("ducc.sm.default.monitor.class", UimaAsPing.class.getName());
+ init_failure_max = SystemPropertyResolver.getIntProperty("ducc.sm.init.failure.limit" , init_failure_max);
+ failure_max = SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.limit" , failure_max);
+ failure_window = SystemPropertyResolver.getIntProperty("ducc.sm.instance.failure.window" , failure_window);
+
+ meta_ping_rate = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.rate" , meta_ping_rate);
+ meta_ping_timeout = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.timeout" , meta_ping_timeout);
+ meta_ping_stability = SystemPropertyResolver.getIntProperty("ducc.sm.meta.ping.stability" , meta_ping_stability);
+ default_ping_class = SystemPropertyResolver.getStringProperty("ducc.sm.default.monitor.class", UimaAsPing.class.getName());
+
String rm = SystemPropertyResolver.getStringProperty("ducc.runmode", "");
if ( rm.equals("Test") ) testmode = true;
- // yuck
String sig = SystemPropertyResolver.getStringProperty("ducc.signature.required", "on");
signature_required = true;
if ( sig.equals("on") ) signature_required = true;
@@ -275,18 +278,27 @@ public class ServiceManagerComponent
logger.info(methodName, null, "Service Manager starting:");
logger.info(methodName, null, " DUCC home : ", System.getProperty("DUCC_HOME"));
logger.info(methodName, null, " ActiveMQ URL : ", System.getProperty("ducc.broker.url"));
+ logger.info(methodName, null, "");
logger.info(methodName, null, " JVM : ", System.getProperty("java.vendor") +
" "+ System.getProperty("java.version"));
logger.info(methodName, null, " JAVA_HOME : ", System.getProperty("java.home"));
logger.info(methodName, null, " JVM Path : ", System.getProperty("ducc.jvm"));
logger.info(methodName, null, " JMX URL : ", System.getProperty("ducc.jmx.url"));
+ logger.info(methodName, null, "");
logger.info(methodName, null, " OS Architecture : ", System.getProperty("os.arch"));
logger.info(methodName, null, " Crypto enabled : ", signature_required);
+ logger.info(methodName, null, "");
logger.info(methodName, null, " Test mode enabled : ", testmode);
+ logger.info(methodName, null, "");
logger.info(methodName, null, " Service ping rate : ", meta_ping_rate);
logger.info(methodName, null, " Service ping timeout : ", meta_ping_timeout);
logger.info(methodName, null, " Service ping stability : ", meta_ping_stability);
+ logger.info(methodName, null, " Default ping class : ", default_ping_class);
+ logger.info(methodName, null, "");
+ logger.info(methodName, null, " Init Failure Max : ", init_failure_max);
logger.info(methodName, null, " Instance Failure Max : ", failure_max);
+ logger.info(methodName, null, " Instance Failure Window : ", failure_window);
+ logger.info(methodName, null, "");
logger.info(methodName, null, " DUCC Version : ", Version.version());
logger.info(methodName, null, " SM Version : ", version);
logger.info(methodName, null, "------------------------------------------------------------------------------------");
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java?rev=1571023&r1=1571022&r2=1571023&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-sm/src/main/java/org/apache/uima/ducc/sm/ServiceSet.java Sun Feb 23 16:25:56 2014
@@ -133,10 +133,10 @@ public class ServiceSet
LingerTask linger = null;
long linger_time = 60000;
- int init_failures_max = ServiceManagerComponent.failure_max;
+ int init_failure_max = ServiceManagerComponent.init_failure_max;
int init_failures = 0; // max allowed consecutive failures, current failure count
- int ping_failures_max = ServiceManagerComponent.failure_max;
+ int ping_failure_max = ServiceManagerComponent.failure_max;
int ping_failures = 0; // for ping-only services, if the external pinger throws errors we
// need to govern it
@@ -157,7 +157,6 @@ public class ServiceSet
this.service_state = ServiceState.Stopped;
this.linger_time = props.getLongProperty(RegistrationOption.ServiceLinger.decode(), linger_time);
this.key = meta.getProperty("endpoint");
- this.init_failures_max = props.getIntProperty(UiOption.InstanceFailuresLimit.pname(), ServiceManagerComponent.failure_max);
parseEndpoint(key);
@@ -167,6 +166,7 @@ public class ServiceSet
this.ping_only = meta.getBooleanProperty("ping-only", false);
this.stopped = meta.getBooleanProperty("stopped", stopped);
this.service_class = ServiceClass.Registered;
+ this.init_failure_max = props.getIntProperty("instance_init_failures_limit", init_failure_max);
parseIndependentServices();
@@ -429,7 +429,7 @@ public class ServiceSet
String methodName = "enforceAutostart";
if ( ! autostart ) return; // not doing auto, nothing to do
if ( stopped ) return; // doing auto, but we've been manually stopped
- if ( init_failures >= init_failures_max ) return; // too many init failures, no more enforcement
+ if ( init_failures >= init_failure_max ) return; // too many init failures, no more enforcement
// if ( (isPingOnly()) && (serviceMeta == null) ) { // ping-only and monitor / pinger not alive
// logger.info(methodName, id, "Autostarting 1 ping-only instance.");
@@ -985,7 +985,7 @@ public class ServiceSet
} else {
logger.warn(methodName, id, "Instance", inst_id,
"Excessive initialization failures. Total failures[" + init_failures + "]",
- "allowed [" + init_failures_max + "], not restarting.");
+ "allowed [" + init_failure_max + "], not restarting.");
}
setAutostart(false);
} else {
@@ -1365,7 +1365,7 @@ public class ServiceSet
synchronized boolean excessiveFailures()
{
String methodName = "excessiveFailures";
- if ( init_failures >= init_failures_max ) {
+ if ( init_failures >= init_failure_max ) {
logger.trace(methodName, id, "INIT FAILURES EXCEEDED");
return true;
}
@@ -1406,8 +1406,8 @@ public class ServiceSet
} // otherwise, it was already removed by some intrepid unit
if ( isPingOnly() && (rc != 0) ) {
- if ( ++ping_failures > ping_failures_max ) {
- logger.info(methodName, id, "Stopping pinger due to excessive falutes:", ping_failures_max);
+ if ( ++ping_failures > ping_failure_max ) {
+ logger.info(methodName, id, "Stopping pinger due to excessive falutes:", ping_failure_max);
// stop();
} else {
logger.info(methodName, id, "Ping-only pinger exits with error rc", rc, "total errors:", ping_failures);