You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2019/11/01 14:03:25 UTC
svn commit: r1869261 - in /uima/uima-ducc/trunk:
uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/
uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/
Author: degenaro
Date: Fri Nov 1 14:03:25 2019
New Revision: 1869261
URL: http://svn.apache.org/viewvc?rev=1869261&view=rev
Log:
UIMA-6142 DUCC Orchestrator (OR) not killing Job when first JP fails to initialize
Modified:
uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
Modified: uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java?rev=1869261&r1=1869260&r2=1869261&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-orchestrator/src/main/java/org/apache/uima/ducc/orchestrator/StateManager.java Fri Nov 1 14:03:25 2019
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.orchestrator;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
@@ -1538,6 +1539,26 @@ public class StateManager {
logger.trace(methodName, null, messages.fetch("exit"));
}
+ private DuccId getFirstFailedInitialization(DuccWorkJob job) {
+ String methodName = "getFirstFailedInitialization";
+ DuccId retVal = null;
+ try {
+ IDuccProcessMap processMap = job.getProcessMap();
+ ArrayList<DuccId> list = processMap.getFailedInitialization();
+ for(DuccId duccId : list) {
+ IDuccProcess process = processMap.getProcess(duccId);
+ if(process.getSeqNo() == 0) {
+ retVal = duccId;
+ break;
+ }
+ }
+ }
+ catch(Throwable t) {
+ logger.error(methodName, null, t);
+ }
+ return retVal;
+ }
+
private void inventoryJob(IDuccWork duccWork, IDuccProcess inventoryProcess) {
String methodName = "inventoryJob";
DuccWorkJob job = (DuccWorkJob) duccWork;
@@ -1600,7 +1621,16 @@ public class StateManager {
}
long initFailureCount = job.getProcessInitFailureCount();
long startup_initialization_error_limit = DuccPropertiesResolver.get(DuccPropertiesResolver.ducc_jd_startup_initialization_error_limit, 1);
- if(initFailureCount >= startup_initialization_error_limit) {
+ DuccId pid = getFirstFailedInitialization(job);
+ if(pid != null) {
+ String reason = "process inititialization failure on first job process";
+ logger.warn(methodName, job.getDuccId(), pid, reason);
+ JobCompletionType jobCompletionType = JobCompletionType.CanceledBySystem;
+ Rationale rationale = new Rationale(reason);
+ ProcessDeallocationType processDeallocationType = ProcessDeallocationType.JobCanceled;
+ stateManager.jobTerminate(job, jobCompletionType, rationale, processDeallocationType);
+ }
+ else if(initFailureCount >= startup_initialization_error_limit) {
String reason = "process inititialization failure count["+initFailureCount+"] meets startup initialization error limit["+startup_initialization_error_limit+"]";
logger.warn(methodName, job.getDuccId(), reason);
JobCompletionType jobCompletionType = JobCompletionType.CanceledBySystem;
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java?rev=1869261&r1=1869260&r2=1869261&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcess.java Fri Nov 1 14:03:25 2019
@@ -20,6 +20,7 @@ package org.apache.uima.ducc.transport.e
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.uima.ducc.common.Node;
import org.apache.uima.ducc.common.NodeIdentity;
@@ -52,6 +53,7 @@ public class DuccProcess implements IDuc
*/
private static final long serialVersionUID = 1L;
private long dataVersion=1;
+ private long seqNo = 0; // each JP for a job gets a seqNo; first one is 0 and is special with respect to Initialization Failure
private DuccId duccId = null;
private Node node = null;
private NodeIdentity nodeIdentity = null;
@@ -81,12 +83,16 @@ public class DuccProcess implements IDuc
private long wiMillisInvestment;
private long currentCPU;
+ private AtomicLong sequencer = new AtomicLong(0);
+
public DuccProcess(DuccId duccId, NodeIdentity nodeIdentity) {
+ seqNo = sequencer.getAndIncrement();
setDuccId(duccId);
setNodeIdentity(nodeIdentity);
}
public DuccProcess(DuccId duccId, Node node) {
+ seqNo = sequencer.getAndIncrement();
setDuccId(duccId);
setNode(node);
NodeIdentity nodeIdentity = node.getNodeIdentity();
@@ -94,12 +100,14 @@ public class DuccProcess implements IDuc
}
public DuccProcess(DuccId duccId, NodeIdentity nodeIdentity, ProcessType processType) {
+ seqNo = sequencer.getAndIncrement();
setDuccId(duccId);
setNodeIdentity(nodeIdentity);
setProcessType(processType);
}
public DuccProcess(DuccId duccId, Node node, ProcessType processType) {
+ seqNo = sequencer.getAndIncrement();
setDuccId(duccId);
setNode(node);
NodeIdentity nodeIdentity = node.getNodeIdentity();
@@ -107,6 +115,10 @@ public class DuccProcess implements IDuc
setProcessType(processType);
}
+ public long getSeqNo() {
+ return seqNo;
+ }
+
public long getDataVersion() {
long retVal = 0;
try {
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java?rev=1869261&r1=1869260&r2=1869261&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccProcess.java Fri Nov 1 14:03:25 2019
@@ -36,6 +36,8 @@ public interface IDuccProcess extends Se
public long getDataVersion();
+ public long getSeqNo();
+
public DuccId getDuccId();
public void setDuccId(DuccId duccId);