You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/06/26 15:38:31 UTC
svn commit: r1496934 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd:
IJobDriverAccess.java JobDriver.java client/CallbackState.java
client/IWorkItemMonitor.java client/WorkItem.java
client/WorkItemListener.java
Author: degenaro
Date: Wed Jun 26 13:38:30 2013
New Revision: 1496934
URL: http://svn.apache.org/r1496934
Log:
UIMA-2987 DUCC job driver (JD) should reap unprocessed work items
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java Wed Jun 26 13:38:30 2013
@@ -37,6 +37,7 @@ public interface IJobDriverAccess {
public void accountingWorkItemIsPreempt(DuccId processId);
public void accountingWorkItemIsRetry(DuccId processId);
public void accountingWorkItemIsError(DuccId processId);
+ public void accountingWorkItemIsLost(DuccId processId);
public void accountingWorkItemIsDone(DuccId processId, long time);
public void rectifyStatus();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java Wed Jun 26 13:38:30 2013
@@ -20,6 +20,7 @@ package org.apache.uima.ducc.jd;
import java.io.File;
import java.util.ArrayList;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -59,11 +60,11 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.DuccPerWorkItemStatistics;
import org.apache.uima.ducc.transport.event.common.DuccProcessMap;
import org.apache.uima.ducc.transport.event.common.DuccUimaDeploymentDescriptor;
+import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
import org.apache.uima.ducc.transport.event.common.IDuccCompletionType.JobCompletionType;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
import org.apache.uima.ducc.transport.event.common.IDuccState.JobState;
-import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
import org.apache.uima.ducc.transport.event.common.IDuccUimaDeployableConfiguration;
import org.apache.uima.ducc.transport.event.common.IDuccUimaDeploymentDescriptor;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
@@ -220,6 +221,47 @@ public class JobDriver extends Thread im
return jdJmxUrl;
}
+ private long reapTimeout = 1;
+
+ private void missingCallbackReaper() {
+ String location = "missingCallbackReaper";
+ try {
+ if(casSource.isEmpty()) {
+ IDuccWorkJob job = getJob();
+ DriverStatusReport driverStatusReport = getDriverStatusReportLive();
+ long todo = driverStatusReport.getWorkItemsToDo();
+ long capacity = job.getWorkItemCapacity();
+ if(capacity > 0) {
+ if(todo > 0) {
+ if(capacity > todo) {
+ duccOut.debug(location, jobid, "capacity:"+capacity+" "+"todo:"+todo);
+ Enumeration<WorkItem> workItems = casWorkItemMap.elements();
+ while(workItems.hasMoreElements()) {
+ WorkItem workItem = workItems.nextElement();
+ int seqNo = workItem.getSeqNo();
+ String casId = workItem.getCasId();
+ if(workItem.getCallbackState().isPendingCallback()) {
+ long sTime = workItem.getTimeWindow().getStartLong();
+ long cTime = System.currentTimeMillis();
+ long mTime = 1000*60*reapTimeout;
+ long tdiff = cTime - sTime;
+ if(tdiff > mTime) {
+ duccOut.warn(location, null, "reaping (no callback) seqNo:"+seqNo+" "+"casId:"+casId+" "+"tdiff:"+tdiff);
+ workItem.lost();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ catch(Exception e) {
+ duccOut.error(location, jobid, e);
+ duccErr.error(location, jobid, e);
+ }
+ }
+
private void process() throws JobDriverTerminateException {
String location = "process";
try {
@@ -290,6 +332,7 @@ public class JobDriver extends Thread im
run = false;
continue;
}
+ missingCallbackReaper();
try {
Thread.sleep(10000);
}
@@ -861,9 +904,9 @@ public class JobDriver extends Thread im
IDuccWorkJob job = jobDriver.getJob();
DuccId jobDuccId = job.getDuccId();
ArrayList<WorkItem> removeList = new ArrayList<WorkItem>();
- duccOut.debug(location, jobDuccId, "pending map size:"+casLocationPendingMap.size());
+ duccOut.trace(location, jobDuccId, "pending map size:"+casLocationPendingMap.size());
Iterator<Entry<String, NP>> iterator = casLocationPendingMap.entrySet().iterator();
- duccOut.debug(location, jobDuccId, iterator.hasNext());
+ duccOut.trace(location, jobDuccId, iterator.hasNext());
while(iterator.hasNext()) {
Entry<String, NP> entry = iterator.next();
String casId = entry.getKey();
@@ -875,13 +918,13 @@ public class JobDriver extends Thread im
String nodeIP = entry.getValue().getNodeIP();
if(nodeIP == null) {
DuccId processDuccId = null;
- duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId);
+ duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId);
continue;
}
String PID = entry.getValue().getPID();
if(PID == null) {
DuccId processDuccId = null;
- duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP);
+ duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP);
continue;
}
DuccId processDuccId = null;
@@ -894,7 +937,7 @@ public class JobDriver extends Thread im
duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
continue;
}
- duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
+ duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
}
removeLocations(removeList);
}
@@ -1039,6 +1082,16 @@ public class JobDriver extends Thread im
}
}
+ public void accountingWorkItemIsLost(DuccId processId) {
+ String location = "accountingWorkItemIsLost";
+ try {
+ getPwiMap().lost(processId);
+ }
+ catch(Exception e) {
+ duccOut.error(location, jobid, "accounting error?", e);
+ }
+ }
+
public void accountingWorkItemIsDone(DuccId processId, long time) {
String location = "accountingWorkItemIsDone";
try {
@@ -1247,6 +1300,24 @@ public class JobDriver extends Thread im
}
}
+ public void lost(WorkItem workItem) {
+ String location = "lost";
+ try {
+ duccOut.info(location, workItem.getJobId(), "seqNo:"+workItem.getSeqNo());
+ workItemInactive();
+ driverStatusReport.workItemPendingProcessAssignmentRemove(workItem.getCasId());
+ driverStatusReport.workItemOperatingEnd(workItem.getCasId());
+ workItemStateManager.lost(workItem.getSeqNo());
+ workItemLost(workItem);
+ remove(workItem);
+ casSource.recycle(workItem.getCAS());
+ accountingWorkItemIsLost(workItem.getProcessId());
+ queueCASes(1,queue,workItemFactory);
+ }
+ catch(Exception exception) {
+ duccOut.error(location, jobid, "processing error?", exception);
+ }
+ }
public void exception(WorkItem workItem, Exception e) {
String location = "exception";
@@ -1323,6 +1394,13 @@ public class JobDriver extends Thread im
return;
}
+
+ private void workItemLost(WorkItem workItem) {
+ String location = "workItemLost";
+ driverStatusReport.countWorkItemsLost();
+ duccOut.error(location, workItem.getJobId(), "seqNo:"+workItem.getSeqNo());
+ }
+
private void workItemError(WorkItem workItem, Exception e) {
workItemError(workItem, e, null);
}
@@ -1334,7 +1412,7 @@ public class JobDriver extends Thread im
*/
private void workItemError(WorkItem workItem, Exception e, Directive directive) {
- String location = " workItemError";
+ String location = "workItemError";
driverStatusReport.countWorkItemsProcessingError();
String nodeId = "?";
String pid = "?";
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java?rev=1496934&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java Wed Jun 26 13:38:30 2013
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.jd.client;
+
+public class CallbackState {
+
+ public static enum State { PendingQueued, PendingAssigned, NotPending };
+
+ public State state = State.NotPending;
+
+ private void setState(State value) {
+ synchronized(state) {
+ state = value;
+ }
+ }
+
+ public void statePendingQueued() {
+ setState(State.PendingQueued);
+ }
+
+ public void statePendingAssigned() {
+ setState(State.PendingAssigned);
+ }
+
+ public void stateNotPending() {
+ setState(State.NotPending);
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ public boolean isPendingQueued() {
+ switch(state) {
+ case PendingQueued:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public boolean isPendingAssigned() {
+ switch(state) {
+ case PendingAssigned:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ public boolean isPendingCallback() {
+ switch(state) {
+ case PendingQueued:
+ case PendingAssigned:
+ return true;
+ default:
+ return false;
+ }
+ }
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java Wed Jun 26 13:38:30 2013
@@ -27,4 +27,5 @@ public interface IWorkItemMonitor {
public void dequeued(WorkItem workitem, String node, String pid);
public void ended(WorkItem workitem);
public void exception(WorkItem workitem, Exception e);
+ public void lost(WorkItem workItem);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java Wed Jun 26 13:38:30 2013
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.jd.client;
import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.uima.aae.client.UimaAsynchronousEngine;
import org.apache.uima.aae.monitor.statistics.AnalysisEnginePerformanceMetrics;
@@ -56,6 +57,10 @@ public class WorkItem implements Runnabl
private ArrayList<AnalysisEnginePerformanceMetrics> analysisEnginePerformanceMetricsList = new ArrayList<AnalysisEnginePerformanceMetrics>();
+ private CallbackState callbackState = new CallbackState();
+
+ public AtomicBoolean isLost = new AtomicBoolean(false);
+
public WorkItem(UimaAsynchronousEngine client, CasTuple casTuple, DuccId duccId, IWorkItemMonitor workItemMonitor) {
init(client, casTuple, duccId, workItemMonitor);
}
@@ -115,6 +120,10 @@ public class WorkItem implements Runnabl
return analysisEnginePerformanceMetricsList;
}
+ public CallbackState getCallbackState() {
+ return callbackState;
+ }
+
public void run() {
String methodName = "run";
duccOut.debug(methodName, jobId, duccMsg.fetch("enter"));
@@ -122,10 +131,16 @@ public class WorkItem implements Runnabl
start();
CAS cas = this.casTuple.getCas();
duccOut.debug(methodName, jobId, duccMsg.fetchLabel("CAS.size")+cas.size());
+ callbackState.statePendingQueued();
+ duccOut.debug(methodName, null, "seqNo:"+getSeqNo()+" "+callbackState.getState());
client.sendAndReceiveCAS(cas, analysisEnginePerformanceMetricsList);
- ended();
+ if(!isLost.get()) {
+ ended();
+ }
} catch(Exception e) {
- exception(e);
+ if(!isLost.get()) {
+ exception(e);
+ }
}
duccOut.debug(methodName, jobId, duccMsg.fetch("exit"));
}
@@ -158,4 +173,15 @@ public class WorkItem implements Runnabl
}
}
+ public void lost() {
+ String methodName = "lost";
+ try {
+ isLost.set(true);
+ duccOut.debug(methodName, getJobId(), getProcessId(), "seqNo:"+getSeqNo()+" "+"casId:"+getCAS().hashCode());
+ workItemMonitor.lost(this);
+ }
+ catch(Exception exception) {
+ duccOut.error(methodName, null, exception);
+ }
+ }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java Wed Jun 26 13:38:30 2013
@@ -35,6 +35,10 @@ public class WorkItemListener extends Ui
private IJobDriver jobDriver;
+ // <for testing only!!!>
+ boolean injectError = false;
+ // </for testing only!!!>
+
public WorkItemListener(IJobDriver jobDriver) {
super();
this.jobDriver = jobDriver;
@@ -57,6 +61,18 @@ public class WorkItemListener extends Ui
ThreadLocation threadLocation = null;
try {
casId = ""+status.getCAS().hashCode();
+ // <for testing only!!!>
+ if(injectError) {
+ WorkItem wi = jobDriver.getWorkItem(casId);
+ wi.getCallbackState().statePendingAssigned();
+ duccOut.warn(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
+ int seqNo = wi.getSeqNo();
+ if(seqNo <= 1) {
+ duccOut.warn(methodName, null, "callback #1 discarded seqNo:"+seqNo+" "+"casId:"+casId);
+ return;
+ }
+ }
+ // </for testing only!!!>
jobDriver.queued(jobDriver.getWorkItem(casId));
threadLocation = jobDriver.getCasDispatchMap().get(casId);
DuccId jobid = jobDriver.getJob().getDuccId();
@@ -87,6 +103,20 @@ public class WorkItemListener extends Ui
ThreadLocation threadLocation = null;
try {
casId = ""+status.getCAS().hashCode();
+ WorkItem wi = jobDriver.getWorkItem(casId);
+ // <for testing only!!!>
+ if(injectError) {
+ wi.getCallbackState().statePendingAssigned();
+ duccOut.warn(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
+ int seqNo = wi.getSeqNo();
+ if(seqNo <= 3) {
+ duccOut.warn(methodName, null, "callback #2 discarded seqNo:"+seqNo+" "+"casId:"+casId);
+ return;
+ }
+ }
+ // </for testing only!!!>
+ wi.getCallbackState().stateNotPending();
+ duccOut.debug(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
String PID = pid.split(":")[0];
jobDriver.dequeued(jobDriver.getWorkItem(casId), nodeIP, PID);
threadLocation = jobDriver.getCasDispatchMap().get(casId);