You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/06/26 15:38:31 UTC

svn commit: r1496934 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd: IJobDriverAccess.java JobDriver.java client/CallbackState.java client/IWorkItemMonitor.java client/WorkItem.java client/WorkItemListener.java

Author: degenaro
Date: Wed Jun 26 13:38:30 2013
New Revision: 1496934

URL: http://svn.apache.org/r1496934
Log:
UIMA-2987 DUCC job driver (JD) should reap unprocessed work items

Added:
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java
Modified:
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java Wed Jun 26 13:38:30 2013
@@ -37,6 +37,7 @@ public interface IJobDriverAccess {
 	public void accountingWorkItemIsPreempt(DuccId processId);
 	public void accountingWorkItemIsRetry(DuccId processId);
 	public void accountingWorkItemIsError(DuccId processId);
+	public void accountingWorkItemIsLost(DuccId processId);
 	public void accountingWorkItemIsDone(DuccId processId, long time);
 	public void rectifyStatus();
 }

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java Wed Jun 26 13:38:30 2013
@@ -20,6 +20,7 @@ package org.apache.uima.ducc.jd;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -59,11 +60,11 @@ import org.apache.uima.ducc.transport.ev
 import org.apache.uima.ducc.transport.event.common.DuccPerWorkItemStatistics;
 import org.apache.uima.ducc.transport.event.common.DuccProcessMap;
 import org.apache.uima.ducc.transport.event.common.DuccUimaDeploymentDescriptor;
+import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
 import org.apache.uima.ducc.transport.event.common.IDuccCompletionType.JobCompletionType;
 import org.apache.uima.ducc.transport.event.common.IDuccProcess;
 import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
 import org.apache.uima.ducc.transport.event.common.IDuccState.JobState;
-import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
 import org.apache.uima.ducc.transport.event.common.IDuccUimaDeployableConfiguration;
 import org.apache.uima.ducc.transport.event.common.IDuccUimaDeploymentDescriptor;
 import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
@@ -220,6 +221,47 @@ public class JobDriver extends Thread im
 		return jdJmxUrl;
 	}
 	
+	private long reapTimeout = 1;
+	
+	private void missingCallbackReaper() {
+		String location = "missingCallbackReaper";
+		try {
+			if(casSource.isEmpty()) {
+				IDuccWorkJob job = getJob();
+				DriverStatusReport driverStatusReport = getDriverStatusReportLive();
+				long todo = driverStatusReport.getWorkItemsToDo();
+				long capacity = job.getWorkItemCapacity();
+				if(capacity > 0) {
+					if(todo > 0) {
+						if(capacity > todo) {
+							duccOut.debug(location, jobid, "capacity:"+capacity+" "+"todo:"+todo);
+							Enumeration<WorkItem> workItems = casWorkItemMap.elements();
+							while(workItems.hasMoreElements()) {
+								WorkItem workItem = workItems.nextElement();
+								int seqNo = workItem.getSeqNo();
+								String casId = workItem.getCasId();
+								if(workItem.getCallbackState().isPendingCallback()) {
+									long sTime = workItem.getTimeWindow().getStartLong();
+									long cTime = System.currentTimeMillis();
+									long mTime = 1000*60*reapTimeout;
+									long tdiff = cTime - sTime;
+									if(tdiff > mTime) {
+										duccOut.warn(location, null, "reaping (no callback) seqNo:"+seqNo+" "+"casId:"+casId+" "+"tdiff:"+tdiff);
+										workItem.lost();
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+		catch(Exception e) {
+			duccOut.error(location, jobid, e);
+			duccErr.error(location, jobid, e);
+		}
+	}
+	
 	private void process() throws JobDriverTerminateException {
 		String location = "process";
 		try {
@@ -290,6 +332,7 @@ public class JobDriver extends Thread im
 						run = false;
 						continue;
 					}
+					missingCallbackReaper();
 					try {
 						Thread.sleep(10000);
 					} 
@@ -861,9 +904,9 @@ public class JobDriver extends Thread im
 			IDuccWorkJob job = jobDriver.getJob();
 			DuccId jobDuccId = job.getDuccId();
 			ArrayList<WorkItem> removeList = new ArrayList<WorkItem>();
-			duccOut.debug(location, jobDuccId, "pending map size:"+casLocationPendingMap.size());
+			duccOut.trace(location, jobDuccId, "pending map size:"+casLocationPendingMap.size());
 			Iterator<Entry<String, NP>> iterator = casLocationPendingMap.entrySet().iterator();
-			duccOut.debug(location, jobDuccId, iterator.hasNext());
+			duccOut.trace(location, jobDuccId, iterator.hasNext());
 			while(iterator.hasNext()) {
 				Entry<String, NP> entry = iterator.next();
 				String casId = entry.getKey();
@@ -875,13 +918,13 @@ public class JobDriver extends Thread im
 				String nodeIP = entry.getValue().getNodeIP();
 				if(nodeIP == null) {
 					DuccId processDuccId = null;
-					duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId);
+					duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId);
 					continue;
 				}
 				String PID = entry.getValue().getPID();
 				if(PID == null) {
 					DuccId processDuccId = null;
-					duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP);
+					duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP);
 					continue;
 				}
 				DuccId processDuccId = null;
@@ -894,7 +937,7 @@ public class JobDriver extends Thread im
 					duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
 					continue;
 				}
-				duccOut.debug(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
+				duccOut.trace(location, jobDuccId, processDuccId, "seqNo:"+seqNo+" "+"casId:"+casId+" "+"node:"+nodeIP+" "+"PID:"+PID);
 			}
 			removeLocations(removeList);
 		}
@@ -1039,6 +1082,16 @@ public class JobDriver extends Thread im
 		}
 	}
 	
+	public void accountingWorkItemIsLost(DuccId processId) {
+		String location = "accountingWorkItemIsLost";
+		try {
+			getPwiMap().lost(processId);
+		}
+		catch(Exception e) {
+			duccOut.error(location, jobid, "accounting error?", e);
+		}
+	}
+	
 	public void accountingWorkItemIsDone(DuccId processId, long time) {
 		String location = "accountingWorkItemIsDone";
 		try {
@@ -1247,6 +1300,24 @@ public class JobDriver extends Thread im
 		}
 	}
 	
+	public void lost(WorkItem workItem) {
+		String location = "lost";
+		try {
+			duccOut.info(location, workItem.getJobId(), "seqNo:"+workItem.getSeqNo());
+			workItemInactive();
+			driverStatusReport.workItemPendingProcessAssignmentRemove(workItem.getCasId());
+			driverStatusReport.workItemOperatingEnd(workItem.getCasId());
+			workItemStateManager.lost(workItem.getSeqNo());
+			workItemLost(workItem);
+			remove(workItem);
+			casSource.recycle(workItem.getCAS());
+			accountingWorkItemIsLost(workItem.getProcessId());
+			queueCASes(1,queue,workItemFactory);
+		}
+		catch(Exception exception) {
+			duccOut.error(location, jobid, "processing error?", exception);
+		}
+	}
 	
 	public void exception(WorkItem workItem, Exception e) {
 		String location = "exception";
@@ -1323,6 +1394,13 @@ public class JobDriver extends Thread im
 		return;
 	}
 	
+	
+	private void workItemLost(WorkItem workItem) {
+		String location = "workItemLost";
+		driverStatusReport.countWorkItemsLost();
+		duccOut.error(location, workItem.getJobId(), "seqNo:"+workItem.getSeqNo());
+	}
+	
 	private void workItemError(WorkItem workItem, Exception e) {
 		workItemError(workItem, e, null);
 	}
@@ -1334,7 +1412,7 @@ public class JobDriver extends Thread im
 	*/
 	
 	private void workItemError(WorkItem workItem, Exception e, Directive directive) {
-		String location = " workItemError";
+		String location = "workItemError";
 		driverStatusReport.countWorkItemsProcessingError();
 		String nodeId = "?";
 		String pid = "?";

Added: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java?rev=1496934&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/CallbackState.java Wed Jun 26 13:38:30 2013
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.jd.client;
+
+public class CallbackState {
+	
+	public static enum State { PendingQueued, PendingAssigned, NotPending };
+	
+	public State state = State.NotPending;
+	
+	private void setState(State value) {
+		synchronized(state) {
+			state = value;
+		}
+	}
+	
+	public void statePendingQueued() {
+		setState(State.PendingQueued);
+	}
+	
+	public void statePendingAssigned() {
+		setState(State.PendingAssigned);
+	}
+	
+	public void stateNotPending() {
+		setState(State.NotPending);
+	}
+	
+	public State getState() {
+		return state;
+	}
+	
+	public boolean isPendingQueued() {
+		switch(state) {
+		case PendingQueued:
+			return true;
+		default:
+			return false;
+		}
+	}
+	
+	public boolean isPendingAssigned() {
+		switch(state) {
+		case PendingAssigned:
+			return true;
+		default:
+			return false;
+		}
+	}
+	
+	public boolean isPendingCallback() {
+		switch(state) {
+		case PendingQueued:
+		case PendingAssigned:
+			return true;
+		default:
+			return false;
+		}
+	}
+}

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/IWorkItemMonitor.java Wed Jun 26 13:38:30 2013
@@ -27,4 +27,5 @@ public interface IWorkItemMonitor {
 	public void dequeued(WorkItem workitem, String node, String pid);
 	public void ended(WorkItem workitem);
 	public void exception(WorkItem workitem, Exception e);
+	public void lost(WorkItem workItem);
 }

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItem.java Wed Jun 26 13:38:30 2013
@@ -19,6 +19,7 @@
 package org.apache.uima.ducc.jd.client;
 
 import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.uima.aae.client.UimaAsynchronousEngine;
 import org.apache.uima.aae.monitor.statistics.AnalysisEnginePerformanceMetrics;
@@ -56,6 +57,10 @@ public class WorkItem implements Runnabl
 	
 	private ArrayList<AnalysisEnginePerformanceMetrics> analysisEnginePerformanceMetricsList = new ArrayList<AnalysisEnginePerformanceMetrics>();
 	
+	private CallbackState callbackState = new CallbackState();
+	
+	public AtomicBoolean isLost = new AtomicBoolean(false);
+	
 	public WorkItem(UimaAsynchronousEngine client, CasTuple casTuple, DuccId duccId, IWorkItemMonitor workItemMonitor) {
 		init(client, casTuple, duccId, workItemMonitor);
 	}
@@ -115,6 +120,10 @@ public class WorkItem implements Runnabl
 		return analysisEnginePerformanceMetricsList;
 	}
 	
+	public CallbackState getCallbackState() {
+		return callbackState;
+	}
+	
 	public void run() {
 		String methodName = "run";
 		duccOut.debug(methodName, jobId, duccMsg.fetch("enter"));
@@ -122,10 +131,16 @@ public class WorkItem implements Runnabl
 			start();
 			CAS cas = this.casTuple.getCas();
 			duccOut.debug(methodName, jobId, duccMsg.fetchLabel("CAS.size")+cas.size());
+			callbackState.statePendingQueued();
+			duccOut.debug(methodName, null, "seqNo:"+getSeqNo()+" "+callbackState.getState());
 			client.sendAndReceiveCAS(cas, analysisEnginePerformanceMetricsList);
-			ended();
+			if(!isLost.get()) {
+				ended();
+			}
 		} catch(Exception e) {
-			exception(e);
+			if(!isLost.get()) {
+				exception(e);
+			}
 		}
 		duccOut.debug(methodName, jobId, duccMsg.fetch("exit"));
 	}
@@ -158,4 +173,15 @@ public class WorkItem implements Runnabl
 		}
 	}
 	
+	public void lost() {
+		String methodName = "lost";
+		try {
+			isLost.set(true);
+			duccOut.debug(methodName, getJobId(), getProcessId(), "seqNo:"+getSeqNo()+" "+"casId:"+getCAS().hashCode());
+			workItemMonitor.lost(this);
+		}
+		catch(Exception exception) {
+			duccOut.error(methodName, null, exception);
+		}
+	}
 }

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java?rev=1496934&r1=1496933&r2=1496934&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/client/WorkItemListener.java Wed Jun 26 13:38:30 2013
@@ -35,6 +35,10 @@ public class WorkItemListener extends Ui
 	
 	private IJobDriver jobDriver;
 	
+	// <for testing only!!!>
+	boolean injectError = false;
+	// </for testing only!!!>
+	
 	public WorkItemListener(IJobDriver jobDriver) {
 		super();
 		this.jobDriver = jobDriver;
@@ -57,6 +61,18 @@ public class WorkItemListener extends Ui
 		ThreadLocation threadLocation = null;
 		try {
 			casId = ""+status.getCAS().hashCode();
+			// <for testing only!!!>
+			if(injectError) {
+				WorkItem wi = jobDriver.getWorkItem(casId);
+				wi.getCallbackState().statePendingAssigned();
+				duccOut.warn(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
+				int seqNo = wi.getSeqNo();
+				if(seqNo <= 1) {
+					duccOut.warn(methodName, null, "callback #1 discarded seqNo:"+seqNo+" "+"casId:"+casId);
+					return;
+				}
+			}
+			// </for testing only!!!>
 			jobDriver.queued(jobDriver.getWorkItem(casId));
 			threadLocation = jobDriver.getCasDispatchMap().get(casId);
 			DuccId jobid = jobDriver.getJob().getDuccId();
@@ -87,6 +103,20 @@ public class WorkItemListener extends Ui
 		ThreadLocation threadLocation = null;
 		try {
 			casId = ""+status.getCAS().hashCode();
+			WorkItem wi = jobDriver.getWorkItem(casId);
+			// <for testing only!!!>
+			if(injectError) {
+				wi.getCallbackState().statePendingAssigned();
+				duccOut.warn(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
+				int seqNo = wi.getSeqNo();
+				if(seqNo <= 3) {
+					duccOut.warn(methodName, null, "callback #2 discarded seqNo:"+seqNo+" "+"casId:"+casId);
+					return;
+				}
+			}
+			// </for testing only!!!>
+			wi.getCallbackState().stateNotPending();
+			duccOut.debug(methodName, null, "seqNo:"+wi.getSeqNo()+" "+wi.getCallbackState().getState());
 			String PID = pid.split(":")[0];
 			jobDriver.dequeued(jobDriver.getWorkItem(casId), nodeIP, PID);
 			threadLocation = jobDriver.getCasDispatchMap().get(casId);