You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2014/12/16 17:09:00 UTC

svn commit: r1645994 - in /uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport: configuration/jd/JobDriverComponent.java event/common/DuccProcessWorkItems.java event/jd/JobDriverReport.java

Author: degenaro
Date: Tue Dec 16 16:08:59 2014
New Revision: 1645994

URL: http://svn.apache.org/r1645994
Log:
UIMA-4069 DUCC Job Driver (JD) system classpath

Processes tab accounting (for JPs)

Modified:
    uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java Tue Dec 16 16:08:59 2014
@@ -41,6 +41,7 @@ import org.apache.uima.ducc.container.ne
 import org.apache.uima.ducc.transport.configuration.jd.iface.IJobDriverComponent;
 import org.apache.uima.ducc.transport.event.JdStateDuccEvent;
 import org.apache.uima.ducc.transport.event.OrchestratorAbbreviatedStateDuccEvent;
+import org.apache.uima.ducc.transport.event.common.DuccProcessMap;
 import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
 import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
 import org.apache.uima.ducc.transport.event.common.IDuccProcess;
@@ -122,6 +123,8 @@ implements IJobDriverComponent {
 	
 	private AtomicInteger getStateReqNo = new AtomicInteger(0);
 	
+	private IDuccProcessMap dpMap = null;
+	
 	@Override
 	public JdStateDuccEvent getState() {
 		String location = "getState";
@@ -129,7 +132,7 @@ implements IJobDriverComponent {
 		try {
 			IMessageHandler mh = JobDriver.getInstance().getMessageHandler();
 			IOperatingInfo oi = mh.handleGetOperatingInfo();
-			IDriverStatusReport driverStatusReport = new JobDriverReport(oi);
+			IDriverStatusReport driverStatusReport = new JobDriverReport(oi, dpMap);
 			driverStatusReport.setNode(node);
 			driverStatusReport.setPort(port);
 			state.setState(driverStatusReport);
@@ -166,6 +169,7 @@ implements IJobDriverComponent {
 				DuccWorkJob dwj = (DuccWorkJob) dwMap.findDuccWork(duccId);
 				if(dwj != null) {
 					IDuccProcessMap pMap = dwj.getProcessMap();
+					dpMap = pMap;
 					for(Entry<DuccId, IDuccProcess> entry : pMap.entrySet()) {
 						IDuccProcess p = entry.getValue();
 						ProcessState state = p.getProcessState();

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java Tue Dec 16 16:08:59 2014
@@ -20,6 +20,8 @@ package org.apache.uima.ducc.transport.e
 
 import java.util.concurrent.atomic.AtomicLong;
 
+import org.apache.uima.ducc.container.jd.mh.iface.IProcessInfo;
+
 public class DuccProcessWorkItems implements IDuccProcessWorkItems {
 
 	private static final long serialVersionUID = 1L;
@@ -33,9 +35,25 @@ public class DuccProcessWorkItems implem
 	private AtomicLong lost = new AtomicLong(0);
 	private AtomicLong preempt = new AtomicLong(0);
 	private AtomicLong completedMillisTotal = new AtomicLong(0);
+	private AtomicLong completedMillisAvg = new AtomicLong(0);
 	private AtomicLong completedMillisMax = new AtomicLong(0);
 	private AtomicLong completedMillisMin = new AtomicLong(0);
 
+	public DuccProcessWorkItems() {	
+	}
+	
+	public DuccProcessWorkItems(IProcessInfo pi) {	
+		dispatch.set(pi.getDispatch());
+		done.set(pi.getDone());
+		error.set(pi.getError());
+		retry.set(pi.getRetry());
+		lost.set(0);
+		preempt.set(pi.getPreempt());
+		completedMillisAvg.set(pi.getAvg());
+		completedMillisMax.set(pi.getMax());
+		completedMillisMin.set(pi.getMin());
+	}
+	
 	public long getCountUnassigned() {
 		return unassigned;
 	}
@@ -172,7 +190,7 @@ public class DuccProcessWorkItems implem
 	}
 	
 	
-	public long getSecsAvg() {
+	public long getSecsAvgV1() {
 		long retVal = 0;
 		try {
 			long count = done.get();
@@ -185,7 +203,20 @@ public class DuccProcessWorkItems implem
 		}
 		return retVal;
 	}
-
+	
+	public long getSecsAvg() {
+		long retVal = 0;
+		try {
+			double msecs = (double)completedMillisAvg.get();
+			retVal = (long)(msecs/1000);
+		}
+		catch(Throwable t) {
+		}
+		if(retVal == 0) {
+			retVal = getSecsAvgV1();
+		}
+		return retVal;
+	}
 	
 	public long getSecsMax() {
 		long retVal = 0;

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java Tue Dec 16 16:08:59 2014
@@ -21,23 +21,32 @@ package org.apache.uima.ducc.transport.e
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.uima.ducc.common.jd.files.workitem.RemoteLocation;
 import org.apache.uima.ducc.common.utils.id.DuccId;
 import org.apache.uima.ducc.container.common.Util;
+import org.apache.uima.ducc.container.common.logger.IComponent;
+import org.apache.uima.ducc.container.common.logger.Logger;
 import org.apache.uima.ducc.container.jd.mh.iface.IOperatingInfo;
 import org.apache.uima.ducc.container.jd.mh.iface.IProcessInfo;
 import org.apache.uima.ducc.container.jd.mh.iface.IWorkItemInfo;
 import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
 import org.apache.uima.ducc.transport.event.common.DuccPerWorkItemStatistics;
+import org.apache.uima.ducc.transport.event.common.DuccProcessWorkItems;
 import org.apache.uima.ducc.transport.event.common.IDuccCompletionType.JobCompletionType;
 import org.apache.uima.ducc.transport.event.common.IDuccPerWorkItemStatistics;
+import org.apache.uima.ducc.transport.event.common.IDuccProcess;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
 import org.apache.uima.ducc.transport.event.common.IRationale;
 import org.apache.uima.ducc.transport.event.jd.IDriverState.DriverState;
 
 public class JobDriverReport implements Serializable, IDriverStatusReport {
 
+	private static Logger logger = Logger.getLogger(JobDriverReport.class, IComponent.Id.JD.name());
+	
 	private static final long serialVersionUID = 200L;
 
 	private DuccId duccId = null;
@@ -107,7 +116,8 @@ public class JobDriverReport implements
 		return retVal;
 	}
 	
-	public JobDriverReport(IOperatingInfo operatingInfo) {
+	public JobDriverReport(IOperatingInfo operatingInfo, IDuccProcessMap dpMap) {
+		String location = "JobDriverReport";
 		setDuccId(getDuccId(operatingInfo));
 		setJdState(operatingInfo.getJdState());
 		//setJmxUrl(driverContainer.getJmxUrl());
@@ -115,7 +125,7 @@ public class JobDriverReport implements
 		setWorkItemsProcessingCompleted(operatingInfo.getWorkItemEndSuccesses());
 		setWorkItemsProcessingError(operatingInfo.getWorkItemEndFailures());
 		setWorkItemsRetry(operatingInfo.getWorkItemUserProcessingErrorRetries());
-		setWorkItemsDispatched(operatingInfo.getWorkItemJpGets()-(operatingInfo.getWorkItemEndSuccesses()+operatingInfo.getWorkItemEndFailures()));
+		setWorkItemsDispatched(operatingInfo.getWorkItemDispatcheds());
 		// min of finished & running
 		long fMin = operatingInfo.getWorkItemFinishedMillisMin();
 		long min = fMin;
@@ -163,23 +173,34 @@ public class JobDriverReport implements
 		ArrayList<IProcessInfo> list = operatingInfo.getProcessItemInfo();
 		if(list != null) {
 			if(!list.isEmpty()) {
-				//TODO
-				/*
 				duccProcessWorkItemsMap = new DuccProcessWorkItemsMap();
 				for(IProcessInfo pi : list) {
-					String name = pi.getNodeName();
+					String ip = pi.getNodeAddress();
 					int pid = pi.getPid();
-					DuccId duccId = getProcessDuccId(name, pid);
-					IDuccProcessWorkItems pwi = new DuccProcessWorkItems();
+					IDuccProcess dp = dpMap.findProcess(ip, ""+pid);
+					if(dp != null) {
+						DuccId key = dp.getDuccId();
+						IDuccProcessWorkItems value = new DuccProcessWorkItems(pi);
+						duccProcessWorkItemsMap.put(key, value);
+					}
+					else {
+						logger.debug(location, null, "process not found: "+"ip="+ip+" "+"pid="+pid);
+						int i = 0;
+						for(Entry<DuccId, IDuccProcess> entry : dpMap.entrySet()) {
+							IDuccProcess value = entry.getValue();
+							logger.debug(location, null, "process["+i+"]: "+"ip="+value.getNodeIdentity().getIp()+" "+"pid="+value.getPID());
+							i++;
+						}
+					}
 				}
-				*/
+			}
+			else {
+				logger.debug(location, null, "list is empty");
 			}
 		}
-	}
-	
-	private DuccId getProcessDuccId(String name, int pid) {
-		DuccId duccId = null;
-		return duccId;
+		else {
+			logger.debug(location, null, "list is null");
+		}
 	}
 	
 	private void setDuccId(DuccId value) {