You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2014/12/16 17:09:00 UTC
svn commit: r1645994 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport:
configuration/jd/JobDriverComponent.java
event/common/DuccProcessWorkItems.java event/jd/JobDriverReport.java
Author: degenaro
Date: Tue Dec 16 16:08:59 2014
New Revision: 1645994
URL: http://svn.apache.org/r1645994
Log:
UIMA-4069 DUCC Job Driver (JD) system classpath
Processes tab accounting (for JPs)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/configuration/jd/JobDriverComponent.java Tue Dec 16 16:08:59 2014
@@ -41,6 +41,7 @@ import org.apache.uima.ducc.container.ne
import org.apache.uima.ducc.transport.configuration.jd.iface.IJobDriverComponent;
import org.apache.uima.ducc.transport.event.JdStateDuccEvent;
import org.apache.uima.ducc.transport.event.OrchestratorAbbreviatedStateDuccEvent;
+import org.apache.uima.ducc.transport.event.common.DuccProcessMap;
import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
@@ -122,6 +123,8 @@ implements IJobDriverComponent {
private AtomicInteger getStateReqNo = new AtomicInteger(0);
+ private IDuccProcessMap dpMap = null;
+
@Override
public JdStateDuccEvent getState() {
String location = "getState";
@@ -129,7 +132,7 @@ implements IJobDriverComponent {
try {
IMessageHandler mh = JobDriver.getInstance().getMessageHandler();
IOperatingInfo oi = mh.handleGetOperatingInfo();
- IDriverStatusReport driverStatusReport = new JobDriverReport(oi);
+ IDriverStatusReport driverStatusReport = new JobDriverReport(oi, dpMap);
driverStatusReport.setNode(node);
driverStatusReport.setPort(port);
state.setState(driverStatusReport);
@@ -166,6 +169,7 @@ implements IJobDriverComponent {
DuccWorkJob dwj = (DuccWorkJob) dwMap.findDuccWork(duccId);
if(dwj != null) {
IDuccProcessMap pMap = dwj.getProcessMap();
+ dpMap = pMap;
for(Entry<DuccId, IDuccProcess> entry : pMap.entrySet()) {
IDuccProcess p = entry.getValue();
ProcessState state = p.getProcessState();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccProcessWorkItems.java Tue Dec 16 16:08:59 2014
@@ -20,6 +20,8 @@ package org.apache.uima.ducc.transport.e
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.uima.ducc.container.jd.mh.iface.IProcessInfo;
+
public class DuccProcessWorkItems implements IDuccProcessWorkItems {
private static final long serialVersionUID = 1L;
@@ -33,9 +35,25 @@ public class DuccProcessWorkItems implem
private AtomicLong lost = new AtomicLong(0);
private AtomicLong preempt = new AtomicLong(0);
private AtomicLong completedMillisTotal = new AtomicLong(0);
+ private AtomicLong completedMillisAvg = new AtomicLong(0);
private AtomicLong completedMillisMax = new AtomicLong(0);
private AtomicLong completedMillisMin = new AtomicLong(0);
+ public DuccProcessWorkItems() {
+ }
+
+ public DuccProcessWorkItems(IProcessInfo pi) {
+ dispatch.set(pi.getDispatch());
+ done.set(pi.getDone());
+ error.set(pi.getError());
+ retry.set(pi.getRetry());
+ lost.set(0);
+ preempt.set(pi.getPreempt());
+ completedMillisAvg.set(pi.getAvg());
+ completedMillisMax.set(pi.getMax());
+ completedMillisMin.set(pi.getMin());
+ }
+
public long getCountUnassigned() {
return unassigned;
}
@@ -172,7 +190,7 @@ public class DuccProcessWorkItems implem
}
- public long getSecsAvg() {
+ public long getSecsAvgV1() {
long retVal = 0;
try {
long count = done.get();
@@ -185,7 +203,20 @@ public class DuccProcessWorkItems implem
}
return retVal;
}
-
+
+ public long getSecsAvg() {
+ long retVal = 0;
+ try {
+ double msecs = (double)completedMillisAvg.get();
+ retVal = (long)(msecs/1000);
+ }
+ catch(Throwable t) {
+ }
+ if(retVal == 0) {
+ retVal = getSecsAvgV1();
+ }
+ return retVal;
+ }
public long getSecsMax() {
long retVal = 0;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java?rev=1645994&r1=1645993&r2=1645994&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/JobDriverReport.java Tue Dec 16 16:08:59 2014
@@ -21,23 +21,32 @@ package org.apache.uima.ducc.transport.e
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.uima.ducc.common.jd.files.workitem.RemoteLocation;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.container.common.Util;
+import org.apache.uima.ducc.container.common.logger.IComponent;
+import org.apache.uima.ducc.container.common.logger.Logger;
import org.apache.uima.ducc.container.jd.mh.iface.IOperatingInfo;
import org.apache.uima.ducc.container.jd.mh.iface.IProcessInfo;
import org.apache.uima.ducc.container.jd.mh.iface.IWorkItemInfo;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
import org.apache.uima.ducc.transport.event.common.DuccPerWorkItemStatistics;
+import org.apache.uima.ducc.transport.event.common.DuccProcessWorkItems;
import org.apache.uima.ducc.transport.event.common.IDuccCompletionType.JobCompletionType;
import org.apache.uima.ducc.transport.event.common.IDuccPerWorkItemStatistics;
+import org.apache.uima.ducc.transport.event.common.IDuccProcess;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
import org.apache.uima.ducc.transport.event.common.IRationale;
import org.apache.uima.ducc.transport.event.jd.IDriverState.DriverState;
public class JobDriverReport implements Serializable, IDriverStatusReport {
+ private static Logger logger = Logger.getLogger(JobDriverReport.class, IComponent.Id.JD.name());
+
private static final long serialVersionUID = 200L;
private DuccId duccId = null;
@@ -107,7 +116,8 @@ public class JobDriverReport implements
return retVal;
}
- public JobDriverReport(IOperatingInfo operatingInfo) {
+ public JobDriverReport(IOperatingInfo operatingInfo, IDuccProcessMap dpMap) {
+ String location = "JobDriverReport";
setDuccId(getDuccId(operatingInfo));
setJdState(operatingInfo.getJdState());
//setJmxUrl(driverContainer.getJmxUrl());
@@ -115,7 +125,7 @@ public class JobDriverReport implements
setWorkItemsProcessingCompleted(operatingInfo.getWorkItemEndSuccesses());
setWorkItemsProcessingError(operatingInfo.getWorkItemEndFailures());
setWorkItemsRetry(operatingInfo.getWorkItemUserProcessingErrorRetries());
- setWorkItemsDispatched(operatingInfo.getWorkItemJpGets()-(operatingInfo.getWorkItemEndSuccesses()+operatingInfo.getWorkItemEndFailures()));
+ setWorkItemsDispatched(operatingInfo.getWorkItemDispatcheds());
// min of finished & running
long fMin = operatingInfo.getWorkItemFinishedMillisMin();
long min = fMin;
@@ -163,23 +173,34 @@ public class JobDriverReport implements
ArrayList<IProcessInfo> list = operatingInfo.getProcessItemInfo();
if(list != null) {
if(!list.isEmpty()) {
- //TODO
- /*
duccProcessWorkItemsMap = new DuccProcessWorkItemsMap();
for(IProcessInfo pi : list) {
- String name = pi.getNodeName();
+ String ip = pi.getNodeAddress();
int pid = pi.getPid();
- DuccId duccId = getProcessDuccId(name, pid);
- IDuccProcessWorkItems pwi = new DuccProcessWorkItems();
+ IDuccProcess dp = dpMap.findProcess(ip, ""+pid);
+ if(dp != null) {
+ DuccId key = dp.getDuccId();
+ IDuccProcessWorkItems value = new DuccProcessWorkItems(pi);
+ duccProcessWorkItemsMap.put(key, value);
+ }
+ else {
+ logger.debug(location, null, "process not found: "+"ip="+ip+" "+"pid="+pid);
+ int i = 0;
+ for(Entry<DuccId, IDuccProcess> entry : dpMap.entrySet()) {
+ IDuccProcess value = entry.getValue();
+ logger.debug(location, null, "process["+i+"]: "+"ip="+value.getNodeIdentity().getIp()+" "+"pid="+value.getPID());
+ i++;
+ }
+ }
}
- */
+ }
+ else {
+ logger.debug(location, null, "list is empty");
}
}
- }
-
- private DuccId getProcessDuccId(String name, int pid) {
- DuccId duccId = null;
- return duccId;
+ else {
+ logger.debug(location, null, "list is null");
+ }
}
private void setDuccId(DuccId value) {