You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2015/09/24 23:35:41 UTC
svn commit: r1705163 - in
/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src:
main/java/org/apache/uima/ducc/container/common/
main/java/org/apache/uima/ducc/container/jd/
main/java/org/apache/uima/ducc/container/jd/fsm/wi/ main/j...
Author: burn
Date: Thu Sep 24 21:35:40 2015
New Revision: 1705163
URL: http://svn.apache.org/viewvc?rev=1705163&view=rev
Log:
Merge in Performance tab changes from UIMA-4586 & UIMA-4574 & UIMA-4598
Added:
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualInfo.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualKeeper.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceIndividualKeeper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryInfo.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryKeeper.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceSummaryKeeper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/PerfKey.java
- copied unchanged from r1698001, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/PerfKey.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualInfo.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualKeeper.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceIndividualKeeper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryInfo.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryKeeper.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceSummaryKeeper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/
- copied from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/IJobPerformanceIndividual.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/IJobPerformanceIndividual.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/IPersistenceJobPerformanceIndividual.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/IPersistenceJobPerformanceIndividual.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividual.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividual.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividualData.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividualData.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividualJsonGz.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/JobPerformanceIndividualJsonGz.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/PerformanceMetricsIndividualItem.java
- copied unchanged from r1701002, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/files/PerformanceMetricsIndividualItem.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestPerf.java
- copied unchanged from r1698001, uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/test/java/org/apache/uima/ducc/container/jd/test/TestPerf.java
Removed:
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java
Modified:
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/FlagsExtendedHelper.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/IJdConstants.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java
uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
Modified: uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/FlagsExtendedHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/FlagsExtendedHelper.java?rev=1705163&r1=1705162&r2=1705163&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/FlagsExtendedHelper.java (original)
+++ uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/FlagsExtendedHelper.java Thu Sep 24 21:35:40 2015
@@ -19,6 +19,7 @@
package org.apache.uima.ducc.container.common;
import java.io.File;
+import java.util.Map;
import org.apache.uima.ducc.common.container.FlagsHelper;
import org.apache.uima.ducc.container.common.logger.IComponent;
@@ -72,4 +73,30 @@ public class FlagsExtendedHelper extends
}
return sb.toString();
}
+
+ public boolean isIndividualWorkItemPerformance() {
+ String location = "isIndividualWorkItemPerformance";
+ boolean retVal = false;
+ try {
+ Map<String, String> env = System.getenv();
+ if(env != null) {
+ String key = IJdConstants.environmentVariableNameWorkItemPerformance;
+ if(env.containsKey(key)) {
+ String value = env.get(key);
+ logger.debug(location, ILogger.null_id, key+"="+value);
+ if(value != null) {
+ Boolean b = Boolean.valueOf(value);
+ retVal = b.booleanValue();
+ }
+ }
+ else {
+ logger.debug(location, ILogger.null_id, key+" "+"not found in environment");
+ }
+ }
+ }
+ catch(Exception e) {
+ logger.error(location, ILogger.null_id, e);
+ }
+ return retVal;
+ }
}
Modified: uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/IJdConstants.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/IJdConstants.java?rev=1705163&r1=1705162&r2=1705163&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/IJdConstants.java (original)
+++ uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/IJdConstants.java Thu Sep 24 21:35:40 2015
@@ -21,4 +21,8 @@ package org.apache.uima.ducc.container.c
public interface IJdConstants {
public enum DeallocateReason { WorkItemTimeout };
+
+ public String folderNameWorkItemPerformance = "work-item-performance";
+
+ public String environmentVariableNameWorkItemPerformance = "DUCC_WORK_ITEM_PERFORMANCE";
}
Modified: uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java?rev=1705163&r1=1705162&r2=1705163&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java (original)
+++ uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java Thu Sep 24 21:35:40 2015
@@ -44,8 +44,8 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.WorkItemStatistics;
-import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
-import org.apache.uima.ducc.container.jd.wi.perf.WorkItemPerformanceKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceSummaryKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.WorkItemPerformanceSummaryKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
public class JobDriver {
@@ -84,7 +84,7 @@ public class JobDriver {
private DgenManager ddManager = null;
private IWorkItemStateKeeper wisk = null;
- private IWorkItemPerformanceKeeper wipk = null;
+ private IWorkItemPerformanceSummaryKeeper wipsk = null;
private JdState jdState = JdState.Prelaunch;
@@ -109,7 +109,7 @@ public class JobDriver {
killProcessMap = new ConcurrentHashMap<IRemoteLocation, DeallocateReason>();
wis = new WorkItemStatistics();
wisk = new WorkItemStateKeeper(IComponent.Id.JD.name(), logDir);
- wipk = new WorkItemPerformanceKeeper(logDir);
+ wipsk = new WorkItemPerformanceSummaryKeeper(logDir);
cm = new CasManager();
pjdeh = new ProxyJobDriverErrorHandler();
ddManager = new DgenManager();
@@ -148,6 +148,10 @@ public class JobDriver {
return jobId;
}
+ public String getLogDir() {
+ return logDir;
+ }
+
public ConcurrentHashMap<IRemoteWorkerThread, IWorkItem> getRemoteWorkerThreadMap() {
return remoteWorkerThreadMap;
}
@@ -184,8 +188,8 @@ public class JobDriver {
return wisk;
}
- public IWorkItemPerformanceKeeper getWorkItemPerformanceKeeper() {
- return wipk;
+ public IWorkItemPerformanceSummaryKeeper getWorkItemPerformanceSummaryKeeper() {
+ return wipsk;
}
public Map<IRemoteLocation, DeallocateReason>getkillProcessMap() {
@@ -212,7 +216,7 @@ public class JobDriver {
switch(value) {
case Ended:
jdState = value;
- wipk.publish();
+ wipsk.publish();
break;
}
break;
Modified: uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java?rev=1705163&r1=1705162&r2=1705163&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java (original)
+++ uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java Thu Sep 24 21:35:40 2015
@@ -43,7 +43,9 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.WiTracker;
-import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceIndividualKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceSummaryKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.WorkItemPerformanceIndividualKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction;
import org.apache.uima.ducc.container.net.iface.IPerformanceMetrics;
@@ -100,6 +102,17 @@ public class ActionEnd extends ActionEnd
private String keyUniqueName = "uniqueName";
private String keyAnalysisTime = "analysisTime";
+ private String normalizeUniqueName(String uniqueName) {
+ String retVal = uniqueName;
+ try {
+ // expected format: <thread-number> Components /<annotators-path>
+ retVal = uniqueName.trim().split("\\s++", 3)[2];
+ }
+ catch(Exception e) {
+ }
+ return retVal;
+ }
+
private void updatePerformanceMetrics(IActionData actionData, IWorkItem wi) {
String location = "updatePerformanceMetrics";
if(wi != null) {
@@ -113,11 +126,14 @@ public class ActionEnd extends ActionEnd
if(list != null) {
size = list.size();
JobDriver jd = JobDriver.getInstance();
- IWorkItemPerformanceKeeper wipk = jd.getWorkItemPerformanceKeeper();
- wipk.count();
+ String logdir = jd.getLogDir();
+ String wiNo = ""+wi.getSeqNo();
+ IWorkItemPerformanceIndividualKeeper wipik = new WorkItemPerformanceIndividualKeeper(logdir, wiNo);
+ IWorkItemPerformanceSummaryKeeper wipsk = jd.getWorkItemPerformanceSummaryKeeper();
+ wipsk.count();
for(Properties properties : list) {
String name = properties.getProperty(keyName);
- String uniqueName = properties.getProperty(keyUniqueName);
+ String uniqueName = normalizeUniqueName(properties.getProperty(keyUniqueName));
String analysisTime = properties.getProperty(keyAnalysisTime);
long time = 0;
try {
@@ -126,7 +142,8 @@ public class ActionEnd extends ActionEnd
catch(Exception e) {
logger.error(location, ILogger.null_id, e);
}
- wipk.dataAdd(name, uniqueName, time);
+ wipik.dataAdd(name, uniqueName, time);
+ wipsk.dataAdd(name, uniqueName, time);
for(Entry<Object, Object> entry : properties.entrySet()) {
String key = (String) entry.getKey();
String value = (String) entry.getValue();
@@ -136,6 +153,7 @@ public class ActionEnd extends ActionEnd
logger.debug(location, ILogger.null_id, mb.toString());
}
}
+ wipik.publish();
}
MessageBuffer mb = LoggerHelper.getMessageBuffer(actionData);
mb.append(Standardize.Label.size.get()+size);
Modified: uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java?rev=1705163&r1=1705162&r2=1705163&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java (original)
+++ uima/sandbox/uima-ducc/branches/uima-ducc-2.0.1/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java Thu Sep 24 21:35:40 2015
@@ -55,7 +55,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.RunningWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.WiTracker;
-import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceSummaryKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.Hint;
@@ -117,8 +117,8 @@ public class MessageHandler implements I
if(piggybacking.get()) {
try {
JobDriver jd = JobDriver.getInstance();
- IWorkItemPerformanceKeeper wipk = jd.getWorkItemPerformanceKeeper();
- wipk.publish();
+ IWorkItemPerformanceSummaryKeeper wipsk = jd.getWorkItemPerformanceSummaryKeeper();
+ wipsk.publish();
}
catch(Exception e) {
logger.error(location, ILogger.null_id, e);