You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2016/04/04 20:10:31 UTC
svn commit: r1737712 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container:
common/ jd/mh/ jd/mh/iface/ jd/mh/impl/ jd/wi/
Author: degenaro
Date: Mon Apr 4 18:10:30 2016
New Revision: 1737712
URL: http://svn.apache.org/viewvc?rev=1737712&view=rev
Log:
UIMA-4882 DUCC Resource Manager (RM) does not adequately expand Job Processes (JPs)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/iface/IOperatingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/impl/OperatingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IRunningWorkItemStatistics.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IWorkItemStatistics.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/RunningWorkItemStatistics.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/WorkItemStatistics.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java Mon Apr 4 18:10:30 2016
@@ -113,6 +113,15 @@ public class Standardize {
killWorkItem,
investmentMillis,
operatingMillis,
+ endAvg,
+ endCount,
+ aboveAvgMillis,
+ aboveAvgCount,
+ aboveAvg,
+ skewAvg,
+ skewCount,
+ runMax,
+ runMin,
;
Label() {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java Mon Apr 4 18:10:30 2016
@@ -139,7 +139,39 @@ public class MessageHandler implements I
CasManager cm = jd.getCasManager();
CasManagerStats cms = cm.getCasManagerStats();
IWorkItemStatistics wis = jd.getWorkItemStatistics();
- IRunningWorkItemStatistics rwis = RunningWorkItemStatistics.getCurrent();
+ long avg = wis.getMillisAvg();
+ IRunningWorkItemStatistics rwis = RunningWorkItemStatistics.getCurrent(avg);
+
+ // aboveAvg: running work items running longer than ended work item avg
+ long count = wis.getCount();
+ long aboveAvgCount = rwis.getAboveAvgCount();
+ long aboveAvg = 0;
+ if(aboveAvgCount > 0) {
+ double aboveAvgMillis = rwis.getAboveAvgMillis();
+ double aboveAvgMillis_dbl = aboveAvgMillis/aboveAvgCount;
+ aboveAvg = (long) aboveAvgMillis_dbl;
+ }
+ // skewAvg: running work items running longer than ended work item avg and ended work items
+ long skewCount = aboveAvgCount+count;
+ long skewAvg = 0;
+ if(skewCount > 0) {
+ double skewMillis = (aboveAvg*aboveAvgCount)+(avg*count);
+ double skewAvg_dbl = skewMillis/skewCount;
+ skewAvg = (long) skewAvg_dbl;
+ }
+ /// message
+ MessageBuffer tb = new MessageBuffer();
+ tb.append(Standardize.Label.runMin.get()+rwis.getMillisMin());
+ tb.append(Standardize.Label.runMax.get()+rwis.getMillisMax());
+ tb.append(Standardize.Label.aboveAvgMillis.get()+rwis.getAboveAvgMillis());
+ tb.append(Standardize.Label.aboveAvgCount.get()+aboveAvgCount);
+ tb.append(Standardize.Label.aboveAvg.get()+aboveAvg);
+ tb.append(Standardize.Label.endCount.get()+count);
+ tb.append(Standardize.Label.endAvg.get()+avg);
+ tb.append(Standardize.Label.skewCount.get()+skewCount);
+ tb.append(Standardize.Label.skewAvg.get()+skewAvg);
+ logger.trace(location, ILogger.null_id, tb.toString());
+
DgenManager dgenManager = DgenManager.getInstance();
oi.setJobId(jd.getJobId());
oi.setJpDeployable(dgenManager.getDeployable());
@@ -168,6 +200,9 @@ public class MessageHandler implements I
oi.setWorkItemFinishedMillisStdDev(wis.getMillisStdDev());
oi.setWorkItemRunningMillisMin(rwis.getMillisMin());
oi.setWorkItemRunningMillisMax(rwis.getMillisMax());
+ oi.setWorkItemRunningAboveAvgMillis(aboveAvg);
+ oi.setWorkItemRunningAboveAvgCount(aboveAvgCount);
+ oi.setWorkItemSkewAvg(skewAvg);
oi.setWorkItemTodMostRecentStart(rwis.getTodMostRecentStart());
oi.setActiveWorkItemInfo(jdh.getActiveWorkItemInfo());
oi.setProcessInfo(jdh.getProcessInfo());
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/iface/IOperatingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/iface/IOperatingInfo.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/iface/IOperatingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/iface/IOperatingInfo.java Mon Apr 4 18:10:30 2016
@@ -109,6 +109,17 @@ public interface IOperatingInfo extends
//
+ public void setWorkItemRunningAboveAvgMillis(long value);
+ public long getWorkItemRunningAboveAvgMillis();
+
+ public void setWorkItemRunningAboveAvgCount(long value);
+ public long getWorkItemRunningAboveAvgCount();
+
+ public void setWorkItemSkewAvg(long value);
+ public long getWorkItemSkewAvg();
+
+ //
+
public void setWorkItemTodMostRecentStart(long value);
public long getWorkItemTodMostRecentStart();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/impl/OperatingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/impl/OperatingInfo.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/impl/OperatingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/impl/OperatingInfo.java Mon Apr 4 18:10:30 2016
@@ -60,6 +60,11 @@ public class OperatingInfo implements IO
private long runningMillisMin = 0;
private long runningMillisMax = 0;
+ private long runningAboveAvgMillis = 0;
+ private long runningAboveAvgCount = 0;
+
+ private long skewAvg = 0;
+
private long todMostRecentStart = 0;
private String jdState = null;
@@ -299,11 +304,41 @@ public class OperatingInfo implements IO
}
@Override
+ public void setWorkItemRunningAboveAvgMillis(long value) {
+ runningAboveAvgMillis = value;
+ }
+
+ @Override
+ public long getWorkItemRunningAboveAvgMillis() {
+ return runningAboveAvgMillis;
+ }
+
+ @Override
+ public void setWorkItemRunningAboveAvgCount(long value) {
+ runningAboveAvgCount = value;
+ }
+
+ @Override
+ public long getWorkItemRunningAboveAvgCount() {
+ return runningAboveAvgCount;
+ }
+
+ @Override
public void setWorkItemTodMostRecentStart(long value) {
todMostRecentStart = value;
}
@Override
+ public void setWorkItemSkewAvg(long value) {
+ skewAvg = value;
+ }
+
+ @Override
+ public long getWorkItemSkewAvg() {
+ return skewAvg;
+ }
+
+ @Override
public long getWorkItemTodMostRecentStart() {
return todMostRecentStart;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IRunningWorkItemStatistics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IRunningWorkItemStatistics.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IRunningWorkItemStatistics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IRunningWorkItemStatistics.java Mon Apr 4 18:10:30 2016
@@ -26,6 +26,12 @@ public interface IRunningWorkItemStatist
public void setMillisMin(long value);
public long getMillisMin();
+ public void setAboveAvgMillis(long value);
+ public long getAboveAvgMillis();
+
+ public void setAboveAvgCount(long value);
+ public long getAboveAvgCount();
+
public void setTodMostRecentStart(long value);
public long getTodMostRecentStart();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IWorkItemStatistics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IWorkItemStatistics.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IWorkItemStatistics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/IWorkItemStatistics.java Mon Apr 4 18:10:30 2016
@@ -21,6 +21,7 @@ package org.apache.uima.ducc.container.j
public interface IWorkItemStatistics {
public void ended(IWorkItem wi);
+ public long getCount();
public long getMillisMax();
public long getMillisMin();
public long getMillisAvg();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/RunningWorkItemStatistics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/RunningWorkItemStatistics.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/RunningWorkItemStatistics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/RunningWorkItemStatistics.java Mon Apr 4 18:10:30 2016
@@ -29,12 +29,17 @@ public class RunningWorkItemStatistics i
private long millisMin = 0;
private long millisMax = 0;
+ private long aboveAvgMillis = 0;
+ private long aboveAvgCount = 0;
+
private long todMostRecentStart = 0;
- public static RunningWorkItemStatistics getCurrent() {
+ public static RunningWorkItemStatistics getCurrent(long mean) {
long min = Long.MAX_VALUE;
long max = 0;
long todMrs = 0;
+ long mAbove = 0;
+ long cAbove = 0;
ConcurrentHashMap<IRemoteWorkerThread, IWorkItem> map = JobDriver.getInstance().getRemoteWorkerThreadMap();
for(Entry<IRemoteWorkerThread, IWorkItem> entry : map.entrySet()) {
IWorkItem wi = entry.getValue();
@@ -47,6 +52,10 @@ public class RunningWorkItemStatistics i
min = time;
}
}
+ if(time > mean) {
+ mAbove += time;
+ cAbove += 1;
+ }
long tod = wi.getTodAck();
if(tod > todMrs) {
todMrs = tod;
@@ -55,14 +64,16 @@ public class RunningWorkItemStatistics i
if(min > max) {
min = max;
}
- RunningWorkItemStatistics retVal = new RunningWorkItemStatistics(min,max,todMrs);
+ RunningWorkItemStatistics retVal = new RunningWorkItemStatistics(min,max,todMrs,mAbove,cAbove);
return retVal;
}
- public RunningWorkItemStatistics(long min, long max, long todMRS) {
+ public RunningWorkItemStatistics(long min, long max, long todMRS, long aaMillis, long aaCount) {
setMillisMin(min);
setMillisMax(max);
setTodMostRecentStart(todMRS);
+ setAboveAvgMillis(aaMillis);
+ setAboveAvgCount(aaCount);
}
@Override
@@ -86,6 +97,26 @@ public class RunningWorkItemStatistics i
}
@Override
+ public void setAboveAvgMillis(long value) {
+ aboveAvgMillis = value;
+ }
+
+ @Override
+ public long getAboveAvgMillis() {
+ return aboveAvgMillis;
+ }
+
+ @Override
+ public void setAboveAvgCount(long value) {
+ aboveAvgCount = value;
+ }
+
+ @Override
+ public long getAboveAvgCount() {
+ return aboveAvgCount;
+ }
+
+ @Override
public void setTodMostRecentStart(long value) {
todMostRecentStart = value;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/WorkItemStatistics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/WorkItemStatistics.java?rev=1737712&r1=1737711&r2=1737712&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/WorkItemStatistics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/WorkItemStatistics.java Mon Apr 4 18:10:30 2016
@@ -39,6 +39,11 @@ public class WorkItemStatistics implemen
}
stats.addValue(time);
}
+
+ @Override
+ public long getCount() {
+ return stats.getNum();
+ }
@Override
public long getMillisMin() {