You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2014/12/30 13:34:31 UTC
svn commit: r1648521 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/ u...
Author: degenaro
Date: Tue Dec 30 12:34:30 2014
New Revision: 1648521
URL: http://svn.apache.org/r1648521
Log:
UIMA-4069 DUCC Job Driver (JD) system classpath
Job Details Performance tab + supporting file + accounting + modest code re-org.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java (with props)
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java (with props)
Removed:
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceMetricsSummaryItem.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceMetricsSummaryMap.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceSummary.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceSummaryBase.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceSummaryReader.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/PerformanceSummaryWriter.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/UimaStatistic.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/ViewJobPerformanceSummary.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccPerfStats.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/SynchronizedStats.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java
uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriverComponent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/v1/DriverStatusReportV1.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccPerfStats.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccPerfStats.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccPerfStats.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-cli/src/main/java/org/apache/uima/ducc/cli/DuccPerfStats.java Tue Dec 30 12:34:30 2014
@@ -37,6 +37,9 @@ import org.apache.commons.cli.ParseExcep
import org.apache.commons.cli.PosixParser;
import org.apache.uima.ducc.common.jd.files.IWorkItemState;
import org.apache.uima.ducc.common.jd.files.IWorkItemState.State;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryItem;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceSummaryReader;
import org.apache.uima.ducc.common.jd.files.workitem.WorkItemStateReader;
import org.apache.uima.ducc.common.node.metrics.ProcessGarbageCollectionStats;
import org.apache.uima.ducc.common.utils.Utils;
@@ -45,9 +48,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
import org.apache.uima.ducc.transport.event.common.IDuccProcessWorkItems;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryItem;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
-import org.apache.uima.ducc.transport.event.jd.PerformanceSummaryReader;
/**
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PerformanceMetricsSummaryItem implements Serializable {
+ /**
+ * please increment this sUID when removing or modifying a field
+ */
+ private static final long serialVersionUID = 1L;
+
+ private String name;
+ private String uniqueName;
+
+ private AtomicLong analysisTime = new AtomicLong(0);
+ private AtomicLong numProcessed = new AtomicLong(0);
+
+ private AtomicLong analysisTimeMin = new AtomicLong(-1);
+ private AtomicLong analysisTimeMax = new AtomicLong(-1);
+
+ public PerformanceMetricsSummaryItem(String name, String uniqueName) {
+ this.name = name;
+ this.uniqueName = uniqueName;
+ }
+
+ public PerformanceMetricsSummaryItem(String name,
+ String uniqueName,
+ long analysisTime,
+ long numProcessed,
+ long analysisTimeMin,
+ long analysisTimeMax
+ )
+ {
+ this.name = name;
+ this.uniqueName = uniqueName;
+ this.analysisTime.set(analysisTime);
+ this.numProcessed.set(numProcessed);
+ this.analysisTimeMin.set(analysisTimeMin);
+ this.analysisTimeMax.set(analysisTimeMax);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public static String delim_old = PerformanceMetricsSummaryMap.delim_old;
+ public static String delim_new = PerformanceMetricsSummaryMap.delim_new;
+
+ public String getDisplayName() {
+ String itemName = getName();
+ String displayName = itemName;
+ try {
+ if(itemName.contains(delim_old)) {
+ displayName = itemName.split(delim_old,2)[1];
+ }
+ else if(itemName.contains(delim_new)) {
+ displayName = itemName.split(delim_new,2)[1];
+ }
+ else {
+ displayName = itemName;
+ }
+ }
+ catch(Throwable t) {
+ }
+ return displayName;
+ }
+
+ public String getUniqueName() {
+ return uniqueName;
+ }
+
+ public long getAnalysisTime() {
+ return analysisTime.get();
+ }
+
+ public long getAnalysisTimeMin() {
+ return analysisTimeMin.get();
+ }
+
+ public long getAnalysisTimeMax() {
+ return analysisTimeMax.get();
+ }
+
+ public long getNumProcessed() {
+ return numProcessed.get();
+ }
+
+ //
+
+ private void updateAnalysisTimeMin(long delta) {
+ long currentValue = analysisTimeMin.get();
+ if(currentValue < 0) {
+ analysisTimeMin.compareAndSet(currentValue, delta);
+ currentValue = analysisTimeMin.get();
+ }
+ while(currentValue > delta) {
+ analysisTimeMin.compareAndSet(currentValue, delta);
+ currentValue = analysisTimeMin.get();
+ }
+ }
+
+ private void updateAnalysisTimeMax(long delta) {
+ long currentValue = analysisTimeMax.get();
+ if(currentValue < 0) {
+ analysisTimeMax.compareAndSet(currentValue, delta);
+ currentValue = analysisTimeMax.get();
+ }
+ while(currentValue < delta) {
+ analysisTimeMax.compareAndSet(currentValue, delta);
+ currentValue = analysisTimeMax.get();
+ }
+ }
+
+ public long addAndGetAnalysisTime(long delta) {
+ updateAnalysisTimeMin(delta);
+ updateAnalysisTimeMax(delta);
+ return analysisTime.addAndGet(delta);
+ }
+
+ public long addAndGetNumProcessed(long delta) {
+ return numProcessed.addAndGet(delta);
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryItem.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.uima.aae.monitor.statistics.AnalysisEnginePerformanceMetrics;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+
+
+public class PerformanceMetricsSummaryMap implements Serializable {
+
+ /**
+ * please increment this sUID when removing or modifying a field
+ */
+ private static final long serialVersionUID = 1L;
+
+ private ConcurrentHashMap<String,PerformanceMetricsSummaryItem> map = new ConcurrentHashMap<String,PerformanceMetricsSummaryItem>();
+
+ private AtomicInteger casCount = new AtomicInteger(0);
+
+ public static String delim_old = "Components,";
+ public static String delim_new = " Components ";
+ public static String delim = delim_new;
+
+ private String getKey(AnalysisEnginePerformanceMetrics item) {
+ String key = "?";
+ try {
+ String uniqueName = item.getUniqueName();
+ if(uniqueName.contains(delim_old)) {
+ key = uniqueName.split(delim_old,2)[1];
+ }
+ else if(uniqueName.contains(delim_new)) {
+ key = uniqueName.split(delim_new,2)[1];
+ }
+ else {
+ key = uniqueName;
+ }
+ }
+ catch(Throwable t) {
+ }
+ return key;
+ }
+
+ private String getDisplayName(AnalysisEnginePerformanceMetrics item) {
+ String key = "?";
+ try {
+ String name = item.getName();
+ if(name.contains(delim_old)) {
+ key = name.split(delim_old,2)[1];
+ }
+ else if(name.contains(delim_new)) {
+ key = name.split(delim_new,2)[1];
+ }
+ else {
+ key = name;
+ }
+ }
+ catch(Throwable t) {
+ }
+ return key;
+ }
+
+ private void addEntry(String key, String displayName) {
+ synchronized(map) {
+ if(!map.containsKey(key)) {
+ PerformanceMetricsSummaryItem summaryItem = new PerformanceMetricsSummaryItem(displayName,key);
+ map.put(key, summaryItem);
+ }
+ }
+ }
+
+ /**
+ * For each unique name in completed work item's performance metrics list:
+ *
+ * 1. accumulate analysis time
+ * 2. accumulate number processed
+ *
+ * Also, accumulate number of (CR provided) CASes processed.
+ *
+ */
+ public void update(DuccLogger duccLogger, List<AnalysisEnginePerformanceMetrics> list) {
+ String methodName = "update";
+ int count = casCount.addAndGet(1);
+ for(AnalysisEnginePerformanceMetrics item : list ) {
+ String key = getKey(item);
+ String displayName = getDisplayName(item);
+ addEntry(key,displayName);
+ PerformanceMetricsSummaryItem summaryItem = map.get(key);
+ synchronized(map) {
+ long timeBefore = summaryItem.getAnalysisTime();
+ long timeItem = item.getAnalysisTime();
+ long timeAfter = summaryItem.addAndGetAnalysisTime(item.getAnalysisTime());
+ long numbBefore = summaryItem.getNumProcessed();
+ long numbItem = item.getNumProcessed();
+ long numbAfter = summaryItem.addAndGetNumProcessed(item.getNumProcessed());
+ if(duccLogger != null) {
+ String t0 = "count:"+count;
+ String t1 = "Numb before:"+numbBefore+" item:"+numbItem+" after:"+numbAfter;
+ String t2 = "Time before:"+timeBefore+" item:"+timeItem+" after:"+timeAfter;
+ String text = t0+" "+t1+" "+t2;
+ duccLogger.debug(methodName, null, text);
+ }
+ }
+ }
+ }
+
+ public void update(List<AnalysisEnginePerformanceMetrics> list) {
+ update(null, list);
+ }
+
+ public Set<Entry<String, PerformanceMetricsSummaryItem>> entrySet() {
+ return map.entrySet();
+ }
+
+ public int size() {
+ return map.size();
+ }
+
+ public int casCount() {
+ return casCount.get();
+ }
+
+ protected void putItem(String key, PerformanceMetricsSummaryItem value) {
+ map.put(key, value);
+ }
+
+ protected void putCasCount(int value) {
+ casCount.set(value);
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceMetricsSummaryMap.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+public class PerformanceSummary extends PerformanceSummaryWriter {
+
+ public PerformanceSummary(String dirname) {
+ super(dirname);
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummary.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummaryJsonGz;
+import org.apache.uima.ducc.common.utils.IOHelper;
+
+public class PerformanceSummaryBase {
+
+ public static final String job_performance_summary_ser = "job-performance-summary.ser";
+
+ protected String filename = null;
+ protected PerformanceMetricsSummaryMap summaryMap = null;
+ protected JobPerformanceSummaryJsonGz jsonGz = null;
+
+ protected PerformanceSummaryBase(String dirname) {
+ init(dirname);
+ }
+
+ protected void init(String dirname) {
+ this.filename = IOHelper.marryDir2File(dirname,job_performance_summary_ser);
+ this.summaryMap = new PerformanceMetricsSummaryMap();
+ this.jsonGz = new JobPerformanceSummaryJsonGz(dirname);
+ }
+
+ public PerformanceMetricsSummaryMap getSummaryMap() {
+ return summaryMap;
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryBase.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.uima.ducc.common.jd.files.IJobPerformanceSummary;
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummary;
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummaryData;
+
+public class PerformanceSummaryReader extends PerformanceSummaryBase {
+
+ public PerformanceSummaryReader(String dirname) {
+ super(dirname);
+ }
+
+ public PerformanceMetricsSummaryMap readJsonGz() throws IOException, ClassNotFoundException {
+ PerformanceMetricsSummaryMap map = new PerformanceMetricsSummaryMap();
+ JobPerformanceSummaryData data = jsonGz.importData();
+ Integer casCount = data.getCasCount();
+ map.putCasCount(casCount);
+ ConcurrentSkipListMap<String, JobPerformanceSummary> gzMap = data.getMap();
+ Set<Entry<String, JobPerformanceSummary>> entries = gzMap.entrySet();
+ for(Entry<String, JobPerformanceSummary> entry : entries) {
+ String key = entry.getKey();
+ IJobPerformanceSummary jps = entry.getValue();
+ PerformanceMetricsSummaryItem value = new PerformanceMetricsSummaryItem(jps.getName(),jps.getUniqueName(),jps.getAnalysisTime(),jps.getNumProcessed(),jps.getAnalysisTimeMin(),jps.getAnalysisTimeMax());
+ map.putItem(key, value);
+ }
+ return map;
+ }
+
+ public PerformanceMetricsSummaryMap readJsonGz(String userId) throws IOException, ClassNotFoundException {
+ PerformanceMetricsSummaryMap map = new PerformanceMetricsSummaryMap();
+ JobPerformanceSummaryData data = null;
+ if(data == null) {
+ try {
+ data = jsonGz.importData(userId);
+ }
+ catch(Exception e) {
+ }
+ }
+ if(data == null) {
+ try {
+ data = jsonGz.importData();
+ }
+ catch(Exception e) {
+ }
+ }
+ Integer casCount = data.getCasCount();
+ map.putCasCount(casCount);
+ ConcurrentSkipListMap<String, JobPerformanceSummary> gzMap = data.getMap();
+ Set<Entry<String, JobPerformanceSummary>> entries = gzMap.entrySet();
+ for(Entry<String, JobPerformanceSummary> entry : entries) {
+ String key = entry.getKey();
+ IJobPerformanceSummary jps = entry.getValue();
+ PerformanceMetricsSummaryItem value = new PerformanceMetricsSummaryItem(jps.getName(),jps.getUniqueName(),jps.getAnalysisTime(),jps.getNumProcessed(),jps.getAnalysisTimeMin(),jps.getAnalysisTimeMax());
+ map.putItem(key, value);
+ }
+ return map;
+ }
+
+ @Deprecated
+ private boolean legacy = true;
+
+ @Deprecated
+ private PerformanceMetricsSummaryMap readSer() {
+ PerformanceMetricsSummaryMap map = null;
+ if(legacy) {
+ try {
+ FileInputStream fis = new FileInputStream(filename);
+ ObjectInputStream in = new ObjectInputStream(fis);
+ summaryMap = (PerformanceMetricsSummaryMap)in.readObject();
+ in.close();
+ map = getSummaryMap();
+ }
+ catch(Exception e) {
+ System.err.println("PerformanceMetricsSummaryMap.readSer() could not read file: "+ filename);
+ }
+ }
+ return map;
+ }
+
+ public PerformanceMetricsSummaryMap readSummary() {
+ PerformanceMetricsSummaryMap map = null;
+ try {
+ map = readJsonGz();
+ return map;
+ }
+ catch(Exception e) {
+ if(!legacy) {
+ e.printStackTrace();
+ }
+ }
+ map = readSer();
+ return map;
+ }
+
+ public PerformanceMetricsSummaryMap readSummary(String userId) {
+ PerformanceMetricsSummaryMap map = null;
+ try {
+ map = readJsonGz(userId);
+ return map;
+ }
+ catch(Exception e) {
+ if(!legacy) {
+ e.printStackTrace();
+ }
+ }
+ map = readSer();
+ return map;
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryReader.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import java.io.FileOutputStream;
+import java.io.ObjectOutputStream;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummary;
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummaryData;
+
+public class PerformanceSummaryWriter extends PerformanceSummaryReader {
+
+ public PerformanceSummaryWriter(String dirname) {
+ super(dirname);
+ }
+
+ private void writeJsonGz() {
+ try {
+ ConcurrentSkipListMap<String, JobPerformanceSummary> map = new ConcurrentSkipListMap<String, JobPerformanceSummary>();
+ Set<Entry<String, PerformanceMetricsSummaryItem>> entries = summaryMap.entrySet();
+ for(Entry<String, PerformanceMetricsSummaryItem> entry : entries) {
+ PerformanceMetricsSummaryItem item = entry.getValue();
+ JobPerformanceSummary jps = new JobPerformanceSummary();
+ jps.setAnalysisTime(item.getAnalysisTime());
+ jps.setAnalysisTimeMax(item.getAnalysisTimeMax());
+ jps.setAnalysisTimeMin(item.getAnalysisTimeMin());
+ jps.setNumProcessed(item.getNumProcessed());
+ jps.setName(item.getName());
+ jps.setUniqueName(item.getUniqueName());
+ map.put(jps.getUniqueName(), jps);
+ }
+
+ Integer casCount = summaryMap.casCount();
+ JobPerformanceSummaryData data = new JobPerformanceSummaryData(map,casCount);
+ jsonGz.exportData(data);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Deprecated
+ private boolean legacy = false;
+
+ @Deprecated
+ private void writeSer(PerformanceMetricsSummaryMap map) {
+ if(legacy) {
+ try {
+ FileOutputStream fos = new FileOutputStream(filename);
+ ObjectOutputStream out = new ObjectOutputStream(fos);
+ out.writeObject(map);
+ out.close();
+ }
+ catch(Exception e) {
+ System.err.println("PerformanceMetricsSummaryMap.writeSer() could not write file: "+ filename);
+ }
+ }
+ }
+
+ public void writeSummary() {
+ writeSer(summaryMap);
+ try {
+ writeJsonGz();
+ return;
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/PerformanceSummaryWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+
+public class UimaStatistic implements Comparable<UimaStatistic> {
+
+ private String shortName;
+ private long analysisTime;
+ private long analysisMinTime;
+ private long analysisMaxTime;
+ private String longName;
+
+ public UimaStatistic (String shortName, String longName, long analysisTime, long anMinTime, long anMaxTime) {
+ this.shortName = shortName;
+ this.analysisTime = analysisTime;
+ this.longName = longName;
+ this.analysisMinTime = anMinTime;
+ this.analysisMaxTime = anMaxTime;
+ }
+
+
+ public int compareTo(UimaStatistic other) {
+ return - Long.signum(analysisTime - other.analysisTime);
+ }
+
+
+ public String toString() {
+// return "UimaStatistic [name=" + shortName + ", analysisTime=" + analysisTime
+// + ", longName=" + longName + "]";
+ return String.format(" %s: %.2f",shortName, analysisTime/(1000.0*ViewJobPerformanceSummary.cascount));
+ }
+
+ public String getShortName() {
+ return shortName;
+ }
+
+ public long getAnalysisTime() {
+ return analysisTime;
+ }
+
+ public long getAnalysisMinTime() {
+ return analysisMinTime;
+ }
+
+ public long getAnalysisMaxTime() {
+ return analysisMaxTime;
+ }
+
+ public String getLongName() {
+ return longName;
+ }
+
+ public String getToolTip() {
+ return shortName + " ("+ longName + ")";
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/UimaStatistic.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.jd.files.perf;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map.Entry;
+
+public class ViewJobPerformanceSummary {
+
+ public static int cascount;
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ if (args.length<1) {
+ System.out.println("this command takes one arg: job-log-dir");
+ System.exit(1);
+ }
+ PerformanceSummary performanceSummary = new PerformanceSummary(args[0]);
+ PerformanceMetricsSummaryMap performanceMetricsSummaryMap = performanceSummary.readSummary();
+ if (performanceMetricsSummaryMap == null || performanceMetricsSummaryMap.size() == 0) {
+ System.err.println("Null map");
+ System.exit(1);
+ }
+ cascount = performanceMetricsSummaryMap.casCount();
+ ArrayList <UimaStatistic> uimaStats = new ArrayList<UimaStatistic>();
+ uimaStats.clear();
+ long analysisTime = 0;
+ try {
+ for (Entry<String, PerformanceMetricsSummaryItem> entry : performanceMetricsSummaryMap.entrySet()) {
+ String key = entry.getKey();
+ int posName = key.lastIndexOf('=');
+ long anTime = entry.getValue().getAnalysisTime();
+ long anMinTime = entry.getValue().getAnalysisTimeMin();
+ long anMaxTime = entry.getValue().getAnalysisTimeMax();
+ analysisTime += anTime;
+ if (posName > 0) {
+ String shortname = key.substring(posName+1);
+ UimaStatistic stat = new UimaStatistic(shortname,
+ entry.getKey(), anTime, anMinTime, anMaxTime);
+ uimaStats.add(stat);
+ }
+ }
+ Collections.sort(uimaStats);
+ int numstats = uimaStats.size();
+ System.out.println("Job = "+args[0]);
+ System.out.printf("Processed %d workitems, Average time = %.1f seconds%n", cascount, analysisTime/(1000.0*cascount));
+ System.out.println("Component breakdown (ave time per workitem in sec):");
+ for (int i = 0; i < numstats; ++i) {
+ System.out.println(uimaStats.get(i).toString());
+ }
+ } catch (Exception e) {
+ System.err.println("Problem parsing PerformanceMetricSummaryMap");
+ }
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/jd/files/perf/ViewJobPerformanceSummary.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.common.utils;
+
+import java.math.RoundingMode;
+import java.text.DecimalFormat;
+
+public class FormatHelper {
+
+ public enum Precision { Whole, Tenths };
+
+ private static DecimalFormat df = new DecimalFormat("#.0");
+
+ static {
+ df.setRoundingMode(RoundingMode.DOWN);
+ }
+
+ public static String duration(final long millis, Precision precision) {
+ long seconds = millis / 1000;
+ long dd = seconds / 86400;
+ long hh = (seconds % 86400) / 3600;
+ long mm = ((seconds % 86400) % 3600) / 60;
+ long ss = ((seconds % 86400) % 3600) % 60;
+ String text = String.format("%d:%02d:%02d:%02d", dd, hh, mm, ss);
+ if(dd == 0) {
+ text = String.format("%02d:%02d:%02d", hh, mm, ss);
+ if(hh == 0) {
+ text = String.format("%02d:%02d", mm, ss);
+ if(mm == 0) {
+ text = String.format("%02d", ss);
+ }
+ }
+ }
+ switch(precision) {
+ case Tenths:
+ double subseconds = (millis%1000.0)/1000;
+ String frac = df.format(subseconds);
+ text = text+frac;
+ break;
+ case Whole:
+ default:
+ break;
+ }
+ return text;
+ }
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/utils/FormatHelper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/Standardize.java Tue Dec 30 12:34:30 2014
@@ -21,6 +21,13 @@ package org.apache.uima.ducc.container.c
public class Standardize {
public enum Label {
+ sum,
+ count,
+ total,
+ name,
+ key,
+ value,
+ size,
status,
add,
puts,
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/SynchronizedStats.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/SynchronizedStats.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/SynchronizedStats.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/common/SynchronizedStats.java Tue Dec 30 12:34:30 2014
@@ -46,6 +46,16 @@ public class SynchronizedStats {
}
}
}
+ public long getNum() {
+ synchronized(mux) {
+ return num;
+ }
+ }
+ public double getSum() {
+ synchronized(mux) {
+ return sum;
+ }
+ }
public double getMax() {
synchronized(mux) {
return max;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/JobDriver.java Tue Dec 30 12:34:30 2014
@@ -39,6 +39,8 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.WorkItemStatistics;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
+import org.apache.uima.ducc.container.jd.wi.perf.WorkItemPerformanceKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
public class JobDriver {
@@ -65,6 +67,7 @@ public class JobDriver {
}
private String jobId = null;
+ private String logDir = null;
private ConcurrentHashMap<IRemoteWorkerThread, IWorkItem> remoteThreadMap = null;
private ConcurrentHashMap<IRemotePid, IProcessStatistics> remoteProcessMap = null;
private IWorkItemStatistics wis = null;
@@ -74,6 +77,7 @@ public class JobDriver {
private DgenManager ddManager = null;
private IWorkItemStateKeeper wisk = null;
+ private IWorkItemPerformanceKeeper wipk = null;
private JdState jdState = null;
@@ -87,10 +91,12 @@ public class JobDriver {
jdState = JdState.Initializing;
FlagsExtendedHelper feh = FlagsExtendedHelper.getInstance();
jobId = feh.getJobId();
+ logDir = feh.getLogDirectory();
remoteThreadMap = new ConcurrentHashMap<IRemoteWorkerThread, IWorkItem>();
remoteProcessMap = new ConcurrentHashMap<IRemotePid, IProcessStatistics>();
wis = new WorkItemStatistics();
- wisk = new WorkItemStateKeeper(IComponent.Id.JD.name(), feh.getLogDirectory());
+ wisk = new WorkItemStateKeeper(IComponent.Id.JD.name(), logDir);
+ wipk = new WorkItemPerformanceKeeper(logDir);
cm = new CasManager();
pjdeh = new ProxyJobDriverErrorHandler();
ddManager = new DgenManager();
@@ -138,6 +144,10 @@ public class JobDriver {
return wisk;
}
+ public IWorkItemPerformanceKeeper getWorkItemPerformanceKeeper() {
+ return wipk;
+ }
+
public JdState getJdState() {
synchronized(jdState) {
return jdState;
@@ -158,14 +168,18 @@ public class JobDriver {
switch(value) {
case Ended:
jdState = value;
+ wipk.publish();
break;
}
break;
case Initializing:
switch(value) {
case Ended:
+ jdState = value;
+ break;
case Active:
jdState = value;
+ break;
}
break;
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/fsm/wi/ActionEnd.java Tue Dec 30 12:34:30 2014
@@ -18,6 +18,10 @@
*/
package org.apache.uima.ducc.container.jd.fsm.wi;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+
import org.apache.uima.ducc.common.jd.files.workitem.IWorkItemStateKeeper;
import org.apache.uima.ducc.container.common.MessageBuffer;
import org.apache.uima.ducc.container.common.MetaCasHelper;
@@ -41,9 +45,11 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IProcessStatistics;
import org.apache.uima.ducc.container.jd.wi.IWorkItem;
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
+import org.apache.uima.ducc.container.net.iface.IPerformanceMetrics;
public class ActionEnd extends Action implements IAction {
@@ -91,6 +97,7 @@ public class ActionEnd extends Action im
cm.getCasManagerStats().incEndSuccess();
wi.setTodEnd();
updateStatistics(wi);
+ updatePerformanceMetrics(wi);
MessageBuffer mb = new MessageBuffer();
mb.append(Standardize.Label.transNo.get()+trans.getTransactionId().toString());
mb.append(Standardize.Label.seqNo.get()+metaCas.getSystemKey());
@@ -110,6 +117,48 @@ public class ActionEnd extends Action im
mb.append(Standardize.Label.min.get()+wis.getMillisMin());
logger.debug(location, ILogger.null_id, mb.toString());
}
+
+ private String keyName = "name";
+ private String keyUniqueName = "uniqueName";
+ private String keyAnalysisTime = "analysisTime";
+
+ private void updatePerformanceMetrics(IWorkItem wi) {
+ String location = "updatePerformanceMetrics";
+ IMetaCas metaCas = wi.getMetaCas();
+ IPerformanceMetrics performanceMetrics = metaCas.getPerformanceMetrics();
+ List<Properties> list = performanceMetrics.get();
+ int size = 0;
+ if(list != null) {
+ size = list.size();
+ JobDriver jd = JobDriver.getInstance();
+ IWorkItemPerformanceKeeper wipk = jd.getWorkItemPerformanceKeeper();
+ for(Properties properties : list) {
+ String name = properties.getProperty(keyName);
+ String uniqueName = properties.getProperty(keyUniqueName);
+ String analysisTime = properties.getProperty(keyAnalysisTime);
+ long time = 0;
+ try {
+ time = Long.parseLong(analysisTime);
+ }
+ catch(Exception e) {
+ logger.error(location, ILogger.null_id, e);
+ }
+ wipk.dataAdd(name, uniqueName, time);
+ for(Entry<Object, Object> entry : properties.entrySet()) {
+ String key = (String) entry.getKey();
+ String value = (String) entry.getValue();
+ MessageBuffer mb = new MessageBuffer();
+ mb.append(Standardize.Label.key.get()+key);
+ mb.append(Standardize.Label.value.get()+value);
+ logger.debug(location, ILogger.null_id, mb.toString());
+ }
+ }
+ }
+ MessageBuffer mb = new MessageBuffer();
+ mb.append(Standardize.Label.seqNo.get()+metaCas.getSystemKey());
+ mb.append(Standardize.Label.size.get()+size);
+ logger.debug(location, ILogger.null_id, mb.toString());
+ }
private void checkEnded(CasManager cm) {
String location = "checkEnded";
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/mh/MessageHandler.java Tue Dec 30 12:34:30 2014
@@ -49,6 +49,7 @@ import org.apache.uima.ducc.container.jd
import org.apache.uima.ducc.container.jd.wi.IWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.RunningWorkItemStatistics;
import org.apache.uima.ducc.container.jd.wi.WorkItem;
+import org.apache.uima.ducc.container.jd.wi.perf.IWorkItemPerformanceKeeper;
import org.apache.uima.ducc.container.net.iface.IMetaCas;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction;
import org.apache.uima.ducc.container.net.iface.IMetaCasTransaction.JdState;
@@ -74,9 +75,22 @@ public class MessageHandler implements I
acks.incrementAndGet();
}
+ private void piggyback() {
+ String location = "piggyback";
+ try {
+ JobDriver jd = JobDriver.getInstance();
+ IWorkItemPerformanceKeeper wipk = jd.getWorkItemPerformanceKeeper();
+ wipk.publish();
+ }
+ catch(Exception e) {
+ logger.error(location, ILogger.null_id, e);
+ }
+ }
+
@Override
public IOperatingInfo handleGetOperatingInfo() {
String location = "handleGetOperatingInfo";
+ piggyback();
IOperatingInfo retVal = null;
JobDriver jd = JobDriver.getInstance();
Id jobid = Transform.toId(jd.getJobId());
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.container.jd.wi.perf;
+
+public interface IWorkItemPerformanceInfo {
+
+ public String getName();
+ public String getUniqueName();
+ public double getCount();
+ public double getTime();
+ public double getPctOfTime();
+ public double getAvg();
+ public double getMin();
+ public double getMax();
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceInfo.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.container.jd.wi.perf;
+
+import java.util.List;
+
+
+public interface IWorkItemPerformanceKeeper {
+
+ public List<IWorkItemPerformanceInfo> dataGet();
+ public void dataAdd(String name, String uniqueName, long millis);
+ public void publish();
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/IWorkItemPerformanceKeeper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.container.jd.wi.perf;
+
+public class WorkItemPerformanceInfo implements IWorkItemPerformanceInfo {
+
+ String name = null;
+ String uniqueName = null;
+ double count = 0;
+ double time = 0;
+ double pctOfTime = 0;
+ double avg = 0;
+ double min = 0;
+ double max = 0;
+
+ public WorkItemPerformanceInfo(
+ String name,
+ String uniquename,
+ double count,
+ double time,
+ double pctOfTime,
+ double avg,
+ double min,
+ double max
+ )
+ {
+ setName(name);
+ setUniqueName(uniquename);
+ setCount(count);
+ setTime(time);
+ setPctOfTime(pctOfTime);
+ setAvg(avg);
+ setMin(min);
+ setMax(max);
+ }
+
+ private void setName(String value) {
+ name = value;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ private void setUniqueName(String value) {
+ uniqueName = value;
+ }
+
+ @Override
+ public String getUniqueName() {
+ return uniqueName;
+ }
+
+ private void setCount(double value) {
+ count = value;
+ }
+
+ @Override
+ public double getCount() {
+ return count;
+ }
+
+ private void setTime(double value) {
+ time = value;
+ }
+
+ @Override
+ public double getTime() {
+ return time;
+ }
+
+ private void setPctOfTime(double value) {
+ pctOfTime = value;
+ }
+
+ @Override
+ public double getPctOfTime() {
+ return pctOfTime;
+ }
+
+ private void setAvg(double value) {
+ avg = value;
+ }
+
+ @Override
+ public double getAvg() {
+ return avg;
+ }
+
+ private void setMin(double value) {
+ min = value;
+ }
+
+ @Override
+ public double getMin() {
+ return min;
+ }
+
+ private void setMax(double value) {
+ max = value;
+ }
+
+ @Override
+ public double getMax() {
+ return max;
+ }
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceInfo.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java?rev=1648521&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java Tue Dec 30 12:34:30 2014
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.container.jd.wi.perf;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummary;
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummaryData;
+import org.apache.uima.ducc.common.jd.files.JobPerformanceSummaryJsonGz;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryItem;
+import org.apache.uima.ducc.common.utils.FormatHelper;
+import org.apache.uima.ducc.common.utils.FormatHelper.Precision;
+import org.apache.uima.ducc.container.common.MessageBuffer;
+import org.apache.uima.ducc.container.common.Standardize;
+import org.apache.uima.ducc.container.common.SynchronizedStats;
+import org.apache.uima.ducc.container.common.logger.IComponent;
+import org.apache.uima.ducc.container.common.logger.ILogger;
+import org.apache.uima.ducc.container.common.logger.Logger;
+
+public class WorkItemPerformanceKeeper implements IWorkItemPerformanceKeeper {
+
+ private static Logger logger = Logger.getLogger(WorkItemPerformanceKeeper.class, IComponent.Id.JD.name());
+
+ private String logDir = null;
+
+ private AtomicLong count = new AtomicLong(0);
+ private AtomicLong total = new AtomicLong(0);
+
+ private class Data {
+ public String uniqueName;
+ public SynchronizedStats synchronizedStats;
+ public Data(String uniqueName, SynchronizedStats synchronizedStats) {
+ this.uniqueName = uniqueName;
+ this.synchronizedStats = synchronizedStats;
+ }
+ }
+
+ private ConcurrentHashMap<String, Data> map = new ConcurrentHashMap<String, Data>();
+
+ public WorkItemPerformanceKeeper(String logDir) {
+ setLogDir(logDir);
+ }
+
+ private void setLogDir(String value) {
+ logDir = value;
+ }
+
+ @Override
+ public List<IWorkItemPerformanceInfo> dataGet() {
+ List<IWorkItemPerformanceInfo> list = new ArrayList<IWorkItemPerformanceInfo>();
+ for(Entry<String, Data> entry : map.entrySet()) {
+ String name = entry.getKey();
+ Data data = entry.getValue();
+ String uniqueName = data.uniqueName;
+ SynchronizedStats stats = data.synchronizedStats;
+ double count = stats.getNum();
+ double time = stats.getSum();
+ double pctOfTime = 0;
+ if(total.get()> 0) {
+ pctOfTime = time*(100/total.get());
+ }
+ double avg = stats.getMean();
+ double min = stats.getMin();
+ double max = stats.getMax();
+ IWorkItemPerformanceInfo item = new WorkItemPerformanceInfo(
+ name,
+ uniqueName,
+ count,
+ time,
+ pctOfTime,
+ avg,
+ min,
+ max
+ );
+ list.add(item);
+ }
+ return list;
+ }
+
+ @Override
+ public void dataAdd(String name, String uniqueName, long time) {
+ String location = "dataAdd";
+ try {
+ // name
+ String key = name;
+ if(!map.containsKey(key)) {
+ Data data = new Data(uniqueName, new SynchronizedStats());
+ map.putIfAbsent(key, data);
+ }
+ // data
+ Data data = map.get(key);
+ // stats
+ SynchronizedStats stats = data.synchronizedStats;
+ stats.addValue(time);
+ count.addAndGet(1);
+ total.addAndGet(time);
+ // sum
+ long lTimeSum = (long)stats.getSum();
+ String timeSum = FormatHelper.duration(lTimeSum,Precision.Tenths);
+ // avg
+ long lTimeAvg = (long)stats.getMean();
+ String timeAvg = FormatHelper.duration(lTimeAvg,Precision.Tenths);
+ // min
+ long lTimeMin = (long)stats.getMin();
+ String timeMin = FormatHelper.duration(lTimeMin,Precision.Tenths);
+ // max
+ long lTimeMax = (long)stats.getMax();
+ String timeMax = FormatHelper.duration(lTimeMax,Precision.Tenths);
+ // log
+ MessageBuffer mb = new MessageBuffer();
+ mb.append(Standardize.Label.name.get()+name);
+ mb.append(Standardize.Label.sum.get()+timeSum);
+ mb.append(Standardize.Label.avg.get()+timeAvg);
+ mb.append(Standardize.Label.min.get()+timeMin);
+ mb.append(Standardize.Label.max.get()+timeMax);
+ mb.append(Standardize.Label.count.get()+count.get());
+ mb.append(Standardize.Label.total.get()+total.get());
+ logger.debug(location, ILogger.null_id, mb.toString());
+ }
+ catch(Exception e) {
+ logger.error(location, ILogger.null_id, e);
+ }
+ }
+
+ private PerformanceMetricsSummaryItem create(IWorkItemPerformanceInfo wipi) {
+ PerformanceMetricsSummaryItem retVal = new PerformanceMetricsSummaryItem(
+ wipi.getName(),
+ wipi.getUniqueName(),
+ (long)wipi.getTime(),
+ (long)wipi.getCount(),
+ (long)wipi.getMin(),
+ (long)wipi.getMax()
+ );
+ return retVal;
+ }
+
+ public void publish() {
+ try {
+ List<IWorkItemPerformanceInfo> list = dataGet();
+ ConcurrentSkipListMap<String, JobPerformanceSummary> map = new ConcurrentSkipListMap<String, JobPerformanceSummary>();
+ for(IWorkItemPerformanceInfo wipi : list) {
+ PerformanceMetricsSummaryItem item = create(wipi);
+ JobPerformanceSummary jps = new JobPerformanceSummary();
+ jps.setAnalysisTime(item.getAnalysisTime());
+ jps.setAnalysisTimeMax(item.getAnalysisTimeMax());
+ jps.setAnalysisTimeMin(item.getAnalysisTimeMin());
+ jps.setNumProcessed(item.getNumProcessed());
+ jps.setName(item.getName());
+ jps.setUniqueName(item.getUniqueName());
+ map.put(jps.getUniqueName(), jps);
+ }
+ Integer casCount = new Integer((int)count.get());
+ JobPerformanceSummaryData data = new JobPerformanceSummaryData(map,casCount);
+ JobPerformanceSummaryJsonGz jsonGz = new JobPerformanceSummaryJsonGz(logDir);
+ jsonGz.exportData(data);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+
+}
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: uima/sandbox/uima-ducc/trunk/uima-ducc-container/src/main/java/org/apache/uima/ducc/container/jd/wi/perf/WorkItemPerformanceKeeper.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/IJobDriverAccess.java Tue Dec 30 12:34:30 2014
@@ -18,10 +18,10 @@
*/
package org.apache.uima.ducc.jd;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceSummaryWriter;
import org.apache.uima.ducc.common.jd.files.workitem.WorkItemStateKeeper;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.jd.client.CasDispatchMap;
-import org.apache.uima.ducc.transport.event.jd.PerformanceSummaryWriter;
import org.apache.uima.ducc.transport.event.jd.v1.DriverStatusReportV1;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriver.java Tue Dec 30 12:34:30 2014
@@ -36,6 +36,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.uima.aae.client.UimaAsynchronousEngine;
import org.apache.uima.adapter.jms.client.BaseUIMAAsynchronousEngine_impl;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryItem;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceSummaryWriter;
import org.apache.uima.ducc.common.jd.files.workitem.WorkItemStateKeeper;
import org.apache.uima.ducc.common.jd.files.workitem.WorkItemStatistics;
import org.apache.uima.ducc.common.jd.plugin.IJdProcessExceptionHandler;
@@ -74,9 +77,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IRationale;
import org.apache.uima.ducc.transport.event.common.IResourceState.ProcessDeallocationType;
import org.apache.uima.ducc.transport.event.common.Rationale;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryItem;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
-import org.apache.uima.ducc.transport.event.jd.PerformanceSummaryWriter;
import org.apache.uima.ducc.transport.event.jd.v1.DriverStatusReportV1;
import org.apache.uima.ducc.transport.event.jd.v1.DuccProcessWorkItemsReportV1;
import org.apache.uima.ducc.transport.uima.dd.generator.DeploymentDescriptorGenerator;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriverComponent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriverComponent.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriverComponent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-jd/src/main/java/org/apache/uima/ducc/jd/JobDriverComponent.java Tue Dec 30 12:34:30 2014
@@ -29,6 +29,7 @@ import org.apache.uima.ducc.common.NodeI
import org.apache.uima.ducc.common.component.AbstractDuccComponent;
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.jd.JdConstants;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceSummaryWriter;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.ExceptionHelper;
@@ -40,7 +41,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccProcess;
import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
import org.apache.uima.ducc.transport.event.common.Rationale;
-import org.apache.uima.ducc.transport.event.jd.PerformanceSummaryWriter;
import org.apache.uima.ducc.transport.event.jd.v1.DriverStatusReportV1;
import org.apache.uima.ducc.transport.json.jp.JobProcessCollection;
import org.apache.uima.ducc.transport.json.jp.JobProcessData;
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/DuccSchedulingInfo.java Tue Dec 30 12:34:30 2014
@@ -20,9 +20,9 @@ package org.apache.uima.ducc.transport.e
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
/**
* Data utilized by the work scheduler.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccSchedulingInfo.java Tue Dec 30 12:34:30 2014
@@ -21,9 +21,9 @@ package org.apache.uima.ducc.transport.e
import java.io.Serializable;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
import org.apache.uima.ducc.common.utils.id.DuccId;
import org.apache.uima.ducc.transport.event.common.IDuccUnits.MemoryUnits;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
public interface IDuccSchedulingInfo extends Serializable {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/v1/DriverStatusReportV1.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/v1/DriverStatusReportV1.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/v1/DriverStatusReportV1.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/jd/v1/DriverStatusReportV1.java Tue Dec 30 12:34:30 2014
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.SerializationUtils;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
import org.apache.uima.ducc.common.jd.files.workitem.RemoteLocation;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
@@ -38,7 +39,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.Rationale;
import org.apache.uima.ducc.transport.event.common.Util;
import org.apache.uima.ducc.transport.event.jd.DuccProcessWorkItemsReport;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
import org.apache.uima.ducc.transport.event.jd.IDriverState.DriverState;
@Deprecated
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1648521&r1=1648520&r2=1648521&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java Tue Dec 30 12:34:30 2014
@@ -52,6 +52,10 @@ import org.apache.uima.ducc.common.boot.
import org.apache.uima.ducc.common.internationalization.Messages;
import org.apache.uima.ducc.common.jd.files.IWorkItemState;
import org.apache.uima.ducc.common.jd.files.IWorkItemState.State;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryItem;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceMetricsSummaryMap;
+import org.apache.uima.ducc.common.jd.files.perf.PerformanceSummary;
+import org.apache.uima.ducc.common.jd.files.perf.UimaStatistic;
import org.apache.uima.ducc.common.jd.files.workitem.WorkItemStateReader;
import org.apache.uima.ducc.common.system.SystemState;
import org.apache.uima.ducc.common.utils.DuccLogger;
@@ -83,10 +87,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
import org.apache.uima.ducc.transport.event.common.IResourceState.ProcessDeallocationType;
import org.apache.uima.ducc.transport.event.common.TimeWindow;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryItem;
-import org.apache.uima.ducc.transport.event.jd.PerformanceMetricsSummaryMap;
-import org.apache.uima.ducc.transport.event.jd.PerformanceSummary;
-import org.apache.uima.ducc.transport.event.jd.UimaStatistic;
import org.apache.uima.ducc.ws.DuccDaemonsData;
import org.apache.uima.ducc.ws.DuccData;
import org.apache.uima.ducc.ws.DuccDataHelper;