You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/04/26 03:01:29 UTC
svn commit: r1476007 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws:
DuccData.java PagingInfo.java PagingObserver.java server/DuccHandler.java
server/DuccHandlerJsonFormat.java server/DuccHandlerLegacy.java
Author: degenaro
Date: Fri Apr 26 01:01:29 2013
New Revision: 1476007
URL: http://svn.apache.org/r1476007
Log:
UIMA-2838 DUCC webserver (WS) tracks and reports on Paging, replacing less accurate Swapping notification on Jobs page
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingObserver.java
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java?rev=1476007&r1=1476006&r2=1476007&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java Fri Apr 26 01:01:29 2013
@@ -55,6 +55,8 @@ public class DuccData {
private static ConcurrentSkipListMap<Info,Info> sortedCombinedReservations = new ConcurrentSkipListMap<Info,Info>();
private static ConcurrentSkipListMap<DuccId,Info> keyMapCombinedReservations = new ConcurrentSkipListMap<DuccId,Info>();
+ private static PagingObserver pagingObserver = PagingObserver.getInstance();
+
private static DuccData duccData = new DuccData();
private static long slack = 100;
@@ -113,6 +115,7 @@ public class DuccData {
}
public void put(DuccWorkMap map) {
+ String location = "put";
synchronized(this) {
DuccWorkMap mapCopy = map.deepCopy();
mergeHistory(map);
@@ -123,6 +126,19 @@ public class DuccData {
IDuccWork duccWork = map.findDuccWork(duccId);
duccWorkMap.addDuccWork(duccWork);
updateSortedMaps(duccWork);
+ pagingObserver.put(duccWork);
+ PagingInfo pi;
+ long dataTotal = 0;
+ long diffTotal = 0;
+ pi = pagingObserver.getData(duccId);
+ if(pi != null) {
+ dataTotal = pi.total;
+ }
+ pi = pagingObserver.getDiff(duccId);
+ if(pi != null) {
+ diffTotal = pi.total;
+ }
+ logger.debug(location, duccId, "dataTotal:"+dataTotal+" "+"diffTotal:"+diffTotal);
}
}
prune();
@@ -141,6 +157,7 @@ public class DuccData {
DuccId duccId = jobInfo.getJob().getDuccId();
sortedJobs.remove(jobInfo);
keyMapJobs.remove(duccId);
+ pagingObserver.remove(duccId);
logger.info(location, duccId, "size: "+sortedJobs.size());
pruned++;
}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingInfo.java?rev=1476007&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingInfo.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingInfo.java Fri Apr 26 01:01:29 2013
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.ws;
+
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkPopDriver;
+import org.apache.uima.ducc.transport.event.common.IDuccProcess;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+
+public class PagingInfo {
+
+ public long total = 0;
+ public long procs = 0;
+
+ public long timestamp = System.currentTimeMillis();
+
+ PagingInfo() {
+ }
+
+ PagingInfo(long total, long procs) {
+ this.total = total;
+ this.procs = procs;
+ }
+
+ PagingInfo(IDuccWorkJob job) {
+ if(job != null) {
+ count(this, job.getProcessMap());
+ DuccWorkPopDriver driver = job.getDriver();
+ if(driver != null) {
+ count(this, driver.getProcessMap());
+ }
+ }
+ }
+
+ private void count(PagingInfo pagingInfo, IDuccProcessMap pMap) {
+ if(pMap != null) {
+ for(DuccId pId : pMap.keySet()) {
+ IDuccProcess process = pMap.get(pId);
+ total += process.getMajorFaults();
+ procs += 1;
+ }
+ }
+ }
+
+ public void display() {
+ System.out.println("total:"+total+" "+"procs:"+procs);
+ }
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingObserver.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingObserver.java?rev=1476007&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingObserver.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/PagingObserver.java Fri Apr 26 01:01:29 2013
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.ws;
+
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.IDuccWork;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+
+public class PagingObserver {
+
+ private static PagingObserver instance = new PagingObserver();
+
+ public static int intervalInSeconds = 60;
+
+ public static PagingObserver getInstance() {
+ return instance;
+ }
+
+ private ConcurrentSkipListMap<DuccId,PagingInfo> mapData = new ConcurrentSkipListMap<DuccId,PagingInfo>();
+ private ConcurrentSkipListMap<DuccId,PagingInfo> mapDiff = new ConcurrentSkipListMap<DuccId,PagingInfo>();
+
+ public PagingObserver() {
+ }
+
+ public void put(IDuccWork dw) {
+ if(dw != null) {
+ if(dw instanceof IDuccWorkJob) {
+ IDuccWorkJob job = (IDuccWorkJob) dw;
+ if(!job.isCompleted()) {
+ put(job);
+ }
+ }
+ }
+ }
+
+ private void update(DuccId id, PagingInfo pCurr, PagingInfo pPrev) {
+ PagingInfo pDiff = new PagingInfo(pCurr.total-pPrev.total, pCurr.procs-pPrev.procs);
+ mapData.put(id, pCurr);
+ mapDiff.put(id, pDiff);
+ }
+
+ public void put(IDuccWorkJob job) {
+ if(job != null) {
+ DuccId id = job.getDuccId();
+ if(id != null) {
+ PagingInfo pCurr = new PagingInfo(job);
+ PagingInfo pPrev = mapData.get(id);
+ if(pPrev == null) {
+ update(id, pCurr, pCurr);
+ }
+ else {
+ long elapsedMillis = pCurr.timestamp - pPrev.timestamp;
+ if(elapsedMillis > intervalInSeconds * 1000) {
+ update(id, pCurr, pPrev);
+ }
+ }
+ }
+ }
+ }
+
+ public void remove(DuccId duccId) {
+ if(duccId != null) {
+ mapData.remove(duccId);
+ mapDiff.remove(duccId);
+ }
+ }
+
+ public PagingInfo getData(DuccId duccId) {
+ PagingInfo retVal = null;
+ if(duccId != null) {
+ retVal = mapData.get(duccId);
+ }
+ if(retVal == null) {
+ retVal = new PagingInfo();
+ }
+ return retVal;
+ }
+
+ public PagingInfo getDiff(DuccId duccId) {
+ PagingInfo retVal = null;
+ if(duccId != null) {
+ retVal = mapDiff.get(duccId);
+ }
+ if(retVal == null) {
+ retVal = new PagingInfo();
+ }
+ return retVal;
+ }
+
+ public long getDiffCount(DuccId duccId) {
+ return getDiff(duccId).total;
+ }
+
+ public boolean isPaging(DuccId duccId) {
+ return getDiffCount(duccId) > 0;
+ }
+
+ public boolean isPaging(IDuccWorkJob job) {
+ boolean retVal = false;
+ if(job != null) {
+ DuccId duccId = job.getDuccId();
+ retVal = isPaging(duccId);
+ }
+ return retVal;
+ }
+
+ public static void main(String[] args) {
+ PagingObserver po = PagingObserver.getInstance();
+ DuccId id = null;
+ po.remove(id);
+ id = new DuccId(0);
+ po.remove(id);
+ PagingInfo pi = new PagingInfo();
+ pi.display();
+ IDuccWorkJob job = new DuccWorkJob();
+ pi = new PagingInfo(job);
+ pi.display();
+ }
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java?rev=1476007&r1=1476006&r2=1476007&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandler.java Fri Apr 26 01:01:29 2013
@@ -43,7 +43,6 @@ import javax.servlet.http.HttpServletReq
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
-import org.apache.uima.ducc.common.NodeIdentity;
import org.apache.uima.ducc.common.authentication.AuthenticationManager;
import org.apache.uima.ducc.common.authentication.IAuthenticationManager;
import org.apache.uima.ducc.common.boot.DuccDaemonRuntimeProperties;
@@ -424,6 +423,7 @@ public class DuccHandler extends DuccAbs
case Initializing:
case Running:
sb.append("<td>");
+ /*
NodeIdentity nodeId = jp.getNodeIdentity();
if(nodeId != null) {
String ip = nodeId.getIp();
@@ -433,6 +433,7 @@ public class DuccHandler extends DuccAbs
sb.append("</span>");
}
}
+ */
sb.append("</td>");
break;
default:
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java?rev=1476007&r1=1476006&r2=1476007&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerJsonFormat.java Fri Apr 26 01:01:29 2013
@@ -72,6 +72,7 @@ import org.apache.uima.ducc.ws.DuccMachi
import org.apache.uima.ducc.ws.Info;
import org.apache.uima.ducc.ws.JobInfo;
import org.apache.uima.ducc.ws.MachineInfo;
+import org.apache.uima.ducc.ws.PagingObserver;
import org.apache.uima.ducc.ws.ReservationInfo;
import org.apache.uima.ducc.ws.registry.IServicesRegistry;
import org.apache.uima.ducc.ws.registry.ServicesRegistry;
@@ -90,6 +91,8 @@ public class DuccHandlerJsonFormat exten
private static Messages messages = Messages.getInstance();
private static DuccId jobid = null;
+ private static PagingObserver pagingObserver = PagingObserver.getInstance();
+
private final String jsonFormatJobsAaData = duccContextJsonFormat+"-aaData-jobs";
private final String jsonFormatReservationsAaData = duccContextJsonFormat+"-aaData-reservations";
private final String jsonFormatServicesAaData = duccContextJsonFormat+"-aaData-services";
@@ -121,26 +124,20 @@ public class DuccHandlerJsonFormat exten
if(job.isOperational()) {
boolean multi = false;
sb.append("<span>");
- ArrayList<String> swappingMachines = getSwappingMachines(job);
- if(!swappingMachines.isEmpty()) {
- StringBuffer mb = new StringBuffer();
- for(String machine : swappingMachines) {
- mb.append(machine);
- mb.append(" ");
- }
- String ml = mb.toString().trim();
- if(multi) {
- sb.append(" ");
- }
+ if(pagingObserver.isPaging(job)) {
multi = true;
- sb.append("<span class=\"health_red\" title=\""+ml+"\">");
- sb.append("Swapping");
+ String title = "a page-in operation occurred for at least one process during the past "+PagingObserver.intervalInSeconds+" seconds";
+ sb.append("<span class=\"health_red\" title=\""+title+"\">");
+ sb.append("Paging");
sb.append("</span>");
}
sb.append("</span>");
//
String monitor = getMonitor(duccId, type, multi);
if(monitor.length() > 0) {
+ if(multi) {
+ sb.append(" ");
+ }
multi = true;
sb.append(monitor);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java?rev=1476007&r1=1476006&r2=1476007&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerLegacy.java Fri Apr 26 01:01:29 2013
@@ -63,6 +63,7 @@ import org.apache.uima.ducc.ws.DuccMachi
import org.apache.uima.ducc.ws.Info;
import org.apache.uima.ducc.ws.JobInfo;
import org.apache.uima.ducc.ws.MachineInfo;
+import org.apache.uima.ducc.ws.PagingObserver;
import org.apache.uima.ducc.ws.registry.IServicesRegistry;
import org.apache.uima.ducc.ws.registry.ServicesRegistry;
import org.apache.uima.ducc.ws.registry.ServicesRegistryMap;
@@ -77,6 +78,8 @@ public class DuccHandlerLegacy extends D
private static Messages messages = Messages.getInstance();
private static DuccId jobid = null;
+ private static PagingObserver pagingObserver = PagingObserver.getInstance();
+
public final String legacyJobs = duccContextLegacy+"-jobs-data";
public final String legacyReservations = duccContextLegacy+"-reservations-data";
public final String legacyServices = duccContextLegacy+"-services-data";
@@ -105,25 +108,19 @@ public class DuccHandlerLegacy extends D
if(job.isOperational()) {
boolean multi = false;
sb.append("<td valign=\"bottom\">");
- ArrayList<String> swappingMachines = getSwappingMachines(job);
- if(!swappingMachines.isEmpty()) {
- StringBuffer mb = new StringBuffer();
- for(String machine : swappingMachines) {
- mb.append(machine);
- mb.append(" ");
- }
- String ml = mb.toString().trim();
- if(multi) {
- sb.append(" ");
- }
+ if(pagingObserver.isPaging(job)) {
multi = true;
- sb.append("<span class=\"health_red\" title=\""+ml+"\">");
- sb.append("Swapping");
+ String title = "a page-in operation occurred for at least one process during the past "+PagingObserver.intervalInSeconds+" seconds";
+ sb.append("<span class=\"health_red\" title=\""+title+"\">");
+ sb.append("Paging");
sb.append("</span>");
}
//
String monitor = getMonitor(duccId, type, multi);
if(monitor.length() > 0) {
+ if(multi) {
+ sb.append(" ");
+ }
multi = true;
sb.append(monitor);
}