You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2013/03/06 12:29:26 UTC
svn commit: r1453298 - in
/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws:
DuccBoot.java DuccData.java Info.java server/DuccWebMonitor.java
Author: degenaro
Date: Wed Mar 6 11:29:25 2013
New Revision: 1453298
URL: http://svn.apache.org/r1453298
Log:
UIMA-2691 DUCC webserver (WS) support for "managed reservations", fix improper default sorting of Reservations page (again!)
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/Info.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebMonitor.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1453298&r1=1453297&r2=1453298&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Wed Mar 6 11:29:25 2013
@@ -43,9 +43,9 @@ public class DuccBoot extends Thread {
private static DuccId jobid = null;
- private static long maxJobs = 4096;
- private static long maxReservations = 4096;
- private static long maxServices = 4096;
+ public static long maxJobs = 4096;
+ public static long maxReservations = 4096;
+ public static long maxServices = 4096;
public static void boot(CommonConfiguration commonConfiguration) {
DuccBoot duccBoot = new DuccBoot();
@@ -202,5 +202,6 @@ public class DuccBoot extends Thread {
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
restoreServices(hpm, duccData);
+ duccData.report();
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java?rev=1453298&r1=1453297&r2=1453298&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java Wed Mar 6 11:29:25 2013
@@ -28,6 +28,7 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
import org.apache.uima.ducc.transport.event.common.DuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkService.ServiceDeploymentType;
import org.apache.uima.ducc.transport.event.common.history.HistoryPersistenceManager;
@@ -36,21 +37,27 @@ import org.apache.uima.ducc.transport.ev
public class DuccData {
-
private static DuccLogger logger = DuccLoggerComponents.getWsLogger(DuccData.class.getName());
+ private static DuccId jobid = null;
private static DuccWorkMap duccWorkMap = new DuccWorkMap();
private static DuccWorkMap duccWorkLive = new DuccWorkMap();
private static ConcurrentSkipListMap<JobInfo,JobInfo> sortedJobs = new ConcurrentSkipListMap<JobInfo,JobInfo>();
+ private static ConcurrentSkipListMap<DuccId,JobInfo> keyMapJobs = new ConcurrentSkipListMap<DuccId,JobInfo>();
+
private static ConcurrentSkipListMap<ReservationInfo,ReservationInfo> sortedReservations = new ConcurrentSkipListMap<ReservationInfo,ReservationInfo>();
+ private static ConcurrentSkipListMap<DuccId,ReservationInfo> keyMapReservations = new ConcurrentSkipListMap<DuccId,ReservationInfo>();
+
private static ConcurrentSkipListMap<JobInfo,JobInfo> sortedServices = new ConcurrentSkipListMap<JobInfo,JobInfo>();
-
+ private static ConcurrentSkipListMap<DuccId,JobInfo> keyMapServices = new ConcurrentSkipListMap<DuccId,JobInfo>();
+
private static ConcurrentSkipListMap<Info,Info> sortedCombinedReservations = new ConcurrentSkipListMap<Info,Info>();
-
+ private static ConcurrentSkipListMap<DuccId,Info> keyMapCombinedReservations = new ConcurrentSkipListMap<DuccId,Info>();
+
private static DuccData duccData = new DuccData();
- private static DuccId jobid = null;
+ private static long slack = 100;
public static DuccData getInstance() {
return duccData;
@@ -118,8 +125,115 @@ public class DuccData {
updateSortedMaps(duccWork);
}
}
+ prune();
setPublished();
}
+
+ private int pruneJobs() {
+ String location = "pruneJobs";
+ int pruned = 0;
+ if(sortedJobs.size() > (DuccBoot.maxJobs + slack)) {
+ int count = 0;
+ for(JobInfo jobInfo : sortedJobs.keySet()) {
+ if(!jobInfo.isOperational()) {
+ count++;
+ if(count > DuccBoot.maxJobs) {
+ DuccId duccId = jobInfo.getJob().getDuccId();
+ sortedJobs.remove(jobInfo);
+ keyMapJobs.remove(duccId);
+ logger.info(location, duccId, "size: "+sortedJobs.size());
+ pruned++;
+ }
+ }
+ }
+ logger.debug(location, jobid, "pruned: "+pruned);
+ }
+ return pruned;
+ }
+
+ private int pruneReservations() {
+ String location = "pruneReservations";
+ int pruned = 0;
+ if(sortedReservations.size() > (DuccBoot.maxReservations + slack)) {
+ int count = 0;
+ for(ReservationInfo reservationInfo : sortedReservations.keySet()) {
+ if(!reservationInfo.isOperational()) {
+ count++;
+ if(count > DuccBoot.maxReservations) {
+ DuccId duccId = reservationInfo.getReservation().getDuccId();
+ sortedReservations.remove(reservationInfo);
+ keyMapReservations.remove(duccId);
+ logger.info(location, duccId, "size: "+sortedReservations.size());
+ pruned++;
+ }
+ }
+ }
+ logger.debug(location, jobid, "pruned: "+pruned);
+ }
+ return pruned;
+ }
+
+ private int pruneServices() {
+ String location = "pruneServices";
+ int pruned = 0;
+ if(sortedServices.size() > (DuccBoot.maxServices + slack)) {
+ int count = 0;
+ for(JobInfo jobInfo : sortedServices.keySet()) {
+ if(!jobInfo.isOperational()) {
+ count++;
+ if(count > DuccBoot.maxServices) {
+ DuccId duccId = jobInfo.getJob().getDuccId();
+ sortedServices.remove(jobInfo);
+ keyMapServices.remove(duccId);
+ logger.info(location, duccId, "size: "+sortedServices.size());
+ pruned++;
+ }
+ }
+ }
+ logger.debug(location, jobid, "pruned: "+pruned);
+ }
+ return pruned;
+ }
+
+ private int pruneCombinedReservations() {
+ String location = "pruneCombinedReservations";
+ int pruned = 0;
+ if(sortedCombinedReservations.size() > (DuccBoot.maxReservations + slack)) {
+ int count = 0;
+ for(Info info : sortedCombinedReservations.keySet()) {
+ if(!info.isOperational()) {
+ count++;
+ if(count > DuccBoot.maxReservations) {
+ DuccId duccId = info.getDuccWork().getDuccId();
+ sortedCombinedReservations.remove(info);
+ keyMapCombinedReservations.remove(duccId);
+ logger.info(location, duccId, "size: "+sortedCombinedReservations.size());
+ pruned++;
+ }
+ }
+ }
+ logger.debug(location, jobid, "pruned: "+pruned);
+ }
+ return pruned;
+ }
+
+ public void report() {
+ String location = "report";
+ int jc = sortedJobs.size();
+ int rc = sortedReservations.size();
+ int sc = sortedServices.size();
+ int cc = sortedCombinedReservations.size();
+ logger.info(location, jobid, ""+jc+":"+rc+":"+sc+":"+cc);
+ }
+
+ private void prune() {
+ String location = "prune";
+ int jc = pruneJobs();
+ int rc = pruneReservations();
+ int sc = pruneServices();
+ int cc = pruneCombinedReservations();
+ logger.debug(location, jobid, ""+jc+":"+rc+":"+sc+":"+cc);
+ }
public DuccWorkMap get() {
return duccWorkMap;
@@ -129,64 +243,104 @@ public class DuccData {
return duccWorkLive;
}
- private void cacheManager(IDuccWork duccWork, Object cacheKey) {
- switch(duccWork.getDuccType()) {
- case Job:
- sortedJobs.remove(cacheKey);
- break;
- case Reservation:
- if(cacheKey instanceof ReservationInfo) {
- sortedReservations.remove(cacheKey); }
- else if(cacheKey instanceof Info)
- sortedCombinedReservations.remove(cacheKey);
-
- break;
- case Service:
- sortedServices.remove(cacheKey);
- break;
+ private void updateJobs(IDuccWork duccWork) {
+ String location = "updateJobs";
+ DuccId duccId = duccWork.getDuccId();
+ DuccWorkJob job = (DuccWorkJob)duccWork;
+ if(keyMapJobs.containsKey(duccId)) {
+ sortedJobs.remove(keyMapJobs.get(duccId));
+ keyMapJobs.remove(duccId);
+ }
+ JobInfo jobInfo = new JobInfo(job);
+ sortedJobs.put(jobInfo, jobInfo);
+ if(!duccWork.isCompleted()) {
+ keyMapJobs.put(duccId, jobInfo);
+ logger.debug(location, duccId, "put job");
+ }
+ }
+
+ private void updateReservations(IDuccWork duccWork) {
+ String location = "updateReservations";
+ DuccId duccId = duccWork.getDuccId();
+ DuccWorkReservation reservation = (DuccWorkReservation)duccWork;
+ if(keyMapReservations.containsKey(duccId)) {
+ sortedReservations.remove(keyMapReservations.get(duccId));
+ keyMapReservations.remove(duccId);
+ }
+ ReservationInfo reservationInfo = new ReservationInfo(reservation);
+ sortedReservations.put(reservationInfo, reservationInfo);
+ if(!duccWork.isCompleted()) {
+ keyMapReservations.put(duccId, reservationInfo);
+ logger.debug(location, duccId, "put reservation");
+ }
+ //
+ if(keyMapCombinedReservations.containsKey(duccId)) {
+ sortedCombinedReservations.remove(keyMapCombinedReservations.get(duccId));
+ keyMapCombinedReservations.remove(duccId);
+ }
+ Info rInfo = new Info(reservation);
+ sortedCombinedReservations.put(rInfo, rInfo);
+ if(!duccWork.isCompleted()) {
+ keyMapCombinedReservations.put(duccId, rInfo);
+ logger.debug(location, duccId, "put combined");
+ }
+ }
+
+ private void updateServices(IDuccWork duccWork) {
+ String location = "updateServices";
+ DuccId duccId = duccWork.getDuccId();
+ DuccWorkJob service = (DuccWorkJob)duccWork;
+ if(keyMapServices.containsKey(duccId)) {
+ sortedServices.remove(keyMapServices.get(duccId));
+ keyMapServices.remove(duccId);
+ }
+ JobInfo serviceInfo = new JobInfo(service);
+ sortedServices.put(serviceInfo, serviceInfo);
+ if(!duccWork.isCompleted()) {
+ keyMapServices.put(duccId, serviceInfo);
+ logger.debug(location, duccId, "put service");
+ }
+ //
+ ServiceDeploymentType sdt = service.getServiceDeploymentType();
+ if(sdt != null) {
+ switch(sdt) {
+ case other:
+ if(keyMapCombinedReservations.containsKey(duccId)) {
+ sortedCombinedReservations.remove(keyMapCombinedReservations.get(duccId));
+ keyMapCombinedReservations.remove(duccId);
+ }
+ Info sInfo = new Info(service);
+ sortedCombinedReservations.put(sInfo, sInfo);
+ if(!duccWork.isCompleted()) {
+ keyMapCombinedReservations.put(duccId, sInfo);
+ logger.debug(location, duccId, "put combined");
+ }
+ break;
+ default:
+ break;
+ }
}
}
private void updateSortedMaps(IDuccWork duccWork) {
- String location = "updateSortedMaps";
- Info info;
- switch(duccWork.getDuccType()) {
- case Job:
- DuccWorkJob job = (DuccWorkJob)duccWork;
- JobInfo jobInfo = new JobInfo(job);
- cacheManager(job, jobInfo);
- sortedJobs.put(jobInfo, jobInfo);
- break;
- case Reservation:
- DuccWorkReservation reservation = (DuccWorkReservation)duccWork;
- ReservationInfo reservationInfo = new ReservationInfo(reservation);
- cacheManager(reservation, reservationInfo);
- sortedReservations.put(reservationInfo, reservationInfo);
- info= new Info(reservation);
- cacheManager(reservation, info);
- sortedCombinedReservations.put(info, info);
- break;
- case Service:
- DuccWorkJob service = (DuccWorkJob)duccWork;
- JobInfo serviceInfo = new JobInfo(service);
- cacheManager(service, serviceInfo);
- sortedServices.put(serviceInfo, serviceInfo);
- ServiceDeploymentType sdt = service.getServiceDeploymentType();
- if(sdt != null) {
- switch(sdt) {
- case other:
- info = new Info(service);
- sortedCombinedReservations.put(info, info);
+ if(duccWork != null) {
+ DuccType duccType = duccWork.getDuccType();
+ if(duccType != null) {
+ switch(duccWork.getDuccType()) {
+ case Job:
+ updateJobs(duccWork);
+ break;
+ case Reservation:
+ updateReservations(duccWork);
+ break;
+ case Service:
+ updateServices(duccWork);
break;
default:
break;
}
}
}
- logger.trace(location, jobid, "sortedJobs:"+sortedJobs.size()+" ");
- logger.trace(location, jobid, "sortedReservations:"+sortedReservations.size()+" ");
- logger.trace(location, jobid, "sortedServices:"+sortedServices.size()+" ");
- logger.trace(location, jobid, "sortedCombinedReservations:"+sortedCombinedReservations.size()+" ");
}
public ConcurrentSkipListMap<JobInfo,JobInfo> getSortedJobs() {
@@ -197,14 +351,14 @@ public class DuccData {
return sortedReservations;
}
- public ConcurrentSkipListMap<Info,Info> getSortedCombinedReservations() {
- return sortedCombinedReservations;
- }
-
public ConcurrentSkipListMap<JobInfo,JobInfo> getSortedServices() {
return sortedServices;
}
+ public ConcurrentSkipListMap<Info,Info> getSortedCombinedReservations() {
+ return sortedCombinedReservations;
+ }
+
public boolean isLive(DuccId duccId) {
return duccWorkLive.containsKey(duccId);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/Info.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/Info.java?rev=1453298&r1=1453297&r2=1453298&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/Info.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/Info.java Wed Mar 6 11:29:25 2013
@@ -20,6 +20,7 @@ package org.apache.uima.ducc.ws;
import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
import org.apache.uima.ducc.transport.event.common.DuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
import org.apache.uima.ducc.transport.event.common.IDuccWork;
public class Info implements Comparable<Info> {
@@ -90,7 +91,11 @@ public class Info implements Comparable<
IDuccWork dw2 = (IDuccWork)object;
String s1 = dw1.getDuccId().toString();
String s2 = dw2.getDuccId().toString();
- retVal = s1.equals(s2);
+ DuccType dt1 = dw1.getDuccType();
+ DuccType dt2 = dw2.getDuccType();
+ if(dt1 == dt2) {
+ retVal = s1.equals(s2);
+ }
}
catch(Throwable t) {
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebMonitor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebMonitor.java?rev=1453298&r1=1453297&r2=1453298&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebMonitor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccWebMonitor.java Wed Mar 6 11:29:25 2013
@@ -406,26 +406,26 @@ public class DuccWebMonitor implements I
ArrayList<String> stateSequence = monitorInfo.stateSequence;
if(stateSequence != null) {
if(stateSequence.contains(JobState.Completing.toString())) {
- duccLogger.info(location, duccId, "state: <uncancelable> "+stateSequence);
+ duccLogger.debug(location, duccId, "state: <uncancelable> "+stateSequence);
}
else if(stateSequence.contains(JobState.Completed.toString())) {
- duccLogger.info(location, duccId, "state: <uncancelable> "+stateSequence);
+ duccLogger.debug(location, duccId, "state: <uncancelable> "+stateSequence);
}
else {
- duccLogger.info(location, duccId, "state: <cancelable> "+stateSequence);
+ duccLogger.debug(location, duccId, "state: <cancelable> "+stateSequence);
retVal = true;
}
}
else {
- duccLogger.info(location, duccId, "stateSequence: <null>");
+ duccLogger.warn(location, duccId, "stateSequence: <null>");
}
}
else {
- duccLogger.info(location, duccId, "monitorInfo: <null>");
+ duccLogger.warn(location, duccId, "monitorInfo: <null>");
}
}
else {
- duccLogger.info(location, duccId, "already canceled");
+ duccLogger.debug(location, duccId, "already canceled");
}
duccLogger.trace(location, duccId, "exit");
return retVal;