You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2015/09/15 16:31:25 UTC
svn commit: r1703203 [5/5] - in /uima/sandbox/uima-ducc/trunk: ./
src/main/admin/ src/main/assembly/ src/main/resources/
src/main/resources/private/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/
uima-ducc-common/src/main/java/org/apache/u...
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/IDuccPerWorkItemStatistics.java Tue Sep 15 14:31:24 2015
@@ -26,4 +26,5 @@ public interface IDuccPerWorkItemStatist
public double getMin();
public double getMean();
public double getStandardDeviation();
+ public void setStandardDeviation(double s);
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryFactory.java Tue Sep 15 14:31:24 2015
@@ -18,15 +18,62 @@
*/
package org.apache.uima.ducc.transport.event.common.history;
+import org.apache.uima.ducc.common.main.DuccService;
+import org.apache.uima.ducc.common.persistence.services.IStateServices;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+
public class HistoryFactory
{
- private static IHistoryPersistenceManager instance = new HistoryPersistenceManager();
+ private static IHistoryPersistenceManager instance = null; //new HistoryPersistenceManager();
- public static IHistoryPersistenceManager getInstance() {
- return instance;
+ public static IHistoryPersistenceManager getInstance(String callerClass)
+ {
+ if ( instance != null ) return instance;
+
+ String methodName = "getInstance";
+
+ // log4j logging annoyance. We require the caller to give us its base package so
+ // we can configure a logger that writes to the right appender
+ int ndx = callerClass.lastIndexOf(".");
+ String stem = callerClass.substring(0, ndx);
+
+ String clname = System.getProperty("ducc.job.history.impl");
+
+ if ( clname == null ) {
+ DuccLogger logger = DuccService.getDuccLogger();
+ logger.error(methodName, null, "Job history class is not configured.");
+ instance = new NullHistoryManager();
+ return instance;
+ }
+
+ ndx = clname.lastIndexOf(".");
+ String clfile = clname.substring(ndx+1);
+ //
+ // We try to construct the persistence object. If it fails, we return a
+ // "null" object conforming to the interface but doing nothing to hopefully
+ // reduce NPEs.
+ //
+ DuccLogger logger = DuccLogger.getLogger(stem + "." + clfile, "DB"); // get the component logger
+
+ //
+ // We try to construct the persistence object. If it fails, we return a
+ // "null" object conforming to the interface but doing nothing to hopefully
+ // reduce NPEs.
+ //
+
+ try {
+ @SuppressWarnings("unchecked")
+ Class<IStateServices> iss = (Class<IStateServices>) Class.forName(clname);
+ instance = (IHistoryPersistenceManager) iss.newInstance();
+ instance.setLogger(logger);
+ } catch ( Throwable t ) {
+ logger.error(methodName, null, "Cannot instantiate service persistence class", clname, ":", t);
+ instance = new NullHistoryManager();
+ }
+ return instance;
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java Tue Sep 15 14:31:24 2015
@@ -25,7 +25,10 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.ListIterator;
+import java.util.Map;
import org.apache.uima.ducc.common.IDuccEnv;
import org.apache.uima.ducc.common.main.DuccService;
@@ -33,6 +36,7 @@ import org.apache.uima.ducc.common.utils
import org.apache.uima.ducc.common.utils.IOHelper;
import org.apache.uima.ducc.common.utils.Utils;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
@@ -40,353 +44,551 @@ import org.apache.uima.ducc.transport.ev
public class HistoryPersistenceManager implements IHistoryPersistenceManager {
-
- // private static final DuccLogger logger = DuccLoggerComponents.getTrLogger(HistoryPersistenceManager.class.getName());
- private static final DuccLogger logger = DuccService.getDuccLogger(HistoryPersistenceManager.class.getName());
-
- private String historyDirectory_jobs = IDuccEnv.DUCC_HISTORY_JOBS_DIR;
- private String historyDirectory_reservations = IDuccEnv.DUCC_HISTORY_RESERVATIONS_DIR;
- private String historyDirectory_services = IDuccEnv.DUCC_HISTORY_SERVICES_DIR;
-
- private String dwj = "dwj";
- private String dwr = "dwr";
- private String dws = "dws";
-
- private enum Verbosity {
- QUIET,
- SPEAK,
- }
-
- HistoryPersistenceManager() {
- mkdirs();
- }
-
- private void mkdirs() {
- IOHelper.mkdirs(historyDirectory_jobs);
- IOHelper.mkdirs(historyDirectory_reservations);
- IOHelper.mkdirs(historyDirectory_services);
- }
-
- private String normalize(String id) {
- String retVal = id;
- return retVal;
- }
-
-
- public void jobSaveConditional(IDuccWorkJob duccWorkJob) throws IOException {
- String id = normalize(""+duccWorkJob.getDuccId().getFriendly());
- String fileName = historyDirectory_jobs+File.separator+id+"."+dwj;
- File file = new File(fileName);
- if(!file.exists()) {
- jobSave(duccWorkJob);
- }
- }
-
-
- public void jobSave(IDuccWorkJob duccWorkJob) throws IOException {
- String id = normalize(""+duccWorkJob.getDuccId().getFriendly());
- String fileName = historyDirectory_jobs+File.separator+id+"."+dwj;
- FileOutputStream fos = null;
- ObjectOutputStream out = null;
- fos = new FileOutputStream(fileName);
- out = new ObjectOutputStream(fos);
- out.writeObject(duccWorkJob);
- out.close();
- }
-
- public IDuccWorkJob jobRestore(String fileName) {
- return jobRestore(fileName, Verbosity.SPEAK);
- }
-
- private IDuccWorkJob jobRestore(String fileName, Verbosity level) {
- String methodName = "jobRestore";
- IDuccWorkJob job = null;
+
+ // private static final DuccLogger logger = DuccLoggerComponents.getTrLogger(HistoryPersistenceManager.class.getName());
+ private DuccLogger logger = DuccService.getDuccLogger(HistoryPersistenceManager.class.getName());
+
+ private String historyDirectory_jobs = IDuccEnv.DUCC_HISTORY_JOBS_DIR;
+ private String historyDirectory_reservations = IDuccEnv.DUCC_HISTORY_RESERVATIONS_DIR;
+ private String historyDirectory_services = IDuccEnv.DUCC_HISTORY_SERVICES_DIR;
+
+ private String dwj = "dwj";
+ private String dwr = "dwr";
+ private String dws = "dws";
+
+ private enum Verbosity {
+ QUIET,
+ SPEAK,
+ }
+
+ HistoryPersistenceManager() {
+ mkdirs();
+ }
+
+ public void setLogger(DuccLogger logger)
+ {
+ this.logger = logger;
+ }
+
+ private void mkdirs() {
+ IOHelper.mkdirs(historyDirectory_jobs);
+ IOHelper.mkdirs(historyDirectory_reservations);
+ IOHelper.mkdirs(historyDirectory_services);
+ }
+
+ private String normalize(String id) {
+ String retVal = id;
+ return retVal;
+ }
+
+ public void saveJob(IDuccWorkJob duccWorkJob) throws IOException {
+ String id = normalize(""+duccWorkJob.getDuccId().getFriendly());
+ String fileName = historyDirectory_jobs+File.separator+id+"."+dwj;
+ FileOutputStream fos = null;
+ ObjectOutputStream out = null;
+ fos = new FileOutputStream(fileName);
+ out = new ObjectOutputStream(fos);
+ out.writeObject(duccWorkJob);
+ out.close();
+ }
+
+ public List<IDuccWorkJob> restoreJobs(long max)
+ throws Exception
+ {
+ String methodName = "restoreJobs";
+ // Find the 'max' most current jobs.
+ // How: list all files; if ends with 'dwj' the first part is the duccid.
+ // put all the duccids into an array and sort it, (ascending so we don't need a comparator)
+ // Starting at the end, call restoreJob(Long) to get the file.
+ //
+ List<Long> allJobs = new ArrayList<Long>();
+ File folder = new File(historyDirectory_jobs);
+ File[] listOfFiles = folder.listFiles();
+ if(listOfFiles != null) {
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ String name = listOfFiles[i].getName();
+ if(name.endsWith("."+dwj)) {
+ int ndx = name.indexOf(".");
+ try {
+ allJobs.add(Long.parseLong(name.substring(0, ndx)));
+ } catch ( NumberFormatException e ) {
+ logger.warn(methodName, null, "Job file", name, ": cannot extract ducc id. Not restored.");
+ }
+
+ }
+ }
+ }
+ }
+
+ max = Math.min(allJobs.size(), max);
+ Collections.sort(allJobs);
+ List<IDuccWorkJob> ret = new ArrayList<IDuccWorkJob>((int)max);
+ for ( int i = allJobs.size()-1; i >= 0; i-- ) {
+ ret.add(restoreJob(allJobs.get(i)));
+ }
+
+ return ret;
+ }
+
+ public IDuccWorkJob restoreJob(long duccid)
+ throws Exception
+ {
+ //String methodName = "jobRestore";
+ IDuccWorkJob job = null;
+ String fileName = ""+duccid + "." + dwj;
+ FileInputStream fis = null;
+ ObjectInputStream in = null;
+ fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName);
+ in = new ObjectInputStream(fis);
+ job = (IDuccWorkJob) in.readObject();
+ in.close();
+ return job;
+ }
+
+ // public IDuccWorkJob jobRestore(String fileName) {
+ // return jobRestore(fileName, Verbosity.SPEAK);
+ // }
+
+ // private IDuccWorkJob jobRestore(String fileName, Verbosity level) {
+ // String methodName = "jobRestore";
+ // IDuccWorkJob job = null;
+ // try {
+ // logger.trace(methodName, null, "restore:"+fileName);
+ // FileInputStream fis = null;
+ // ObjectInputStream in = null;
+ // fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName);
+ // in = new ObjectInputStream(fis);
+ // job = (IDuccWorkJob) in.readObject();
+ // in.close();
+ // }
+ // catch(Exception e) {
+ // switch(level) {
+ // case QUIET:
+ // break;
+ // case SPEAK:
+ // logger.warn(methodName, null, "unable to restore:"+fileName, e);
+ // break;
+ // }
+ // }
+ // return job;
+ // }
+
+
+ // public IDuccWorkJob jobRestore(DuccId duccId) {
+ // String fileName = duccId.getFriendly()+"."+dwj;
+ // return jobRestore(fileName, Verbosity.QUIET);
+ // }
+
+
+ // public ArrayList<Long> jobList() { return null; }
+// public ArrayList<String> jobList() {
+// ArrayList<String> retVal = new ArrayList<String>();
+// File folder = new File(historyDirectory_jobs);
+// File[] listOfFiles = folder.listFiles();
+// if(listOfFiles != null) {
+// for (int i = 0; i < listOfFiles.length; i++) {
+// if (listOfFiles[i].isFile()) {
+// String name = listOfFiles[i].getName();
+// if(name.endsWith("."+dwj)) {
+// retVal.add(name);
+// }
+// }
+// }
+// }
+// return retVal;
+// }
+
+
+ // public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException {
+ // ArrayList<IDuccWorkJob> retVal = new ArrayList<IDuccWorkJob>();
+ // ArrayList<String> jobFileNames = jobList();
+ // ListIterator<String> listIterator = jobFileNames.listIterator();
+ // while(listIterator.hasNext()) {
+ // String fileName = listIterator.next();
+ // IDuccWorkJob job = jobRestore(fileName);
+ // if(job != null) {
+ // retVal.add(job);
+ // }
+ // }
+ // return retVal;
+ // }
+
+ // public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException
+ // {
+ // return null;
+ // }
+
+
+// public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException {
+// String id = normalize(""+duccWorkReservation.getDuccId().getFriendly());
+// String fileName = historyDirectory_jobs+File.separator+id+"."+dwr;
+// File file = new File(fileName);
+// if(!file.exists()) {
+// reservationSave(duccWorkReservation);
+// }
+// }
+
+
+ public void saveReservation(IDuccWorkReservation duccWorkReservation)
+ throws Exception
+ {
+ String id = normalize(""+duccWorkReservation.getDuccId().getFriendly());
+ String fileName = historyDirectory_reservations+File.separator+id+"."+dwr;
+ FileOutputStream fos = null;
+ ObjectOutputStream out = null;
+ fos = new FileOutputStream(fileName);
+ out = new ObjectOutputStream(fos);
+ out.writeObject(duccWorkReservation);
+ out.close();
+ }
+
+
+ public IDuccWorkReservation restoreReservation(long duccid)
+ throws Exception
+ {
+ //String methodName = "reservationRestore";
+ IDuccWorkReservation reservation = null;
+ FileInputStream fis = null;
+ ObjectInputStream in = null;
+ String fileName = ""+duccid + "." + dwr;
+
+ fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName);
+ in = new ObjectInputStream(fis);
+ reservation = (IDuccWorkReservation) in.readObject();
+ in.close();
+ return reservation;
+ }
+
+ public List<IDuccWorkReservation> restoreReservations(long max)
+ throws Exception
+ {
+ String methodName = "restoreReservations";
+ // Find the 'max' most current jobs.
+ // How: list all files; if ends with 'dwj' the first part is the duccid.
+ // put all the duccids into an array and sort it, (ascending so we don't need a comparator)
+ // Starting at the end, call restoreJob(Long) to get the file.
+ //
+ List<Long> allReservations = new ArrayList<Long>();
+ File folder = new File(historyDirectory_reservations);
+ File[] listOfFiles = folder.listFiles();
+ if(listOfFiles != null) {
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ String name = listOfFiles[i].getName();
+ if(name.endsWith("."+dwr)) {
+ int ndx = name.indexOf(".");
+ try {
+ allReservations.add(Long.parseLong(name.substring(0, ndx)));
+ } catch ( NumberFormatException e ) {
+ logger.warn(methodName, null, "Reservation file", name, ": cannot extract ducc id. Not restored.");
+ }
+
+ }
+ }
+ }
+ }
+
+ max = Math.min(allReservations.size(), max);
+ Collections.sort(allReservations);
+ List<IDuccWorkReservation> ret = new ArrayList<IDuccWorkReservation>((int)max);
+ for ( int i = allReservations.size()-1; i >= 0; i-- ) {
+ ret.add(restoreReservation(allReservations.get(i)));
+ }
+
+ return ret;
+ }
+
+
+ // private IDuccWorkReservation reservationRestore(String fileName, Verbosity level) {
+ // String methodName = "reservationRestore";
+ // IDuccWorkReservation reservation = null;
+ // try {
+ // logger.trace(methodName, null, "restore:"+fileName);
+ // FileInputStream fis = null;
+ // ObjectInputStream in = null;
+ // fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName);
+ // in = new ObjectInputStream(fis);
+ // reservation = (IDuccWorkReservation) in.readObject();
+ // in.close();
+ // }
+ // catch(Exception e) {
+ // switch(level) {
+ // case QUIET:
+ // break;
+ // case SPEAK:
+ // logger.warn(methodName, null, "unable to restore:"+fileName);
+ // break;
+ // }
+ // }
+ // return reservation;
+ // }
+
+
+ // public ArrayList<String> reservationList() {
+ // ArrayList<String> retVal = new ArrayList<String>();
+ // File folder = new File(historyDirectory_reservations);
+ // File[] listOfFiles = folder.listFiles();
+ // if(listOfFiles != null) {
+ // for (int i = 0; i < listOfFiles.length; i++) {
+ // if (listOfFiles[i].isFile()) {
+ // String name = listOfFiles[i].getName();
+ // if(name.endsWith("."+dwr)) {
+ // retVal.add(name);
+ // }
+ // }
+ // }
+ // }
+ // return retVal;
+ // }
+
+
+ // public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException {
+ // ArrayList<IDuccWorkReservation> retVal = new ArrayList<IDuccWorkReservation>();
+ // ArrayList<String> reservationFileNames = reservationList();
+ // ListIterator<String> listIterator = reservationFileNames.listIterator();
+ // while(listIterator.hasNext()) {
+ // String fileName = listIterator.next();
+ // IDuccWorkReservation reservation = reservationRestore(fileName);
+ // if(reservation != null) {
+ // retVal.add(reservation);
+ // }
+ // }
+ // return retVal;
+ // }
+
+
+ // public IDuccWorkReservation reservationRestore(DuccId duccId) {
+ // String fileName = duccId.getFriendly()+"."+dwr;
+ // return reservationRestore(fileName, Verbosity.QUIET);
+ // }
+
+
+ public void saveService(IDuccWorkService duccWorkService)
+ throws Exception
+ {
+ String id = normalize(""+duccWorkService.getDuccId().getFriendly());
+ String fileName = historyDirectory_services+File.separator+id+"."+dws;
+ FileOutputStream fos = null;
+ ObjectOutputStream out = null;
+ fos = new FileOutputStream(fileName);
+ out = new ObjectOutputStream(fos);
+ out.writeObject(duccWorkService);
+ out.close();
+ }
+
+
+ public IDuccWorkService restoreService(long duccid)
+ throws Exception
+ {
+ //String methodName = "restoreService";
+ IDuccWorkService service = null;
+ FileInputStream fis = null;
+ ObjectInputStream in = null;
+ String fileName = ""+duccid + "." + dws;
+
+ fis = new FileInputStream(historyDirectory_services+File.separator+fileName);
+ in = new ObjectInputStream(fis);
+ service = (IDuccWorkService) in.readObject();
+ in.close();
+ return service;
+ }
+
+ public List<IDuccWorkService> restoreServices(long max)
+ throws Exception
+ {
+ String methodName = "restoreServices";
+ // Find the 'max' most current jobs.
+ // How: list all files; if ends with 'dwj' the first part is the duccid.
+ // put all the duccids into an array and sort it, (ascending so we don't need a comparator)
+ // Starting at the end, call restoreJob(Long) to get the file.
+ //
+ List<Long> allServices = new ArrayList<Long>();
+ File folder = new File(historyDirectory_services);
+ File[] listOfFiles = folder.listFiles();
+ if(listOfFiles != null) {
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ String name = listOfFiles[i].getName();
+ if(name.endsWith("."+dws)) {
+ int ndx = name.indexOf(".");
+ try {
+ allServices.add(Long.parseLong(name.substring(0, ndx)));
+ } catch ( NumberFormatException e ) {
+ logger.warn(methodName, null, "Reservation file", name, ": cannot extract ducc id. Not restored.");
+ }
+
+ }
+ }
+ }
+ }
+
+ max = Math.min(allServices.size(), max);
+ Collections.sort(allServices);
+ List<IDuccWorkService> ret = new ArrayList<IDuccWorkService>((int)max);
+ for ( int i = allServices.size()-1; i >= 0; i-- ) {
+ ret.add(restoreService(allServices.get(i)));
+ }
+
+ return ret;
+ }
+
+ // public void serviceSaveConditional(IDuccWorkService duccWorkService)
+ // throws IOException {
+ // String id = normalize(""+duccWorkService.getDuccId().getFriendly());
+ // String fileName = historyDirectory_services+File.separator+id+"."+dws;
+ // File file = new File(fileName);
+ // if(!file.exists()) {
+ // serviceSave(duccWorkService);
+ // }
+ // }
+
+
+
+ public void serviceSave(IDuccWorkService duccWorkService)
+ throws IOException {
+ String id = normalize(""+duccWorkService.getDuccId().getFriendly());
+ String fileName = historyDirectory_services+File.separator+id+"."+dws;
+ FileOutputStream fos = null;
+ ObjectOutputStream out = null;
+ fos = new FileOutputStream(fileName);
+ out = new ObjectOutputStream(fos);
+ out.writeObject(duccWorkService);
+ out.close();
+ }
+
+
+ public IDuccWorkService serviceRestore(String fileName) {
+ return serviceRestore(fileName, Verbosity.SPEAK);
+ }
+
+ private IDuccWorkService serviceRestore(String fileName, Verbosity level) {
+ String methodName = "serviceRestore";
+ IDuccWorkService service = null;
+ try {
+ logger.trace(methodName, null, "restore:"+fileName);
+ FileInputStream fis = null;
+ ObjectInputStream in = null;
+ fis = new FileInputStream(historyDirectory_services+File.separator+fileName);
+ in = new ObjectInputStream(fis);
+ service = (IDuccWorkService) in.readObject();
+ in.close();
+ }
+ catch(Exception e) {
+ switch(level) {
+ case QUIET:
+ break;
+ case SPEAK:
+ logger.warn(methodName, null, "unable to restore:"+fileName);
+ break;
+ }
+ }
+ return service;
+ }
+
+
+ public ArrayList<String> serviceList() {
+ ArrayList<String> retVal = new ArrayList<String>();
+ File folder = new File(historyDirectory_services);
+ File[] listOfFiles = folder.listFiles();
+ if(listOfFiles != null) {
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ String name = listOfFiles[i].getName();
+ if(name.endsWith("."+dws)) {
+ retVal.add(name);
+ }
+ }
+ }
+ }
+ return retVal;
+ }
+
+
+ public ArrayList<IDuccWorkService> serviceRestore() throws IOException,
+ ClassNotFoundException {
+ ArrayList<IDuccWorkService> retVal = new ArrayList<IDuccWorkService>();
+ ArrayList<String> serviceFileNames = serviceList();
+ ListIterator<String> listIterator = serviceFileNames.listIterator();
+ while(listIterator.hasNext()) {
+ String fileName = listIterator.next();
+ IDuccWorkService service = serviceRestore(fileName);
+ if(service != null) {
+ retVal.add(service);
+ }
+ }
+ return retVal;
+ }
+
+
+ public IDuccWorkService serviceRestore(DuccId duccId) {
+ String fileName = duccId.getFriendly()+"."+dws;
+ return serviceRestore(fileName, Verbosity.QUIET);
+ }
+
+ public boolean checkpoint(DuccWorkMap m, Map<DuccId, DuccId> processToJob)
+ throws Exception
+ {
+ return false;
+ }
+
+ public boolean restore(DuccWorkMap m, Map<DuccId, DuccId> processToJob)
+ throws Exception
+ {
+ return false;
+ }
+
+ private static int doJobs(HistoryPersistenceManager hpm)
+ throws Exception
+ {
+ List<IDuccWorkJob> duccWorkJobs = hpm.restoreJobs(-1L);
+ ListIterator<IDuccWorkJob> listIterator = duccWorkJobs.listIterator();
+ int acc = 0;
+ while(listIterator.hasNext()) {
+ IDuccWorkJob duccWorkJob = listIterator.next();
+ System.out.println(duccWorkJob.getId());
+ acc++;
+ }
+ return acc;
+ }
+
+ private static int doReservations(HistoryPersistenceManager hpm)
+ throws Exception
+ {
+ List<IDuccWorkReservation> duccWorkReservations = hpm.restoreReservations(-1L);
+ ListIterator<IDuccWorkReservation> listIterator = duccWorkReservations.listIterator();
+ int acc = 0;
+ while(listIterator.hasNext()) {
+ IDuccWorkReservation duccWorkReservation = listIterator.next();
+ System.out.println(duccWorkReservation.getId());
+ acc++;
+ }
+ return acc;
+ }
+
+ public static void main(String[] args) throws IOException, ClassNotFoundException {
+ String ducc_home = Utils.findDuccHome();
+ if(ducc_home == null) {
+ System.out.println("DUCC_HOME not set in environment");
+ return;
+ }
+ if(ducc_home.trim() == "") {
+ System.out.println("DUCC_HOME not set in environment");
+ return;
+ }
+ int reservations = 0;
try {
- logger.trace(methodName, null, "restore:"+fileName);
- FileInputStream fis = null;
- ObjectInputStream in = null;
- fis = new FileInputStream(historyDirectory_jobs+File.separator+fileName);
- in = new ObjectInputStream(fis);
- job = (IDuccWorkJob) in.readObject();
- in.close();
- }
- catch(Exception e) {
- switch(level) {
- case QUIET:
- break;
- case SPEAK:
- logger.warn(methodName, null, "unable to restore:"+fileName, e);
- break;
- }
- }
- return job;
- }
-
-
- public IDuccWorkJob jobRestore(DuccId duccId) {
- String fileName = duccId.getFriendly()+"."+dwj;
- return jobRestore(fileName, Verbosity.QUIET);
- }
-
-
- public ArrayList<String> jobList() {
- ArrayList<String> retVal = new ArrayList<String>();
- File folder = new File(historyDirectory_jobs);
- File[] listOfFiles = folder.listFiles();
- if(listOfFiles != null) {
- for (int i = 0; i < listOfFiles.length; i++) {
- if (listOfFiles[i].isFile()) {
- String name = listOfFiles[i].getName();
- if(name.endsWith("."+dwj)) {
- retVal.add(name);
- }
- }
- }
- }
- return retVal;
- }
-
-
- public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException {
- ArrayList<IDuccWorkJob> retVal = new ArrayList<IDuccWorkJob>();
- ArrayList<String> jobFileNames = jobList();
- ListIterator<String> listIterator = jobFileNames.listIterator();
- while(listIterator.hasNext()) {
- String fileName = listIterator.next();
- IDuccWorkJob job = jobRestore(fileName);
- if(job != null) {
- retVal.add(job);
- }
- }
- return retVal;
- }
-
-
- public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException {
- String id = normalize(""+duccWorkReservation.getDuccId().getFriendly());
- String fileName = historyDirectory_jobs+File.separator+id+"."+dwr;
- File file = new File(fileName);
- if(!file.exists()) {
- reservationSave(duccWorkReservation);
- }
- }
-
-
- public void reservationSave(IDuccWorkReservation duccWorkReservation) throws IOException {
- String id = normalize(""+duccWorkReservation.getDuccId().getFriendly());
- String fileName = historyDirectory_reservations+File.separator+id+"."+dwr;
- FileOutputStream fos = null;
- ObjectOutputStream out = null;
- fos = new FileOutputStream(fileName);
- out = new ObjectOutputStream(fos);
- out.writeObject(duccWorkReservation);
- out.close();
- }
-
-
- public IDuccWorkReservation reservationRestore(String fileName) {
- return reservationRestore(fileName, Verbosity.SPEAK);
- }
-
- private IDuccWorkReservation reservationRestore(String fileName, Verbosity level) {
- String methodName = "reservationRestore";
- IDuccWorkReservation reservation = null;
- try {
- logger.trace(methodName, null, "restore:"+fileName);
- FileInputStream fis = null;
- ObjectInputStream in = null;
- fis = new FileInputStream(historyDirectory_reservations+File.separator+fileName);
- in = new ObjectInputStream(fis);
- reservation = (IDuccWorkReservation) in.readObject();
- in.close();
- }
- catch(Exception e) {
- switch(level) {
- case QUIET:
- break;
- case SPEAK:
- logger.warn(methodName, null, "unable to restore:"+fileName);
- break;
- }
- }
- return reservation;
- }
-
-
- public ArrayList<String> reservationList() {
- ArrayList<String> retVal = new ArrayList<String>();
- File folder = new File(historyDirectory_reservations);
- File[] listOfFiles = folder.listFiles();
- if(listOfFiles != null) {
- for (int i = 0; i < listOfFiles.length; i++) {
- if (listOfFiles[i].isFile()) {
- String name = listOfFiles[i].getName();
- if(name.endsWith("."+dwr)) {
- retVal.add(name);
- }
- }
- }
- }
- return retVal;
- }
-
-
- public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException {
- ArrayList<IDuccWorkReservation> retVal = new ArrayList<IDuccWorkReservation>();
- ArrayList<String> reservationFileNames = reservationList();
- ListIterator<String> listIterator = reservationFileNames.listIterator();
- while(listIterator.hasNext()) {
- String fileName = listIterator.next();
- IDuccWorkReservation reservation = reservationRestore(fileName);
- if(reservation != null) {
- retVal.add(reservation);
- }
- }
- return retVal;
- }
-
-
- public IDuccWorkReservation reservationRestore(DuccId duccId) {
- String fileName = duccId.getFriendly()+"."+dwr;
- return reservationRestore(fileName, Verbosity.QUIET);
- }
-
-
- public void serviceSaveConditional(IDuccWorkService duccWorkService)
- throws IOException {
- String id = normalize(""+duccWorkService.getDuccId().getFriendly());
- String fileName = historyDirectory_services+File.separator+id+"."+dws;
- File file = new File(fileName);
- if(!file.exists()) {
- serviceSave(duccWorkService);
- }
- }
-
-
- public void serviceSave(IDuccWorkService duccWorkService)
- throws IOException {
- String id = normalize(""+duccWorkService.getDuccId().getFriendly());
- String fileName = historyDirectory_services+File.separator+id+"."+dws;
- FileOutputStream fos = null;
- ObjectOutputStream out = null;
- fos = new FileOutputStream(fileName);
- out = new ObjectOutputStream(fos);
- out.writeObject(duccWorkService);
- out.close();
- }
-
-
- public IDuccWorkService serviceRestore(String fileName) {
- return serviceRestore(fileName, Verbosity.SPEAK);
- }
-
- private IDuccWorkService serviceRestore(String fileName, Verbosity level) {
- String methodName = "serviceRestore";
- IDuccWorkService service = null;
- try {
- logger.trace(methodName, null, "restore:"+fileName);
- FileInputStream fis = null;
- ObjectInputStream in = null;
- fis = new FileInputStream(historyDirectory_services+File.separator+fileName);
- in = new ObjectInputStream(fis);
- service = (IDuccWorkService) in.readObject();
- in.close();
- }
- catch(Exception e) {
- switch(level) {
- case QUIET:
- break;
- case SPEAK:
- logger.warn(methodName, null, "unable to restore:"+fileName);
- break;
- }
- }
- return service;
- }
-
-
- public ArrayList<String> serviceList() {
- ArrayList<String> retVal = new ArrayList<String>();
- File folder = new File(historyDirectory_services);
- File[] listOfFiles = folder.listFiles();
- if(listOfFiles != null) {
- for (int i = 0; i < listOfFiles.length; i++) {
- if (listOfFiles[i].isFile()) {
- String name = listOfFiles[i].getName();
- if(name.endsWith("."+dws)) {
- retVal.add(name);
- }
- }
- }
- }
- return retVal;
- }
-
-
- public ArrayList<IDuccWorkService> serviceRestore() throws IOException,
- ClassNotFoundException {
- ArrayList<IDuccWorkService> retVal = new ArrayList<IDuccWorkService>();
- ArrayList<String> serviceFileNames = serviceList();
- ListIterator<String> listIterator = serviceFileNames.listIterator();
- while(listIterator.hasNext()) {
- String fileName = listIterator.next();
- IDuccWorkService service = serviceRestore(fileName);
- if(service != null) {
- retVal.add(service);
- }
- }
- return retVal;
- }
-
-
- public IDuccWorkService serviceRestore(DuccId duccId) {
- String fileName = duccId.getFriendly()+"."+dws;
- return serviceRestore(fileName, Verbosity.QUIET);
- }
-
- ///// <tool>
-
- private static int doJobs(HistoryPersistenceManager hpm) throws IOException, ClassNotFoundException {
- ArrayList<IDuccWorkJob> duccWorkJobs = hpm.jobRestore();
- ListIterator<IDuccWorkJob> listIterator = duccWorkJobs.listIterator();
- int acc = 0;
- while(listIterator.hasNext()) {
- IDuccWorkJob duccWorkJob = listIterator.next();
- System.out.println(duccWorkJob.getId());
- acc++;
- }
- return acc;
- }
-
- private static int doReservations(HistoryPersistenceManager hpm) throws IOException, ClassNotFoundException {
- ArrayList<IDuccWorkReservation> duccWorkReservations = hpm.reservationRestore();
- ListIterator<IDuccWorkReservation> listIterator = duccWorkReservations.listIterator();
- int acc = 0;
- while(listIterator.hasNext()) {
- IDuccWorkReservation duccWorkReservation = listIterator.next();
- System.out.println(duccWorkReservation.getId());
- acc++;
- }
- return acc;
- }
-
- public static void main(String[] args) throws IOException, ClassNotFoundException {
- String ducc_home = Utils.findDuccHome();
- if(ducc_home == null) {
- System.out.println("DUCC_HOME not set in environment");
- return;
- }
- if(ducc_home.trim() == "") {
- System.out.println("DUCC_HOME not set in environment");
- return;
+ HistoryPersistenceManager hpm = new HistoryPersistenceManager();
+ int jobs = doJobs(hpm);
+ System.out.println("jobs: "+jobs);
+ reservations = doReservations(hpm);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
}
- HistoryPersistenceManager hpm = new HistoryPersistenceManager();
- int jobs = doJobs(hpm);
- System.out.println("jobs: "+jobs);
- int reservations = doReservations(hpm);
- System.out.println("reservations: "+reservations);
- }
+ System.out.println("reservations: "+reservations);
+ }
- ///// </tool>
+ ///// </tool>
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java Tue Sep 15 14:31:24 2015
@@ -18,35 +18,39 @@
*/
package org.apache.uima.ducc.transport.event.common.history;
-import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
-public interface IHistoryPersistenceManager {
-
- public void jobSaveConditional(IDuccWorkJob duccWorkJob) throws IOException;
- public void jobSave(IDuccWorkJob duccWorkJob) throws IOException;
- public IDuccWorkJob jobRestore(String fileName);
- public IDuccWorkJob jobRestore(DuccId duccId);
- public ArrayList<String> jobList();
- public ArrayList<IDuccWorkJob> jobRestore() throws IOException, ClassNotFoundException;
-
- public void reservationSaveConditional(IDuccWorkReservation duccWorkReservation) throws IOException;
- public void reservationSave(IDuccWorkReservation duccWorkReservation) throws IOException;
- public IDuccWorkReservation reservationRestore(String fileName);
- public IDuccWorkReservation reservationRestore(DuccId duccId);
- public ArrayList<String> reservationList();
- public ArrayList<IDuccWorkReservation> reservationRestore() throws IOException, ClassNotFoundException;
+public interface IHistoryPersistenceManager
+{
+ public void saveJob(IDuccWorkJob duccWorkJob) throws Exception;
+ public IDuccWorkJob restoreJob(long friendly_id) throws Exception;
+ public List<IDuccWorkJob> restoreJobs(long max) throws Exception;
- public void serviceSaveConditional(IDuccWorkService duccWorkService) throws IOException;
- public void serviceSave(IDuccWorkService duccWorkService) throws IOException;
- public IDuccWorkService serviceRestore(String fileName);
- public IDuccWorkService serviceRestore(DuccId duccId);
- public ArrayList<String> serviceList();
- public ArrayList<IDuccWorkService> serviceRestore() throws IOException, ClassNotFoundException;
+ public void saveReservation(IDuccWorkReservation reservation) throws Exception;
+ public IDuccWorkReservation restoreReservation(long friendly_id) throws Exception;
+ public List<IDuccWorkReservation> restoreReservations(long max) throws Exception;
+
+ public void saveService(IDuccWorkService duccWorkService) throws Exception;
+ public IDuccWorkService restoreService(long friendly_id) throws Exception;
+ public List<IDuccWorkService> restoreServices(long max) throws Exception;
+
+ public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception;
+ public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception;
+
+ public void setLogger(DuccLogger logger);
+ //public void serviceSaveConditional(IDuccWorkService duccWorkService) throws Exception;
+ // public void serviceSave(IDuccWorkService duccWorkService) throws Exception;
+ //public IDuccWorkService serviceRestore(String fileName);
+ //public IDuccWorkService serviceRestore(DuccId duccId);
+ //public ArrayList<String> serviceList();
+ //public ArrayList<IDuccWorkService> serviceRestore() throws IOException, ClassNotFoundException;
}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java?rev=1703203&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java Tue Sep 15 14:31:24 2015
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+*/
+package org.apache.uima.ducc.transport.event.common.history;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
+
+
+public class NullHistoryManager
+ implements IHistoryPersistenceManager
+{
+ NullHistoryManager()
+ {
+ }
+
+ public void setLogger(DuccLogger logger) {}
+
+ public void saveJob(IDuccWorkJob duccWorkJob)
+ throws Exception
+ {
+ }
+
+ public IDuccWorkJob restoreJob(long duccid)
+ throws Exception
+ {
+ return null;
+ }
+
+ public List<IDuccWorkJob> restoreJobs(long max)
+ throws Exception
+ {
+ return new ArrayList<IDuccWorkJob>();
+ }
+
+
+ public void saveReservation(IDuccWorkReservation reservation)
+ throws Exception
+ {
+ }
+
+ public IDuccWorkReservation restoreReservation(long friendly_id)
+ throws Exception
+ {
+ return null;
+ }
+
+ public List<IDuccWorkReservation> restoreReservations(long max)
+ throws Exception
+ {
+ return new ArrayList<IDuccWorkReservation>();
+ }
+
+
+ public void saveService(IDuccWorkService service)
+ throws Exception
+ {
+ }
+
+
+ public void serviceSave(IDuccWorkService service)
+ throws Exception
+ {
+ }
+
+
+ public IDuccWorkService restoreService(long duccid)
+ throws Exception
+ {
+ return null;
+ }
+
+ public List<IDuccWorkService> restoreServices(long max)
+ throws Exception
+ {
+ return new ArrayList<IDuccWorkService>();
+ }
+
+
+ public IDuccWorkService serviceRestore(String fileName)
+ {
+ return null;
+ }
+
+ public ArrayList<String> serviceList()
+ {
+ return new ArrayList<String>();
+ }
+
+
+ public ArrayList<IDuccWorkService> serviceRestore()
+ throws IOException,
+ ClassNotFoundException
+ {
+ return new ArrayList<IDuccWorkService>();
+ }
+
+
+ public IDuccWorkService serviceRestore(DuccId duccId)
+ {
+ return null;
+ }
+
+ public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; }
+ public boolean restore(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception { return false; }
+
+}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Tue Sep 15 14:31:24 2015
@@ -19,7 +19,7 @@
package org.apache.uima.ducc.ws;
import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.List;
import java.util.ListIterator;
import java.util.TreeMap;
@@ -105,89 +105,176 @@ public class DuccBoot extends Thread {
return map;
}
- private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) {
- String location = "restoreReservations";
- ArrayList<String> duccWorkReservations = hpm.reservationList();
- logger.info(location, jobid, messages.fetchLabel("Number of Reservations to restore")+duccWorkReservations.size());
- TreeMap<Integer,String> map = sort(duccWorkReservations);
- Iterator<Integer> iterator = map.descendingKeySet().iterator();
- int i = 0;
- int restored = 0;
- while(iterator.hasNext() && (++i < maxReservations)) {
- try {
- Integer key = iterator.next();
- logger.debug(location, jobid, messages.fetchLabel("restore")+key);
- String fileName = map.get(key);
- IDuccWorkReservation duccWorkReservation;
- duccWorkReservation = hpm.reservationRestore(fileName);
- if(duccWorkReservation != null) {
- duccData.putIfNotPresent(duccWorkReservation);
- duccPlugins.restore(duccWorkReservation);
- restored++;
- }
- }
- catch(Throwable t) {
- logger.warn(location, jobid, t);
- }
+ // private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData) {
+ // String location = "restoreReservations";
+ // ArrayList<String> duccWorkReservations = hpm.reservationList();
+ // logger.info(location, jobid, messages.fetchLabel("Number of Reservations to restore")+duccWorkReservations.size());
+ // TreeMap<Integer,String> map = sort(duccWorkReservations);
+ // Iterator<Integer> iterator = map.descendingKeySet().iterator();
+ // int i = 0;
+ // int restored = 0;
+ // while(iterator.hasNext() && (++i < maxReservations)) {
+ // try {
+ // Integer key = iterator.next();
+ // logger.debug(location, jobid, messages.fetchLabel("restore")+key);
+ // String fileName = map.get(key);
+ // IDuccWorkReservation duccWorkReservation;
+ // duccWorkReservation = hpm.reservationRestore(fileName);
+ // if(duccWorkReservation != null) {
+ // duccData.putIfNotPresent(duccWorkReservation);
+ // duccPlugins.restore(duccWorkReservation);
+ // restored++;
+ // }
+ // }
+ // catch(Throwable t) {
+ // logger.warn(location, jobid, t);
+ // }
+ // }
+ // logger.info(location, jobid, messages.fetch("Reservations restored: "+restored));
+ // }
+
+ private void restoreReservations(IHistoryPersistenceManager hpm, DuccData duccData)
+ {
+ // Replaced for database. Both file and database now do all the looping and sorting internally.
+ String location = "restoreReservations";
+ List<IDuccWorkReservation> duccWorkReservations = null;;
+ try {
+ duccWorkReservations = hpm.restoreReservations(maxReservations);
+ } catch (Exception e) {
+ logger.warn(location, null, e);
+ return; // Nothing to do if this fails
}
- logger.info(location, jobid, messages.fetch("Reservations restored: "+restored));
- }
-
- private void restoreJobs(IHistoryPersistenceManager hpm, DuccData duccData) {
- String location = "restoreJobs";
- ArrayList<String> duccWorkJobs = hpm.jobList();
- logger.info(location, jobid, messages.fetchLabel("Number of Jobs to restore")+duccWorkJobs.size());
- TreeMap<Integer,String> map = sort(duccWorkJobs);
- Iterator<Integer> iterator = map.descendingKeySet().iterator();
- int i = 0;
- int restored = 0;
- while(iterator.hasNext() && (++i < maxJobs)) {
- try {
- Integer key = iterator.next();
- logger.debug(location, jobid, messages.fetchLabel("restore")+key);
- String fileName = map.get(key);
- IDuccWorkJob duccWorkJob;
- duccWorkJob = hpm.jobRestore(fileName);
- if(duccWorkJob != null) {
- duccData.putIfNotPresent(duccWorkJob);
- duccPlugins.restore(duccWorkJob);
- restored++;
- }
- }
- catch(Throwable t) {
- logger.warn(location, jobid, t);
- }
+
+ logger.info(location, jobid, messages.fetchLabel("Number of Reservations fetched from DB"), duccWorkReservations.size());
+
+ int restored = 0;
+ for ( IDuccWorkReservation duccWorkReservation : duccWorkReservations ) {
+ try {
+ logger.debug(location, duccWorkReservation.getDuccId(), messages.fetchLabel("restore"));
+ duccData.putIfNotPresent(duccWorkReservation);
+ duccPlugins.restore(duccWorkReservation);
+ restored++;
+ }
+ catch(Throwable t) {
+ logger.warn(location, duccWorkReservation.getDuccId(), t);
+ }
+ }
+ logger.info(location,null, messages.fetch("Reservations restored: "+restored));
+ }
+
+ private void restoreJobs(IHistoryPersistenceManager hpm, DuccData duccData)
+ {
+ // Replaced for database. Both file and database now do all the looping and sorting internally.
+ String location = "restoreJobs";
+ List<IDuccWorkJob> duccWorkJobs = null;;
+ try {
+ duccWorkJobs = hpm.restoreJobs(maxJobs);
+ } catch (Exception e) {
+ logger.warn(location, null, e);
+ return; // Nothing to do if this fails
}
- logger.info(location, jobid, messages.fetch("Jobs restored: "+restored));
- }
-
- private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) {
- String location = "restoreServices";
- ArrayList<String> duccWorkServices = hpm.serviceList();
- logger.info(location, jobid, messages.fetchLabel("Number of Services to restore")+duccWorkServices.size());
- TreeMap<Integer,String> map = sort(duccWorkServices);
- Iterator<Integer> iterator = map.descendingKeySet().iterator();
- int i = 0;
- int restored = 0;
- while(iterator.hasNext() && (++i < maxServices)) {
- try {
- Integer key = iterator.next();
- logger.debug(location, jobid, messages.fetchLabel("restore")+key);
- String fileName = map.get(key);
- IDuccWorkService duccWorkService;
- duccWorkService = hpm.serviceRestore(fileName);
- if(duccWorkService != null) {
- duccData.putIfNotPresent(duccWorkService);
- duccPlugins.restore(duccWorkService);
- restored++;
- }
- }
- catch(Throwable t) {
- logger.warn(location, jobid, t);
- }
+
+ logger.info(location, jobid, messages.fetchLabel("Number of Jobs fetched from DB"), duccWorkJobs.size());
+
+ int restored = 0;
+ for ( IDuccWorkJob duccWorkJob : duccWorkJobs ) {
+ try {
+ logger.debug(location, duccWorkJob.getDuccId(), messages.fetchLabel("restore"));
+ duccData.putIfNotPresent(duccWorkJob);
+ duccPlugins.restore(duccWorkJob);
+ restored++;
+ }
+ catch(Throwable t) {
+ logger.warn(location, duccWorkJob.getDuccId(), t);
+ }
+ }
+ logger.info(location,null, messages.fetch("Jobs restored: "+restored));
+ }
+
+// private void restoreJobsX(IHistoryPersistenceManager hpm, DuccData duccData) {
+// String location = "restoreJobs";
+// ArrayList<String> duccWorkJobs = hpm.jobList();
+// logger.info(location, jobid, messages.fetchLabel("Number of Jobs to restore")+duccWorkJobs.size());
+// TreeMap<Integer,String> map = sort(duccWorkJobs);
+// Iterator<Integer> iterator = map.descendingKeySet().iterator();
+// int i = 0;
+// int restored = 0;
+// while(iterator.hasNext() && (++i < maxJobs)) {
+// try {
+// Integer key = iterator.next();
+// logger.debug(location, jobid, messages.fetchLabel("restore")+key);
+// String fileName = map.get(key);
+// IDuccWorkJob duccWorkJob;
+// duccWorkJob = hpm.jobRestore(fileName);
+// if(duccWorkJob != null) {
+// duccData.putIfNotPresent(duccWorkJob);
+// duccPlugins.restore(duccWorkJob);
+// restored++;
+// }
+// }
+// catch(Throwable t) {
+// logger.warn(location, jobid, t);
+// }
+// }
+// logger.info(location, jobid, messages.fetch("Jobs restored: "+restored));
+// }
+
+ private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData)
+ {
+ // Replaced for database. Both file and database now do all the looping and sorting internally.
+ String location = "restoreServices";
+ List<IDuccWorkService> duccWorkServices = null;;
+ try {
+ duccWorkServices = hpm.restoreServices(maxServices);
+ } catch (Exception e) {
+ logger.warn(location, null, e);
+ return; // Nothing to do if this fails
}
- logger.info(location, jobid, messages.fetch("Services restored: "+restored));
- }
+
+ logger.info(location, jobid, messages.fetchLabel("Number of services fetched from DB"), duccWorkServices.size());
+
+ int restored = 0;
+ for ( IDuccWorkService duccWorkService : duccWorkServices ) {
+ try {
+ logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore"));
+ duccData.putIfNotPresent(duccWorkService);
+ duccPlugins.restore(duccWorkService);
+ restored++;
+ }
+ catch(Throwable t) {
+ logger.warn(location, duccWorkService.getDuccId(), t);
+ }
+ }
+ logger.info(location,null, messages.fetch("Services restored: "+restored));
+ }
+
+ // private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) {
+ // String location = "restoreServices";
+ // ArrayList<String> duccWorkServices = hpm.serviceList();
+ // logger.info(location, jobid, messages.fetchLabel("Number of Services to restore")+duccWorkServices.size());
+ // TreeMap<Integer,String> map = sort(duccWorkServices);
+ // Iterator<Integer> iterator = map.descendingKeySet().iterator();
+ // int i = 0;
+ // int restored = 0;
+ // while(iterator.hasNext() && (++i < maxServices)) {
+ // try {
+ // Integer key = iterator.next();
+ // logger.debug(location, jobid, messages.fetchLabel("restore")+key);
+ // String fileName = map.get(key);
+ // IDuccWorkService duccWorkService;
+ // duccWorkService = hpm.serviceRestore(fileName);
+ // if(duccWorkService != null) {
+ // duccData.putIfNotPresent(duccWorkService);
+ // duccPlugins.restore(duccWorkService);
+ // restored++;
+ // }
+ // }
+ // catch(Throwable t) {
+ // logger.warn(location, jobid, t);
+ // }
+ // }
+ // logger.info(location, jobid, messages.fetch("Services restored: "+restored));
+ // }
private void initialize(CommonConfiguration commonConfiguration) {
String location = "initialize";
@@ -203,7 +290,7 @@ public class DuccBoot extends Thread {
private void restore() {
String location = "restore";
logger.info(location, jobid, messages.fetchLabel("History directory")+IDuccEnv.DUCC_HISTORY_DIR);
- IHistoryPersistenceManager hpm = HistoryFactory.getInstance();
+ IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName());
DuccData duccData = DuccData.getInstance();
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccData.java Tue Sep 15 14:31:24 2015
@@ -70,7 +70,7 @@ public class DuccData {
private volatile String published = null;
- private IHistoryPersistenceManager hpm = HistoryFactory.getInstance();
+ private IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName());
public boolean isPublished() {
return published != null;
@@ -95,26 +95,32 @@ public class DuccData {
}
@SuppressWarnings("unchecked")
- private void mergeHistory(IDuccWorkMap map) {
+ private void mergeHistory(IDuccWorkMap map)
+ {
+ String methodName = "mergeHistory";
Iterator<DuccId> iterator = duccWorkLive.keySet().iterator();
while(iterator.hasNext()) {
DuccId duccId = iterator.next();
IDuccWork duccWork = duccWorkLive.findDuccWork(duccId);
IDuccWork history = null;
- switch(duccWork.getDuccType()) {
- case Job:
- history = hpm.jobRestore(duccId);
- break;
- case Reservation:
- history = hpm.reservationRestore(duccId);
- break;
- case Service:
- history = hpm.serviceRestore(duccId);
- break;
- }
- if(history != null) {
- map.put(duccId, history);
- }
+ try {
+ switch(duccWork.getDuccType()) {
+ case Job:
+ history = hpm.restoreJob(duccId.getFriendly());
+ break;
+ case Reservation:
+ history = hpm.restoreReservation(duccId.getFriendly());
+ break;
+ case Service:
+ history = hpm.restoreService(duccId.getFriendly());
+ break;
+ }
+ if(history != null) {
+ map.put(duccId, history);
+ }
+ } catch ( Exception e ) {
+ logger.warn(methodName, duccId, "Cannot recover", duccWork.getDuccType(), "from database");
+ }
}
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/IServicesRegistry.java Tue Sep 15 14:31:24 2015
@@ -26,28 +26,28 @@ public class IServicesRegistry {
public static final String svc = IStateServices.svc;
// meta
- public static final String autostart = IStateServices.autostart;
- public static final String reference = IStateServices.reference;
- public static final String enabled = IStateServices.enabled;
- public static final String disable_reason = IStateServices.disable_reason;
- public static final String endpoint = IStateServices.endpoint;
- public static final String implementors = IStateServices.implementors;
- public static final String instances = IStateServices.instances;
- public static final String numeric_id = IStateServices.numeric_id;
- public static final String ping_active = IStateServices.ping_active;
- public static final String ping_only = IStateServices.ping_only;
- public static final String service_alive = IStateServices.service_alive;
- public static final String service_class = IStateServices.service_class;
- public static final String service_dependency = IStateServices.service_dependency;
- public static final String service_healthy = IStateServices.service_healthy;
- public static final String service_state = IStateServices.service_state;
- public static final String last_use = IStateServices.last_use;
- public static final String service_statistics = IStateServices.service_statistics;
- public static final String service_type = IStateServices.service_type;
- public static final String submit_error = IStateServices.submit_error;
- public static final String user = IStateServices.user;
+ public static final String autostart = IStateServices.SvcProps.autostart.pname();
+ public static final String reference = IStateServices.SvcProps.reference.pname();
+ public static final String enabled = IStateServices.SvcProps.enabled.pname();
+ public static final String disable_reason = IStateServices.SvcProps.disable_reason.pname();
+ public static final String endpoint = IStateServices.SvcProps.endpoint.pname();
+ public static final String implementors = IStateServices.SvcProps.implementors.pname();
+ public static final String instances = IStateServices.SvcProps.instances.pname();
+ public static final String numeric_id = IStateServices.SvcProps.numeric_id.pname();
+ public static final String ping_active = IStateServices.SvcProps.ping_active.pname();
+ public static final String ping_only = IStateServices.SvcProps.ping_only.pname();
+ public static final String service_alive = IStateServices.SvcProps.service_alive.pname();
+ public static final String service_class = IStateServices.SvcProps.service_class.pname();
+ public static final String service_dependency = IStateServices.SvcProps.service_dependency.pname();
+ public static final String service_healthy = IStateServices.SvcProps.service_healthy.pname();
+ public static final String service_state = IStateServices.SvcProps.service_state.pname();
+ public static final String last_use = IStateServices.SvcProps.last_use.pname();
+ public static final String service_statistics = IStateServices.SvcProps.service_statistics.pname();
+ public static final String service_type = IStateServices.SvcProps.service_type.pname();
+ public static final String submit_error = IStateServices.SvcProps.submit_error.pname();
+ public static final String user = IStateServices.SvcProps.user.pname();
- public static final String service_type_CUSTOM = "CUSTOM";
+ public static final String service_type_CUSTOM = IStateServices.SvcProps.CUSTOM.pname();
public static final String constant_Available = "Available";
public static final String constant_true = "true";
@@ -60,9 +60,9 @@ public class IServicesRegistry {
public static final String constant_OK = "OK";
// svc
- public static final String description = IStateServices.description;
- public static final String process_memory_size = IStateServices.process_memory_size;
- public static final String scheduling_class = IStateServices.scheduling_class;
- public static final String log_directory = IStateServices.log_directory;
+ public static final String description = IStateServices.SvcProps.description.pname();
+ public static final String process_memory_size = IStateServices.SvcProps.process_memory_size.pname();
+ public static final String scheduling_class = IStateServices.SvcProps.scheduling_class.pname();
+ public static final String log_directory = IStateServices.SvcProps.log_directory.pname();
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistry.java Tue Sep 15 14:31:24 2015
@@ -18,7 +18,6 @@
*/
package org.apache.uima.ducc.ws.registry;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -75,10 +74,10 @@ public class ServicesRegistry {
String location = "refreshCache";
try {
ServicesRegistryMap mapRevised = new ServicesRegistryMap();
- IStateServices iss = StateServicesFactory.getInstance();
+ IStateServices iss = StateServicesFactory.getInstance(this.getClass().getName(), "WS");
StateServicesDirectory ssd = iss.getStateServicesDirectory();
if(!ssd.getDescendingKeySet().isEmpty()) {
- for(Integer key : ssd.getDescendingKeySet()) {
+ for(Long key : ssd.getDescendingKeySet()) {
StateServicesSet entry = ssd.get(key);
Properties propertiesSvc = entry.get(IServicesRegistry.svc);
Properties propertiesMeta = entry.get(IServicesRegistry.meta);
@@ -92,7 +91,7 @@ public class ServicesRegistry {
logger.debug(location, jobid, "size: "+map.size());
ServicesSortCache.getInstance().update(map);
}
- catch(IOException e) {
+ catch(Exception e) {
logger.error(location, jobid, e);
}
}
@@ -149,7 +148,7 @@ public class ServicesRegistry {
try {
logger.debug(location, jobid, "size: "+map.size());
logger.debug(location, jobid, "search: "+name);
- for(Integer key : map.keySet()) {
+ for(Long key : map.keySet()) {
ServicesRegistryMapPayload payload = map.get(key);
Properties meta = payload.meta;
if(meta != null) {
@@ -176,7 +175,7 @@ public class ServicesRegistry {
public String findServiceUser(String id) {
String retVal = null;
try {
- for(Integer key : map.keySet()) {
+ for(Long key : map.keySet()) {
ServicesRegistryMapPayload payload = map.get(key);
Properties meta = payload.meta;
if(meta != null) {
@@ -200,7 +199,7 @@ public class ServicesRegistry {
String retVal = null;
try {
long id = duccId.getFriendly();
- for(Integer key : map.keySet()) {
+ for(Long key : map.keySet()) {
ServicesRegistryMapPayload payload = map.get(key);
Properties meta = payload.meta;
if(meta != null) {
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/ServicesRegistryMap.java Tue Sep 15 14:31:24 2015
@@ -23,15 +23,15 @@ import java.util.NavigableSet;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
-public class ServicesRegistryMap extends ConcurrentHashMap<Integer,ServicesRegistryMapPayload> {
+public class ServicesRegistryMap extends ConcurrentHashMap<Long,ServicesRegistryMapPayload> {
private static final long serialVersionUID = 1L;
- public NavigableSet<Integer> getDescendingKeySet() {
- TreeMap<Integer,ServicesRegistryMapPayload> tMap = new TreeMap<Integer,ServicesRegistryMapPayload>();
- Iterator<java.util.Map.Entry<Integer, ServicesRegistryMapPayload>> iterator = entrySet().iterator();
+ public NavigableSet<Long> getDescendingKeySet() {
+ TreeMap<Long,ServicesRegistryMapPayload> tMap = new TreeMap<Long,ServicesRegistryMapPayload>();
+ Iterator<java.util.Map.Entry<Long, ServicesRegistryMapPayload>> iterator = entrySet().iterator();
while(iterator.hasNext()) {
- java.util.Map.Entry<Integer, ServicesRegistryMapPayload> entry = iterator.next();
+ java.util.Map.Entry<Long, ServicesRegistryMapPayload> entry = iterator.next();
tMap.put(entry.getKey(), entry.getValue());
}
return tMap.descendingKeySet();
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java?rev=1703203&r1=1703202&r2=1703203&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/registry/sort/ServicesSortCache.java Tue Sep 15 14:31:24 2015
@@ -46,7 +46,7 @@ public class ServicesSortCache {
public void update(ServicesRegistryMap registryMap) {
TreeMap<SortableService,IServiceAdapter> mapRevised = new TreeMap<SortableService,IServiceAdapter>();
- for(Entry<Integer, ServicesRegistryMapPayload> entry : registryMap.entrySet()) {
+ for(Entry<Long, ServicesRegistryMapPayload> entry : registryMap.entrySet()) {
ServicesRegistryMapPayload payload = entry.getValue();
Properties meta = payload.meta;
Properties svc = payload.svc;
@@ -62,11 +62,11 @@ public class ServicesSortCache {
for(Entry<SortableService, IServiceAdapter> entry : map.entrySet()) {
IServiceAdapter payload = entry.getValue();
Properties meta = payload.getMeta();
- String key = IStateServices.numeric_id;
+ String key = IStateServices.SvcProps.numeric_id.pname();
String value = meta.getProperty(key);
int numeric_id = Integer.parseInt(value);
if(numeric_id == id) {
- meta.setProperty(IStateServices.enabled, Boolean.toString(bool));
+ meta.setProperty(IStateServices.SvcProps.enabled.pname(), Boolean.toString(bool));
payload.setMeta(meta);
break;
}