You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2016/11/04 14:33:37 UTC
svn commit: r1768041 -
/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
Author: burn
Date: Fri Nov 4 14:33:37 2016
New Revision: 1768041
URL: http://svn.apache.org/viewvc?rev=1768041&view=rev
Log:
UIMA-5170 Skip restore of work that shares an already-processed experiment directory
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1768041&r1=1768040&r2=1768041&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Fri Nov 4 14:33:37 2016
@@ -18,6 +18,7 @@
*/
package org.apache.uima.ducc.ws;
+import java.util.HashSet;
import java.util.List;
import org.apache.uima.ducc.common.IDuccEnv;
@@ -52,6 +53,8 @@ public class DuccBoot extends Thread {
duccBoot.initialize(commonConfiguration);
duccBoot.start();
}
+
+ private HashSet<String> experimentsFound;
public void run() {
String location = "run";
@@ -128,11 +131,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of Reservations fetched from history"), duccWorkReservations.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkReservation duccWorkReservation : duccWorkReservations ) {
try {
logger.debug(location, duccWorkReservation.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkReservation);
- duccPlugins.restore(duccWorkReservation);
+ String directory = duccWorkReservation.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkReservation);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -140,13 +148,16 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Reservations restored: "+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found: "+nExperiments));
+ }
}
private void restoreJobs(IHistoryPersistenceManager hpm, DuccData duccData)
{
// Replaced for database. Both file and database now do all the looping and sorting internally.
String location = "restoreJobs";
- List<IDuccWorkJob> duccWorkJobs = null;;
+ List<IDuccWorkJob> duccWorkJobs = null;
try {
duccWorkJobs = hpm.restoreJobs(maxJobs);
} catch (Exception e) {
@@ -157,11 +168,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of Jobs fetched from history"), duccWorkJobs.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkJob duccWorkJob : duccWorkJobs ) {
try {
logger.debug(location, duccWorkJob.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkJob);
- duccPlugins.restore(duccWorkJob);
+ String directory = duccWorkJob.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkJob);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -169,6 +185,9 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Jobs restored: "+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found: "+nExperiments));
+ }
}
// private void restoreJobsX(IHistoryPersistenceManager hpm, DuccData duccData) {
@@ -214,11 +233,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of services fetched from history"), duccWorkServices.size());
int restored = 0;
+ int nExperiments = 0;
for ( IDuccWorkService duccWorkService : duccWorkServices ) {
try {
logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkService);
- duccPlugins.restore(duccWorkService);
+ String directory = duccWorkService.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkService);
+ nExperiments++;
+ }
restored++;
}
catch(Throwable t) {
@@ -226,6 +250,9 @@ public class DuccBoot extends Thread {
}
}
logger.info(location,null, messages.fetch("Services restored: "+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found: "+nExperiments));
+ }
}
// private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) {
@@ -272,9 +299,11 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("History directory")+IDuccEnv.DUCC_HISTORY_DIR);
IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName());
DuccData duccData = DuccData.getInstance();
+ experimentsFound = new HashSet<String>(); // Lets the restore methods avoid inspecting already-found experiments
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
restoreServices(hpm, duccData);
+ experimentsFound = null;
duccData.report();
}
}