You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by de...@apache.org on 2017/07/24 18:21:28 UTC
svn commit: r1802849 - in /uima/uima-ducc/trunk:
uima-ducc-database/src/main/java/org/apache/uima/ducc/database/
uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/
uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/
Author: degenaro
Date: Mon Jul 24 18:21:28 2017
New Revision: 1802849
URL: http://svn.apache.org/viewvc?rev=1802849&view=rev
Log:
UIMA-5509 DUCC Web Server (WS) some historical ManagedReservations missing?
Modified:
uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
Modified: uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java?rev=1802849&r1=1802848&r2=1802849&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-database/src/main/java/org/apache/uima/ducc/database/HistoryManagerDb.java Mon Jul 24 18:21:28 2017
@@ -564,14 +564,24 @@ public class HistoryManagerDb
* Reminder to self, we need to pass Clas<T> cl so compiler can infer T.
*/
@SuppressWarnings("unchecked")
- public <T> ArrayList<T> restoreSeveralThings(Class<T> cl, String tablename, long max)
+ public <T> ArrayList<T> restoreSeveralThings(Class<T> cl, String tablename, String where_clause, long max)
throws Exception
{
String methodName = "restoreSeveralThings";
ArrayList<T> ret = new ArrayList<T>();
DbHandle h = dbManager.open();
- SimpleStatement s = new SimpleStatement("SELECT * from " + tablename + " limit " + max);
+ StringBuffer sb = new StringBuffer();
+ sb.append("SELECT * from ");
+ sb.append(tablename);
+ if(where_clause != null) {
+ sb.append(" "+where_clause);
+ }
+ sb.append(" limit "+max);
+ sb.append(" ALLOW FILTERING");
+ String query = sb.toString();
+ logger.info(methodName, null, query);
+ SimpleStatement s = new SimpleStatement(query);
s.setFetchSize(100);
long now = System.currentTimeMillis();
@@ -625,7 +635,7 @@ public class HistoryManagerDb
public ArrayList<IDuccWorkJob> restoreJobs(long max)
throws Exception
{
- return restoreSeveralThings(IDuccWorkJob.class, JOB_HISTORY_TABLE, max);
+ return restoreSeveralThings(IDuccWorkJob.class, JOB_HISTORY_TABLE, null, max);
}
// End of jobs section
// ----------------------------------------------------------------------------------------------------
@@ -656,7 +666,7 @@ public class HistoryManagerDb
public ArrayList<IDuccWorkReservation> restoreReservations(long max)
throws Exception
{
- return restoreSeveralThings(IDuccWorkReservation.class, RES_HISTORY_TABLE, max);
+ return restoreSeveralThings(IDuccWorkReservation.class, RES_HISTORY_TABLE, null, max);
}
// End of reservations section
@@ -688,8 +698,18 @@ public class HistoryManagerDb
public ArrayList<IDuccWorkService> restoreServices(long max)
throws Exception
{
- return restoreSeveralThings(IDuccWorkService.class, SVC_HISTORY_TABLE, max);
+ return restoreSeveralThings(IDuccWorkService.class, SVC_HISTORY_TABLE, "WHERE TYPE = 'service'", max);
}
+
+ /**
+ * Part of history management, recover these indicated AP instances from history.
+ */
+ public ArrayList<IDuccWorkService> restoreArbitraryProcesses(long max)
+ throws Exception
+ {
+ return restoreSeveralThings(IDuccWorkService.class, SVC_HISTORY_TABLE, "WHERE TYPE = 'AP'", max);
+ }
+
// End of services section
// ----------------------------------------------------------------------------------------------------
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java?rev=1802849&r1=1802848&r2=1802849&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/HistoryPersistenceManager.java Mon Jul 24 18:21:28 2017
@@ -463,6 +463,12 @@ public class HistoryPersistenceManager i
return ret;
}
+ public List<IDuccWorkService> restoreArbitraryProcesses(long max)
+ throws Exception
+ {
+ return null;
+ }
+
// public void serviceSaveConditional(IDuccWorkService duccWorkService)
// throws IOException {
// String id = normalize(""+duccWorkService.getDuccId().getFriendly());
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java?rev=1802849&r1=1802848&r2=1802849&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/IHistoryPersistenceManager.java Mon Jul 24 18:21:28 2017
@@ -44,7 +44,8 @@ public interface IHistoryPersistenceMana
public void saveService(IDuccWorkService duccWorkService) throws Exception;
public IDuccWorkService restoreService(long friendly_id) throws Exception;
public List<IDuccWorkService> restoreServices(long max) throws Exception;
-
+ public List<IDuccWorkService> restoreArbitraryProcesses(long max) throws Exception;
+
public boolean checkpoint(DuccWorkMap work, Map<DuccId, DuccId> processToJob) throws Exception;
public Pair<DuccWorkMap, Map<DuccId, DuccId>> restore() throws Exception;
Modified: uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java?rev=1802849&r1=1802848&r2=1802849&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-transport/src/main/java/org/apache/uima/ducc/transport/event/common/history/NullHistoryManager.java Mon Jul 24 18:21:28 2017
@@ -100,7 +100,12 @@ public class NullHistoryManager
{
return new ArrayList<IDuccWorkService>();
}
-
+
+ public List<IDuccWorkService> restoreArbitraryProcesses(long max)
+ throws Exception
+ {
+ return null;
+ }
public IDuccWorkService serviceRestore(String fileName)
{
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1802849&r1=1802848&r2=1802849&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Mon Jul 24 18:21:28 2017
@@ -295,6 +295,43 @@ public class DuccBoot extends Thread {
}
}
+ private void restoreArbitraryProcesses(IHistoryPersistenceManager hpm, DuccData duccData)
+ {
+ // Replaced for database. Both file and database now do all the looping and sorting internally.
+ String location = "restoreArbitraryProcesses";
+ List<IDuccWorkService> duccWorkServices = null;;
+ try {
+ duccWorkServices = hpm.restoreArbitraryProcesses(maxServices);
+ } catch (Exception e) {
+ logger.warn(location, null, e);
+ return; // Nothing to do if this fails
+ }
+
+ logger.info(location, jobid, messages.fetchLabel("Number of services fetched from history"), duccWorkServices.size());
+
+ int restored = 0;
+ int nExperiments = 0;
+ for ( IDuccWorkService duccWorkService : duccWorkServices ) {
+ try {
+ logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore"));
+ duccData.putIfNotPresent(duccWorkService);
+ String directory = duccWorkService.getStandardInfo().getLogDirectory();
+ if (experimentsFound.add(directory)) {
+ duccPlugins.restore(duccWorkService);
+ nExperiments++;
+ }
+ restored++;
+ }
+ catch(Throwable t) {
+ logger.warn(location, duccWorkService.getDuccId(), t);
+ }
+ }
+ logger.info(location,null, messages.fetch("Services restored: "+restored));
+ if (nExperiments > 0) {
+ logger.info(location,null, messages.fetch("Experiments found: "+nExperiments));
+ }
+ }
+
// private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) {
// String location = "restoreServices";
// ArrayList<String> duccWorkServices = hpm.serviceList();
@@ -343,6 +380,7 @@ public class DuccBoot extends Thread {
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
restoreServices(hpm, duccData);
+ restoreArbitraryProcesses(hpm, duccData);
experimentsFound = null;
duccData.report();
}