You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by bu...@apache.org on 2019/10/25 21:22:04 UTC
svn commit: r1868959 - in /uima/uima-ducc/trunk/uima-ducc-web/src/main:
java/org/apache/uima/ducc/ws/
java/org/apache/uima/ducc/ws/handlers/experiments/
java/org/apache/uima/ducc/ws/xd/ webapp/root/js/
Author: burn
Date: Fri Oct 25 21:22:04 2019
New Revision: 1868959
URL: http://svn.apache.org/viewvc?rev=1868959&view=rev
Log:
UIMA-6130 Simplified handling of updates; use directory instead of UUID on URLs
Modified:
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccPlugins.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/handlers/experiments/HandlerExperimentsServlets.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Experiment.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryManager.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryUtilities.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/IExperiment.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Task.java
uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.local.js
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccBoot.java Fri Oct 25 21:22:04 2019
@@ -20,7 +20,6 @@ package org.apache.uima.ducc.ws;
import java.io.File;
import java.lang.reflect.Field;
-import java.util.HashSet;
import java.util.List;
import org.apache.uima.ducc.common.IDuccEnv;
@@ -58,8 +57,6 @@ public class DuccBoot extends Thread {
duccBoot.start();
}
- private HashSet<String> experimentsFound;
-
public void run() {
String location = "run";
try {
@@ -103,26 +100,15 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of Reservations fetched from history"), duccWorkReservations.size());
- int restored = 0;
- int nExperiments = 0;
for (IDuccWorkReservation duccWorkReservation : duccWorkReservations) {
try {
logger.debug(location, duccWorkReservation.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkReservation);
- String directory = duccWorkReservation.getStandardInfo().getLogDirectory();
- if (experimentsFound.add(directory)) {
- duccPlugins.restore(duccWorkReservation);
- nExperiments++;
- }
- restored++;
+ duccPlugins.restore(duccWorkReservation);
} catch (Throwable t) {
logger.warn(location, duccWorkReservation.getDuccId(), t);
}
}
- logger.info(location, null, messages.fetch("Reservations restored: " + restored));
- if (nExperiments > 0) {
- logger.info(location, null, messages.fetch("Experiments found: " + nExperiments));
- }
}
/**
@@ -171,27 +157,16 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of Jobs fetched from history"), duccWorkJobs.size());
- int restored = 0;
- int nExperiments = 0;
for (IDuccWorkJob duccWorkJob : duccWorkJobs) {
fixup(duccWorkJob);
try {
logger.debug(location, duccWorkJob.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkJob);
- String directory = duccWorkJob.getStandardInfo().getLogDirectory();
- if (experimentsFound.add(directory)) {
- duccPlugins.restore(duccWorkJob);
- nExperiments++;
- }
- restored++;
+ duccPlugins.restore(duccWorkJob);
} catch (Throwable t) {
logger.warn(location, duccWorkJob.getDuccId(), t);
}
}
- logger.info(location, null, messages.fetch("Jobs restored: " + restored));
- if (nExperiments > 0) {
- logger.info(location, null, messages.fetch("Experiments found: " + nExperiments));
- }
}
private void restoreServices(IHistoryPersistenceManager hpm, DuccData duccData) {
@@ -208,26 +183,15 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of services fetched from history"), duccWorkServices.size());
- int restored = 0;
- int nExperiments = 0;
for (IDuccWorkService duccWorkService : duccWorkServices) {
try {
logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkService);
- String directory = duccWorkService.getStandardInfo().getLogDirectory();
- if (experimentsFound.add(directory)) {
- duccPlugins.restore(duccWorkService);
- nExperiments++;
- }
- restored++;
+ duccPlugins.restore(duccWorkService);
} catch (Throwable t) {
logger.warn(location, duccWorkService.getDuccId(), t);
}
}
- logger.info(location, null, messages.fetch("Services restored: " + restored));
- if (nExperiments > 0) {
- logger.info(location, null, messages.fetch("Experiments found: " + nExperiments));
- }
}
private void restoreArbitraryProcesses(IHistoryPersistenceManager hpm, DuccData duccData) {
@@ -244,33 +208,22 @@ public class DuccBoot extends Thread {
logger.info(location, jobid, messages.fetchLabel("Number of APs fetched from history"), duccWorkServices.size());
- int restored = 0;
- int nExperiments = 0;
for (IDuccWorkService duccWorkService : duccWorkServices) {
try {
logger.debug(location, duccWorkService.getDuccId(), messages.fetchLabel("restore"));
duccData.putIfNotPresent(duccWorkService);
- String directory = duccWorkService.getStandardInfo().getLogDirectory();
- if (experimentsFound.add(directory)) {
- duccPlugins.restore(duccWorkService);
- nExperiments++;
- }
- restored++;
+ duccPlugins.restore(duccWorkService); // Filtering of duplicates must be handled later
} catch (Throwable t) {
logger.warn(location, duccWorkService.getDuccId(), t);
}
}
- logger.info(location, null, messages.fetch("Services restored: " + restored));
- if (nExperiments > 0) {
- logger.info(location, null, messages.fetch("Experiments found: " + nExperiments));
- }
}
private void initialize() {
String location = "initialize";
long limit = getLimit();
if (limit > 0) {
- logger.info(location, jobid, messages.fetchLabel("max history limit") + limit);
+ logger.info(location, jobid, messages.fetchLabel("max history limit"), limit);
maxJobs = limit;
maxReservations = limit;
maxServices = limit;
@@ -279,16 +232,14 @@ public class DuccBoot extends Thread {
private void restore() {
String location = "restore";
- logger.info(location, jobid, messages.fetchLabel("History directory") + IDuccEnv.DUCC_HISTORY_DIR);
+ logger.info(location, jobid, messages.fetchLabel("History directory"), IDuccEnv.DUCC_HISTORY_DIR);
IHistoryPersistenceManager hpm = HistoryFactory.getInstance(this.getClass().getName());
DuccData.reset();
DuccData duccData = DuccData.getInstance();
- experimentsFound = new HashSet<String>(); // Lets the restore methods avoid inspecting already-found experiments
restoreReservations(hpm, duccData);
restoreJobs(hpm, duccData);
restoreServices(hpm, duccData);
restoreArbitraryProcesses(hpm, duccData);
- experimentsFound = null;
duccData.report();
}
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccPlugins.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccPlugins.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccPlugins.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/DuccPlugins.java Fri Oct 25 21:22:04 2019
@@ -22,17 +22,13 @@ import java.util.ArrayList;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
-import org.apache.uima.ducc.transport.cmdline.ICommandLine;
-import org.apache.uima.ducc.transport.event.common.DuccWorkJob;
-import org.apache.uima.ducc.transport.event.common.IDuccProcess;
-import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
-import org.apache.uima.ducc.transport.event.common.IDuccStandardInfo;
import org.apache.uima.ducc.transport.event.common.IDuccWorkJob;
import org.apache.uima.ducc.transport.event.common.IDuccWorkMap;
import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
import org.apache.uima.ducc.transport.event.common.IDuccWorkService.ServiceDeploymentType;
import org.apache.uima.ducc.ws.handlers.experiments.HandlerExperimentsServlets;
+import org.apache.uima.ducc.ws.server.DuccWebProperties;
import org.apache.uima.ducc.ws.server.DuccWebServer;
import org.apache.uima.ducc.ws.xd.ExperimentsRegistryManager;
import org.eclipse.jetty.server.Handler;
@@ -49,45 +45,41 @@ public class DuccPlugins {
return instance;
}
- private static ExperimentsRegistryManager experimentsRegistryManager = ExperimentsRegistryManager
- .getInstance();
+ // Do nothing if not enabled
+ private boolean experimentsEnabled;
+ private ExperimentsRegistryManager experimentsRegistryManager = null;
+
+ private DuccPlugins() {
+ String ducc_experiments = DuccWebProperties.get().getProperty("ducc.experiments", "false");
+ experimentsEnabled = ducc_experiments.equalsIgnoreCase("true");
+ if (experimentsEnabled) {
+ experimentsRegistryManager = ExperimentsRegistryManager.getInstance();
+ }
+ }
+
/**
* The restore methods are called during boot of the web server. This is an opportunity to have
* local mods plug-in for processing that may be desirable relative to each Job, Reservation, and
* Service during the restoration from history.
+ * Note - exceptions are caught in DuccBoot
*/
public void restore(IDuccWorkJob job) {
- String location = "restore";
- try {
- if (job != null) {
- experimentsRegistryManager.initialize(job);
- }
- } catch (Throwable t) {
- logger.error(location, jobid, t);
+ if (experimentsRegistryManager != null) {
+ experimentsRegistryManager.initialize(job);
}
}
public void restore(IDuccWorkReservation reservation) {
- String location = "restore";
- try {
- // loc mods here
- } catch (Throwable t) {
- logger.error(location, jobid, t);
- }
+ // Ignore unmanaged reservations
}
public void restore(IDuccWorkService service) {
- String location = "restore";
- try {
- // Also process managed reservations in case the experiment has only these.
- // Note: APs are saved in DB as services of type "other"
- if (service != null && service.getServiceDeploymentType() == ServiceDeploymentType.other) {
- experimentsRegistryManager.initialize(service);
- }
- } catch (Throwable t) {
- logger.error(location, jobid, t);
+ // Also process managed reservations in case the experiment has only these.
+ // Note: APs are saved in DB as services of type "other"
+ if (experimentsRegistryManager != null && service.getServiceDeploymentType() == ServiceDeploymentType.other) {
+ experimentsRegistryManager.initialize(service);
}
}
@@ -100,7 +92,9 @@ public class DuccPlugins {
public void update(IDuccWorkMap dwm) {
String location = "update";
try {
- experimentsRegistryManager.update(dwm);
+ if (experimentsRegistryManager != null) {
+ experimentsRegistryManager.update(dwm);
+ }
} catch (Throwable t) {
logger.error(location, jobid, t);
}
@@ -114,6 +108,9 @@ public class DuccPlugins {
public ArrayList<Handler> gethandlers(DuccWebServer duccWebServer) {
String location = "gethandlers";
ArrayList<Handler> handlersList = new ArrayList<Handler>();
+ if (!experimentsEnabled) {
+ return handlersList;
+ }
try {
HandlerExperimentsServlets handlerExperimentsServlets = new HandlerExperimentsServlets(
duccWebServer);
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/handlers/experiments/HandlerExperimentsServlets.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/handlers/experiments/HandlerExperimentsServlets.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/handlers/experiments/HandlerExperimentsServlets.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/handlers/experiments/HandlerExperimentsServlets.java Fri Oct 25 21:22:04 2019
@@ -22,8 +22,6 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Map.Entry;
-import java.util.TreeMap;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -49,9 +47,9 @@ import org.apache.uima.ducc.ws.utils.For
import org.apache.uima.ducc.ws.utils.FormatServletScroll;
import org.apache.uima.ducc.ws.utils.HandlersHelper;
import org.apache.uima.ducc.ws.utils.HandlersHelper.AuthorizationStatus;
+import org.apache.uima.ducc.ws.xd.IExperiment;
import org.apache.uima.ducc.ws.xd.ExperimentsRegistryManager;
import org.apache.uima.ducc.ws.xd.ExperimentsRegistryUtilities;
-import org.apache.uima.ducc.ws.xd.IExperiment;
import org.apache.uima.ducc.ws.xd.Jed;
import org.apache.uima.ducc.ws.xd.Jed.Status;
import org.apache.uima.ducc.ws.xd.Task;
@@ -95,7 +93,7 @@ public class HandlerExperimentsServlets
private long getRunTime(HttpServletRequest request, IExperiment experiment) {
long runTime = 0;
- Task[] tasks = experiment.getTasks();
+ ArrayList<Task> tasks = experiment.getTasks();
for (Task task : tasks) {
if (task.parentId == 0) {
if (task.runTime > runTime) {
@@ -157,8 +155,7 @@ public class HandlerExperimentsServlets
private String getDirectoryLink(HttpServletRequest request, IExperiment experiment) {
String directory = experiment.getDirectory();
- String id = experiment.getId();
- String href = "href=\"experiment.details.html?id=" + id + "\"";
+ String href = "href='experiment.details.html?dir=" + directory + "'";
String directoryLink = "<a" + " " + href + " " + ">" + directory + "</a>";
return directoryLink;
}
@@ -172,15 +169,12 @@ public class HandlerExperimentsServlets
FormatServlet fmt = tableStyle.equals("scroll") ? new FormatServletScroll() : new FormatServletClassic();
- TreeMap<IExperiment, String> map = experimentsRegistryManager.getMapByStatus();
-
int maxRecords = HandlersUtilities.getExperimentsMax(request);
ArrayList<String> users = HandlersUtilities.getExperimentsUsers(request);
-
- for (Entry<IExperiment, String> entry : map.entrySet()) {
-
- IExperiment experiment = entry.getKey();
+
+ // List experiments in "experiment" order: active, newest start-date, directory
+ for (IExperiment experiment : experimentsRegistryManager.getMapByStatus().keySet()) {
boolean fullTable = fmt.numRows() >= maxRecords;
@@ -196,7 +190,6 @@ public class HandlerExperimentsServlets
Status experimentStatus = experiment.getStatus();
switch (experimentStatus) {
case Running:
- String id = experiment.getId();
String directory = experiment.getDirectory();
String disabled = "";
String resourceOwnerUserid = experiment.getUser();
@@ -205,7 +198,7 @@ public class HandlerExperimentsServlets
disabled = "disabled title=\"Hint: use Login to enable\"";
}
terminate = "<input type=\"button\"" +
- " onclick=\"ducc_confirm_terminate_experiment('" + id + "','" + directory + "')\"" +
+ " onclick=\"ducc_confirm_terminate_experiment('" + directory + "')\"" +
" value=\"Terminate\" " + disabled + "/>";
default:
break;
@@ -219,7 +212,7 @@ public class HandlerExperimentsServlets
fmt.addElemR(duration, runTime);
fmt.addElemL(getUser(request, experiment));
- fmt.addElemR(experiment.getTasks().length);
+ fmt.addElemR(experiment.getTasks().size());
fmt.addElemL(getState(request, experiment));
fmt.addElemL(getDirectoryLink(request, experiment));
@@ -286,7 +279,7 @@ public class HandlerExperimentsServlets
color = "background-color:PaleGreen;";
}
state = "<input type='button' style='" + color + "'"
- + " onclick=\"ducc_toggle_task_state('" + experiment.getId() + "','" + task.taskId + "')\""
+ + " onclick=\"ducc_toggle_task_state('" + task.taskId + "')\""
+ " title=\"Click to toggle state\""
+ " value=\"" + state + "\" />";
}
@@ -297,7 +290,7 @@ public class HandlerExperimentsServlets
}
return state;
}
-
+
private String decorateStepStart(Task task, HttpServletRequest request) {
String startTime = "";
if (task.startTime != null) {
@@ -434,15 +427,11 @@ public class HandlerExperimentsServlets
WsLog.enter(cName, mName);
boolean handled = false;
-
- String id = request.getParameter("id");
-
- IExperiment experiment = experimentsRegistryManager.getById(id);
-
long now = System.currentTimeMillis();
-
FormatServlet fmt = tableStyle.equals("scroll") ? new FormatServletScroll() : new FormatServletClassic();
-
+
+ String dir = request.getParameter("dir");
+ IExperiment experiment = experimentsRegistryManager.getExperiment(dir);
if (experiment != null) {
// Check if the experiment can be restarted, i.e.
// launched by DUCC as a JED AP, stopped, and owned by the logged-in user
@@ -452,15 +441,15 @@ public class HandlerExperimentsServlets
boolean isCanceled = experiment.getStatus() == Jed.Status.Canceled;
- Task[] tasks = experiment.getTasks();
+ ArrayList<Task> tasks = experiment.getTasks();
if (tasks != null) {
// Check if given a task whose state is to be toggled between Completed & Rerun
String toggleTask = request.getParameter("taskid");
if (toggleTask != null) {
- int toggle = Integer.parseInt(toggleTask);
- Task task = tasks[toggle-1];
+ int toggleId = Integer.parseInt(toggleTask);
+ Task task = findTask(tasks, toggleId);
task.rerun = !task.rerun;
- markSubtasks(tasks, toggle, task.rerun);
+ markSubtasks(tasks, toggleId, task.rerun);
}
// Find the latest duccId to display for a task ... omit if not started or has been reset for a rerun
for (Task task : tasks) {
@@ -498,13 +487,21 @@ public class HandlerExperimentsServlets
return handled;
}
+ private Task findTask(ArrayList<Task> tasks, int taskId) {
+ for (Task task : tasks) {
+ if (task.taskId == taskId) {
+ return task;
+ }
+ }
+ return null; // Will cause an NPE ??
+ }
+
// Mark each child with the same rerun state as the parent.
- // Note that children ALWAYS have a larger taskId
- private void markSubtasks(Task[] tasks, int parentId, boolean rerun) {
- for (int childId = parentId + 1; childId <= tasks.length; ++childId) {
- if (tasks[childId-1].parentId == parentId) {
- tasks[childId-1].rerun = rerun;
- markSubtasks(tasks, childId, rerun);
+ private void markSubtasks(ArrayList<Task> tasks, int parentId, boolean rerun) {
+ for (Task task : tasks) {
+ if (task.parentId == parentId) {
+ task.rerun = rerun;
+ markSubtasks(tasks, task.taskId, rerun);
}
}
}
@@ -516,15 +513,13 @@ public class HandlerExperimentsServlets
StringBuffer sb = new StringBuffer();
- String id = request.getParameter("id");
+ String directory = request.getParameter("dir");
boolean restart = false;
- IExperiment experiment = experimentsRegistryManager.getById(id);
+ IExperiment experiment = experimentsRegistryManager.getExperiment(directory);
- if (experiment != null && experiment.getDirectory() != null) {
- String directory = experiment.getDirectory();
-
+ if (experiment != null) {
// Display Terminate/Restart button if DUCC-launched && the owner logged in
String button = null;
if (experiment.getJedDuccId() != null &&
@@ -537,7 +532,7 @@ public class HandlerExperimentsServlets
+ " title='experiment is restarting'>Restarting...</button>";
} else if (status == Jed.Status.Running) {
button = "<button style='background-color:LightPink;font-size:16px' "
- + "onclick=\"ducc_confirm_terminate_experiment('" + id + "','" + directory + "')\""
+ + "onclick=\"ducc_confirm_terminate_experiment('" + directory + "')\""
+ " title='click to terminate experiment'>TERMINATE</button>";
} else {
button = "<button style='background-color:PaleGreen;font-size:16px' "
@@ -576,15 +571,13 @@ public class HandlerExperimentsServlets
StringBuffer sb = new StringBuffer();
- String id = request.getParameter("id");
-
- IExperiment experiment = experimentsRegistryManager.getById(id);
+ String directory = request.getParameter("dir");
+ IExperiment experiment = experimentsRegistryManager.getExperiment(directory);
String resourceOwnerUserId = experiment.getUser();
- String directory = experiment.getDirectory();
String file = "DRIVER.running";
- String path = directory + File.separator + file;
+ String path = new File(directory, file).getAbsolutePath();
WsLog.info(cName, mName, path);
@@ -616,8 +609,9 @@ public class HandlerExperimentsServlets
HttpServletResponse response) throws Exception {
String mName = "handleDuccRequest";
WsLog.enter(cName, mName);
- String reqURI = request.getRequestURI() + "";
+ String reqURI = request.getRequestURI();
boolean handled = false;
+ //if (reqURI.contains("experiment")) WsLog.info(cName, mName, "!! start "+reqURI);
if (handled) {
} else if (reqURI.startsWith(duccContextExperimentDetailsDirectory)) {
handled = handleServletExperimentDetailsDirectory(target, baseRequest, request, response);
@@ -637,7 +631,8 @@ public class HandlerExperimentsServlets
} else if (reqURI.startsWith(duccContextExperimentCancelRequest)) {
handled = handleServletExperimentCancelRequest(target, baseRequest, request, response);
}
-
+ //if (reqURI.contains("experiment")) WsLog.info(cName, mName, "!! end "+reqURI);
+
WsLog.exit(cName, mName);
return handled;
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Experiment.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Experiment.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Experiment.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Experiment.java Fri Oct 25 21:22:04 2019
@@ -25,7 +25,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.UUID;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -48,37 +47,21 @@ public class Experiment implements IExpe
private long fileDate = 0;
- private String id = UUID.randomUUID().toString();
-
- private int version;
-
private DuccId jedDuccId;
- public String umask;
- public Experiment(String user, String directory, long fileDate, int version, ArrayList<Task> tasks, IDuccWork work) {
+ public Experiment(String user, String directory, long fileDate, ArrayList<Task> tasks, IDuccWork work) {
this.user = user;
this.directory = directory;
this.fileDate = fileDate;
- this.version = version;
this.tasks = tasks;
+ tasks.sort(null); // Sort tasks in taskId order for display
if (work != null) {
this.jedDuccId = work.getDuccId();
- this.umask = work.getStandardInfo().getUmask();
}
}
@Override
- public void setId(String value) {
- id = value;
- }
-
- @Override
- public String getId() {
- return id;
- }
-
- @Override
public String getUser() {
return user;
}
@@ -89,62 +72,28 @@ public class Experiment implements IExpe
}
@Override
- public void setJedDuccId(DuccId duccId) {
- this.jedDuccId = duccId;
+ public long getFileDate() {
+ return fileDate;
}
@Override
- public DuccId getJedDuccId() {
- return jedDuccId;
+ public ArrayList<Task> getTasks() {
+ return tasks;
}
- // Create an array indexed by taskId-1
@Override
- public Task[] getTasks() {
- int size = tasks==null ? 0 : tasks.size();
- Task[] tasksArray = new Task[size];
- for (Task task : tasks) {
- tasksArray[task.taskId-1] = task;
- }
- return tasksArray;
+ public DuccId getJedDuccId() {
+ return jedDuccId;
}
+ // Update the duccId for the JED AP if missing or is older
@Override
- public ArrayList<String> getJobIds() {
- ArrayList<String> jobIds = new ArrayList<String>();
- if (tasks != null) {
- for (Task task : tasks) {
- if (task.type != null) {
- Jed.Type jedType = Jed.Type.getEnum(task.type);
- switch (jedType) {
- case Ducc_Job:
- case Java:
- case Trainer:
- long[] duccIdList = task.duccId;
- for (long duccId : duccIdList) {
- if (duccId < 0) {
- // reservation
- } else {
- // job
- String jobId = "" + (0 + duccId);
- jobIds.add(jobId);
- }
- }
- break;
- default:
- break;
- }
- }
- }
+ public void updateJedId(DuccId duccId) {
+ if (jedDuccId == null || jedDuccId.getFriendly() < duccId.getFriendly()) {
+ jedDuccId = duccId;
}
- return jobIds;
}
-
- @Override
- public int getVersion() {
- return version;
- }
-
+
@Override
public boolean isActive() {
boolean retVal = false;
@@ -159,10 +108,10 @@ public class Experiment implements IExpe
return retVal;
}
+ // TODO - the experiment status could be determined in the constructor
@Override
public Jed.Status getStatus() {
Jed.Status retVal = Jed.Status.Unknown;
- Task[] tasks = getTasks();
if (tasks != null) {
boolean canceled = false;
boolean failed = false;
@@ -221,8 +170,7 @@ public class Experiment implements IExpe
+ then rewrite the Experiment.state file
*/
@Override
- public boolean updateStateFile() {
- Task[] tasks = getTasks();
+ public boolean updateStateFile(String umask) {
if (tasks == null) {
return true;
}
@@ -236,7 +184,7 @@ public class Experiment implements IExpe
task.runTime = 0;
}
}
- return writeStateFile();
+ return writeStateFile(umask);
}
/*
@@ -244,7 +192,7 @@ public class Experiment implements IExpe
* as the user copy it to the output directory,
* delete the temp file.
*/
- private boolean writeStateFile() {
+ private boolean writeStateFile(String umask) {
File tempFile = null;
Gson gson = new GsonBuilder().excludeFieldsWithoutExposeAnnotation().setPrettyPrinting().create();
try {
@@ -285,11 +233,6 @@ public class Experiment implements IExpe
}
@Override
- public long getStartTime() {
- return getMillis(getStartDate());
- }
-
- @Override
public String getStartDate() {
String retVal = "";
long experimentStartMillis = Long.MAX_VALUE;
@@ -308,38 +251,22 @@ public class Experiment implements IExpe
return retVal;
}
- @Override
- public void setFileDate(long value) {
- fileDate = value;
- }
-
- @Override
- public long getFileDate() {
- return fileDate;
- }
-
private long staleTime = 1000 * 60 * 6;
- private long staleTimeOld = 1000 * 60 * 60 * 24;
-
@Override
public boolean isStale() {
- // If the log file has been removed then the driver has stopped.
- // If the lock file is still present the driver may have been killed, so check the age of the
- // state file.
- // If an old version-less file then may have a long job running so wait 24 hrs - new ones are
- // touched every 5 mins
- // Check that the lock-file exists by reading it as the user if necessary,
- boolean logLocked = null != ExperimentsRegistryUtilities.getFileContents(user,
- directory + "/DRIVER.log.lck");
- if (logLocked) {
+ // If the log lock file has been removed then the driver has stopped.
+ // If the lock file is still present the driver may have been killed, so check the age of the state file.
+ // If the lock-file exists its timestamp will be > 0
+ File lockFile = new File(directory, "DRIVER.log.lck");
+ boolean lockExists = ExperimentsRegistryUtilities.getFileTime(user, lockFile) > 0;
+ if (lockExists) {
long now = System.currentTimeMillis();
- long fileDate = ExperimentsRegistryUtilities.getFileDate(this);
+ long fileDate = getFileDate();
if (fileDate > 0) {
if (fileDate < now) {
long elapsed = now - fileDate;
- long tStale = (version == 0) ? staleTimeOld : staleTime;
- if (elapsed < tStale) {
+ if (elapsed < staleTime) {
return false;
}
}
@@ -348,84 +275,41 @@ public class Experiment implements IExpe
return true;
}
+ // The main experiments map uses the directory name as a key, and since it is unique its hashcode can be used when experiment is a key
@Override
public int hashCode() {
return directory.hashCode();
}
+ /*
+ * Sort active experiments first, then by start date (newest first) then by directory
+ */
@Override
- public int compareTo(Object object) {
- int retVal = 0;
- if (object != null) {
- if (object instanceof Experiment) {
- Experiment that = (Experiment) object;
- if (retVal == 0) {
- retVal = compareState(that);
- }
- if (retVal == 0) {
- retVal = compareStartDate(that);
- }
- if (retVal == 0) {
- retVal = compareDirectory(that);
- }
- if (retVal == 0) {
- retVal = compareUser(that);
- }
+ public int compareTo(IExperiment that) {
+ // Booleans sort false before true so must reverse and compare that with this
+ int retVal = new Boolean(that.isActive()).compareTo(new Boolean(this.isActive())); // active first
+ if (retVal == 0) {
+ retVal = that.getStartDate().compareTo(this.getStartDate()); // reverse of natural ordering i.e. newest first
+ if (retVal == 0) {
+ retVal = this.getDirectory().compareTo(that.getDirectory()); // natural alphabetic
}
+ // Since the map is keyed by directory we should never see identical directories
}
return retVal;
}
- private int compareState(Experiment that) {
- int retVal = 0;
- if (this.isActive()) {
- if (that.isActive()) {
- // retVal = 0;
- } else {
- retVal = -1;
- }
- } else {
- if (that.isActive()) {
- retVal = 1;
- } else {
- // retVal = 0;
- }
- }
- return retVal;
- }
-
- private int compareStrings(String s0, String s1) {
- int retVal = 0;
- if (s0 != null) {
- if (s1 != null) {
- retVal = s1.compareTo(s0);
- }
- }
- return retVal;
- }
-
- private int compareStartDate(Experiment that) {
- int retVal = 0;
- if (that != null) {
- retVal = compareStrings(this.getStartDate(), that.getStartDate());
- }
- return retVal;
- }
-
- private int compareDirectory(Experiment that) {
- int retVal = 0;
- if (that != null) {
- retVal = compareStrings(this.getDirectory(), that.getDirectory());
- }
- return retVal;
- }
-
- private int compareUser(Experiment that) {
- int retVal = 0;
- if (that != null) {
- retVal = compareStrings(this.getUser(), that.getUser());
+ /*
+ * Satisfy the Comparable contract by making equals returns true only when compareTo returns 0
+ * (which should never happen in practice)
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || ! (obj instanceof Experiment)) {
+ return false;
}
- return retVal;
+ Experiment that = (Experiment) obj;
+ return (this.isActive() == that.isActive()) &&
+ this.getStartDate().equals(that.getStartDate()) &&
+ this.getDirectory().equals(that.getDirectory());
}
-
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryManager.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryManager.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryManager.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryManager.java Fri Oct 25 21:22:04 2019
@@ -23,10 +23,12 @@ import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Type;
import java.util.ArrayList;
-import java.util.Iterator;
+import java.util.HashSet;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.id.DuccId;
@@ -34,7 +36,6 @@ import org.apache.uima.ducc.transport.ev
import org.apache.uima.ducc.transport.event.common.IDuccWork;
import org.apache.uima.ducc.transport.event.common.IDuccWorkMap;
import org.apache.uima.ducc.ws.log.WsLog;
-import org.apache.uima.ducc.ws.server.DuccWebProperties;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
@@ -48,133 +49,99 @@ public class ExperimentsRegistryManager
private static Gson gson = new Gson();
private static ExperimentsRegistryManager instance = new ExperimentsRegistryManager();
+
+ private static String stateFileName = "Experiment.state";
private int MAX_CACHE_SIZE = 4096;
- private TreeMap<String, IExperiment> map = new TreeMap<String, IExperiment>();
-
- private AtomicLong updateCounter = new AtomicLong(0);
+ private Set<String> seenDirs = new HashSet<String>(); // Set of directories seen when booting or during an OR publication
- private boolean enabled; // true if Experiments button is enabled/displayed
+ private Map<String, IExperiment> experimentsByDir = new ConcurrentHashMap<String,IExperiment>();
public static ExperimentsRegistryManager getInstance() {
return instance;
}
- private ExperimentsRegistryManager() {
- String ducc_experiments = DuccWebProperties.get().getProperty("ducc.experiments", "false");
- enabled = ducc_experiments.equalsIgnoreCase("true");
- WsLog.info(cName, "<constructor>", "Experiments enabled: " + enabled);
+ public IExperiment getExperiment(String directory) {
+ return experimentsByDir.get(directory);
}
- public TreeMap<IExperiment, String> getMapByStatus() {
- TreeMap<IExperiment, String> mapInverse = new TreeMap<IExperiment, String>();
- synchronized (map) {
- for (Entry<String, IExperiment> entry : map.entrySet()) {
- mapInverse.put(entry.getValue(), entry.getKey());
+ // Called by DuccPlugins with work restored from the DB when web-server boots
+ public void initialize(IDuccWork dw) {
+ //String mName = "initialize";
+ IDuccStandardInfo stdInfo = dw.getStandardInfo();
+ if (stdInfo != null) {
+ String experimentDirectory = stdInfo.getExperimentDirectory();
+ if (experimentDirectory != null) {
+ // Is a JED AP so check that we have the duccId of the latest AP even if not the first seen
+ seenDirs.add(experimentDirectory);
+ update(stdInfo.getUser(), experimentDirectory, dw);
+ } else {
+ // May be one of the tasks from a local or ducc-launched JED so process only the first
+ String logDirectory = stdInfo.getLogDirectory();
+ String parent = new File(logDirectory).getParent();
+ if (seenDirs.add(parent)) {
+ update(stdInfo.getUser(), parent, null);
+ }
}
}
- return mapInverse;
}
- private TreeMap<Long, String> getMapByDate() {
- TreeMap<Long, String> mapByDate = new TreeMap<Long, String>();
- synchronized (map) {
- for (Entry<String, IExperiment> entry : map.entrySet()) {
- IExperiment experiment = entry.getValue();
- Long key = new Long(experiment.getStartTime());
- String value = experiment.getId();
- mapByDate.put(key, value);
- }
+ // Called by DuccPlugins for each OR publication
+ public void update(IDuccWorkMap dwm) {
+ String mName = "update";
+ if (dwm == null) {
+ WsLog.warn(cName, mName, "missing map");
+ return;
}
- return mapByDate;
- }
-
- public IExperiment getById(String id) {
- IExperiment retVal = null;
- if (id != null) {
- synchronized (map) {
- for (Entry<String, IExperiment> entry : map.entrySet()) {
- IExperiment experiment = entry.getValue();
- if (id.equals(experiment.getId())) {
- retVal = experiment;
- break;
+ seenDirs.clear(); // Clear set of visited dirs before every update
+ try {
+ // Inspect all jobs
+ for (DuccId duccId : dwm.getJobKeySet()) {
+ IDuccWork job = dwm.findDuccWork(duccId);
+ if (job != null) {
+ IDuccStandardInfo stdInfo = job.getStandardInfo();
+ if (stdInfo != null) {
+ String user = stdInfo.getUser();
+ String logDirectory = stdInfo.getLogDirectory();
+ String parent = new File(logDirectory).getParent();
+ if (seenDirs.add(parent)) { // Process the first job with this parent dir
+ update(user, parent, null);
+ }
}
}
}
- }
- return retVal;
- }
-
- private void replace(String directory, Experiment experiment) {
- String mName = "replace";
-
- // Keep the same id
- String id = map.get(directory).getId();
- experiment.setId(id);
-
- // Keep latest JED id ... and umask in case this is not the JED AP
- DuccId oldDuccId = map.get(directory).getJedDuccId();
- long oldNum = oldDuccId==null? 0 : oldDuccId.getFriendly();
- DuccId newDuccId = experiment.getJedDuccId();
- long newNum = newDuccId==null? 0 : newDuccId.getFriendly();
- if (oldNum > newNum) {
- experiment.setJedDuccId(oldDuccId);
- experiment.umask = ((Experiment)map.get(directory)).umask; // Ugh - should update rather than replace ??
- }
-
- map.put(directory, experiment);
- WsLog.debug(cName, mName, directory);
- }
- private void add(String directory, Experiment experiment) {
- String mName = "add";
- map.put(directory, experiment);
- WsLog.debug(cName, mName, directory);
- }
-
- private void put(String directory, Experiment experiment) {
- synchronized (map) {
- if (map.containsKey(directory)) {
- replace(directory, experiment);
- } else {
- add(directory, experiment);
+ // Inspect managed reservations ... check if a JED AP
+ for (DuccId duccId : dwm.getManagedReservationKeySet()) {
+ IDuccWork work = dwm.findDuccWork(duccId);
+ if (work != null) {
+ IDuccStandardInfo stdInfo = work.getStandardInfo();
+ if (stdInfo != null) {
+ String user = stdInfo.getUser();
+ String experimentDirectory = stdInfo.getExperimentDirectory();
+ if (experimentDirectory != null) {
+ // Even if have seen this dir earlier we must check that we have the duccId of the latest AP
+ // JED tasks usually have a log dir 1 level below the exp dir
+ seenDirs.add(experimentDirectory); // This prevents processing of any of its child jobs
+ update(user, experimentDirectory, work);
+ } else {
+ String logDirectory = stdInfo.getLogDirectory();
+ String parent = new File(logDirectory).getParent();
+ if (seenDirs.add(parent)) { // Process the first AP with this parent dir
+ update(user, parent, null); // (unless already seen as a JED exp dir)
+ }
+ }
+ }
+ }
}
+ prune(); // if list too large
+ } catch (Exception e) {
+ WsLog.error(cName, mName, e);
}
}
-
- private void remove(String directory) {
- String mName = "remove";
- synchronized (map) {
- map.remove(directory);
-
- }
- WsLog.debug(cName, mName, directory);
- }
-
- private boolean containsKey(String directory) {
- synchronized (map) {
- return map.containsKey(directory);
- }
- }
-
- // Called by DuccPlugins when web-server boots
- public void initialize(IDuccWork dw) {
- IDuccStandardInfo stdInfo = dw.getStandardInfo();
- if (stdInfo != null) {
- String user = stdInfo.getUser();
- String directory = stdInfo.getLogDirectory();
- String experimentDirectory = stdInfo.getExperimentDirectory();
- if (experimentDirectory != null) {
- update(user, experimentDirectory, false, dw);
- } else {
- directory = ExperimentsRegistryUtilities.upOne(directory);
- update(user, directory, false, null);
- }
- }
- }
- // DuccWork provided only for the AP that launches JED
+ // Load or refresh the experiment
private void update(String user, String directory, IDuccWork work) {
String mName = "update";
@@ -186,194 +153,91 @@ public class ExperimentsRegistryManager
return;
}
- try {
- String fileName = ExperimentsRegistryUtilities.getStateFilePath(directory);
- long date = ExperimentsRegistryUtilities.getFileDate(user, fileName);
- WsLog.debug(cName, mName, "Reading " + fileName + " date: " + date);
- String contents = ExperimentsRegistryUtilities.getFileContents(user, fileName);
- if (contents != null) {
- // Version may precede the initial '[' and may be on a separate line
- int version = 0;
- int offset = contents.indexOf('[');
- if (offset > 0) {
- String s = contents.substring(0, offset).trim();
- try {
- version = Integer.parseInt(s);
- } catch (NumberFormatException e) {
- WsLog.warn(cName, mName, "Invalid version '" + s + "' in state file : " + fileName);
- return;
- }
- contents = contents.substring(offset);
- }
- if (offset < 0) {
- WsLog.warn(cName, mName, "Invalid syntax (missing '[') in state file : " + fileName);
- return;
- }
- StringReader sr = new StringReader(contents);
- Type tasksType = new TypeToken<ArrayList<Task>>() {
- }.getType();
- try {
- ArrayList<Task> taskArray = gson.fromJson(sr, tasksType);
- Experiment experiment = new Experiment(user, directory, date, version, taskArray, work);
- put(directory, experiment);
- } catch (JsonParseException e) {
- WsLog.warn(cName, mName,
- "Ignoring " + fileName + " as has Json syntax error " + e.getMessage());
- }
- } else {
+ File stateFile = new File(directory, stateFileName);
+ long fileTime = ExperimentsRegistryUtilities.getFileTime(user, stateFile);
+ if (fileTime == 0) {
+ return; // File not found
+ }
+
+ // Check if state file is newer than any existing one
+ // If newer refresh it, otherwise just update the JED duccId in case it is a newer AP
+ IExperiment existingExperiment = experimentsByDir.get(directory);
+ if (existingExperiment != null) {
+ if (fileTime <= existingExperiment.getFileDate()) {
if (work != null) {
- WsLog.warn(cName, mName, "State file missing or inaccessible in " + directory + " for JED AP " + work.getDuccId());
+ existingExperiment.updateJedId(work.getDuccId());
}
- WsLog.trace(cName, mName, "state file missing or inaccessible in " + directory);
- remove(directory);
+ return;
}
- } catch (Exception e) {
- WsLog.error(cName, mName, e);
}
- }
-
- private void update(String user, String directory, boolean overwrite) {
- update(user, directory, overwrite, null);
- }
-
- private void update(String user, String directory, boolean overwrite, IDuccWork work) {
- String mName = "update";
- try {
- if (overwrite) {
- update(user, directory, work);
- } else {
- if (!containsKey(directory)) {
- update(user, directory, work);
- } else {
- WsLog.trace(cName, mName, "duplicate directory: " + directory);
- }
+
+ // Load or reload changed state file
+ String contents = ExperimentsRegistryUtilities.readFile(user, stateFile);
+ if (contents == null) {
+ WsLog.warn(cName, mName, "State file not found " + stateFile.getAbsolutePath());
+ return;
+ }
+ // Ignore version prefix - old non-prefixed files had a larger "stale" threshold
+ int offset = contents.indexOf('[');
+ if (offset > 0) {
+ contents = contents.substring(offset);
+ }
+ // Load from a Json array of Tasks
+ StringReader sr = new StringReader(contents);
+ Type typeOfTaskList = new TypeToken<ArrayList<Task>>(){}.getType();
+ try {
+ ArrayList<Task> taskArray = gson.fromJson(sr, typeOfTaskList);
+ IExperiment experiment = new Experiment(user, directory, fileTime, taskArray, work);
+ IExperiment oldExperiment = experimentsByDir.put(directory, experiment);
+ if (oldExperiment != null) {
+ experiment.updateJedId(oldExperiment.getJedDuccId()); // Ensure the new instance has the latest DuccId
}
-
- } catch (Exception e) {
- WsLog.error(cName, mName, e);
+ } catch (JsonParseException e) {
+ WsLog.warn(cName, mName, "Ignoring " + stateFile + " as has Json syntax error " + e.getMessage());
}
}
-
- private void check() {
- String mName = "check";
- try {
- ArrayList<IExperiment> list = new ArrayList<IExperiment>();
- synchronized (map) {
- for (Entry<String, IExperiment> entry : map.entrySet()) {
- IExperiment experiment = entry.getValue();
- if (experiment.isActive()) {
- list.add(experiment);
- }
- }
- }
- for (IExperiment experiment : list) {
- String user = experiment.getUser();
- String directory = experiment.getDirectory();
- WsLog.debug(cName, mName, "user: " + user + " " + "dirextory: " + directory);
- update(experiment.getUser(), experiment.getDirectory(), true);
- }
- } catch (Exception e) {
- WsLog.error(cName, mName, e);
+
+ // Create map ordered by experiment: active, newest start date, directory
+ public TreeMap<IExperiment, String> getMapByStatus() {
+ TreeMap<IExperiment, String> mapInverse = new TreeMap<IExperiment, String>();
+ for (Entry<String, IExperiment> entry : experimentsByDir.entrySet()) {
+ mapInverse.put(entry.getValue(), entry.getKey());
}
+ return mapInverse;
}
- // Prune if map is large or after every 3rd update
- private boolean timeToPrune(int size) {
- boolean retVal = false;
- if (size > MAX_CACHE_SIZE) {
- retVal = true;
- }
- if ((updateCounter.get() % 3) == 0) {
- retVal = true;
+ // Create map from file last update time to directory ordered oldest first
+ private TreeMap<Long, String> getMapByDate() {
+ TreeMap<Long, String> mapByDate = new TreeMap<Long, String>();
+ for (Entry<String, IExperiment> entry : experimentsByDir.entrySet()) {
+ IExperiment experiment = entry.getValue();
+ mapByDate.put(experiment.getFileDate(), entry.getKey());
}
- return retVal;
+ return mapByDate;
}
- // Keep an experiment if it is active or if cache has room
- // i.e. prune entries with the oldest start-time if not active and cache is full
- // ?? could prune a long-running active one ??
+ // Keep all active experiments plus the newer inactive ones.
+ // i.e. prune inactive entries with the oldest start-time
private void prune() {
String mName = "prune";
- WsLog.enter(cName, mName);
- try {
- TreeMap<Long, String> mapByDate = getMapByDate();
- if (timeToPrune(mapByDate.size())) {
- int cacheCount = 0;
- for (Entry<Long, String> entry : mapByDate.entrySet()) {
- String key = entry.getValue();
- IExperiment experiment = getById(key);
- if (experiment != null) {
- if (experiment.isActive()) {
- cacheCount++;
- } else {
- if (cacheCount < MAX_CACHE_SIZE) {
- cacheCount++;
- } else {
- String directory = experiment.getDirectory();
- remove(directory);
- }
- }
- }
- }
- }
- } catch (Exception e) {
- WsLog.error(cName, mName, e);
- }
- // WsLog.exit(cName, mName);
- }
-
- // Called by DuccPlugins for each OR publication
- public void update(IDuccWorkMap dwm) {
- String mName = "update";
- if (!enabled)
+ int excess = experimentsByDir.size() - MAX_CACHE_SIZE;
+ if (excess <= 0) {
return;
- WsLog.enter(cName, mName);
- try {
- if (dwm == null) {
- WsLog.warn(cName, mName, "missing map");
- } else {
- updateCounter.incrementAndGet();
- Iterator<DuccId> iterator = dwm.getJobKeySet().iterator();
- while (iterator.hasNext()) {
- DuccId duccId = iterator.next();
- IDuccWork job = dwm.findDuccWork(duccId);
- if (job != null) {
- IDuccStandardInfo stdInfo = job.getStandardInfo();
- if (stdInfo != null) {
- String user = stdInfo.getUser();
- String directory = stdInfo.getLogDirectory();
- String parent = ExperimentsRegistryUtilities.upOne(directory);
- update(user, parent, true);
- }
- }
- }
-
- // Also process managed reservations in case the experiment has only these.
- iterator = dwm.getManagedReservationKeySet().iterator();
- while (iterator.hasNext()) {
- DuccId duccId = iterator.next();
- IDuccWork job = dwm.findDuccWork(duccId);
- if (job != null) {
- IDuccStandardInfo stdInfo = job.getStandardInfo();
- if (stdInfo != null) {
- String user = stdInfo.getUser();
- String directory = stdInfo.getLogDirectory();
- String experimentDirectory = stdInfo.getExperimentDirectory();
- if (experimentDirectory != null) {
- update(user, experimentDirectory, true, job);
- } else {
- directory = ExperimentsRegistryUtilities.upOne(directory);
- update(user, directory, true, null);
- }
- }
- }
+ }
+ // Create map sorted by filetime, oldest first and discard the first "excess" inactive entries
+ WsLog.info(cName, mName, "Pruning " + excess + " old experiments");
+ TreeMap<Long, String> mapByDate = getMapByDate();
+ for (String directory : mapByDate.values()) {
+ IExperiment experiment = experimentsByDir.get(directory);
+ if (!experiment.isActive()) {
+ WsLog.info(cName, mName, "Pruned " + directory + " filetime " + experiment.getFileDate());
+ experimentsByDir.remove(directory);
+ if (--excess <= 0) {
+ break;
}
}
- check();
- prune();
- } catch (Exception e) {
- WsLog.error(cName, mName, e);
}
- // WsLog.exit(cName, mName);
+ WsLog.info(cName, mName, "Pruned map size: " + experimentsByDir.size());
}
+
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryUtilities.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryUtilities.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryUtilities.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/ExperimentsRegistryUtilities.java Fri Oct 25 21:22:04 2019
@@ -18,11 +18,8 @@
*/
package org.apache.uima.ducc.ws.xd;
-import java.io.BufferedReader;
-import java.io.Closeable;
import java.io.File;
-import java.io.FileReader;
-import java.io.StringReader;
+import java.nio.file.Files;
import java.util.Arrays;
import java.util.Map.Entry;
@@ -43,143 +40,47 @@ public class ExperimentsRegistryUtilitie
// NOTE - this variable used to hold the class name before WsLog was simplified
private static DuccLogger cName = DuccLogger.getLogger(ExperimentsRegistryUtilities.class);
- private static String stateFileName = "Experiment.state";
-
- public static String upOne(String directory) {
- String retVal = directory;
- if (directory != null) {
- retVal = new File(directory).getParent();
- }
- return retVal;
- }
-
- public static String getStateFilePath(String directory) {
- String retVal = new File(directory, stateFileName).getAbsolutePath();
- return retVal;
- }
-
- private static void closer(Closeable object) {
- String mName = "closer";
- try {
- object.close();
- } catch (Exception e) {
- WsLog.debug(cName, mName, e);
- }
- }
-
- public static long getFileDate(IExperiment experiment) {
- String mName = "getFileDate";
- WsLog.enter(cName, mName);
- String user = experiment.getUser();
- String filename = getStateFilePath(experiment.getDirectory());
- long retVal = getFileDate(user, filename);
- return retVal;
- }
-
- public static long getFileDate(String user, String filename) {
- String mName = "getFileDate";
- WsLog.enter(cName, mName);
- long retVal = getDomesticFileDate(user, filename);
- if (retVal == 0) {
- retVal = getAlienFileDate(user, filename);
- }
- WsLog.exit(cName, mName);
- return retVal;
- }
-
- private static long getAlienFileDate(String user, String filename) {
- String mName = "getAlienFileDate";
- WsLog.enter(cName, mName);
- long retVal = 0;
- AlienFile alienFile = new AlienFile(user, filename);
- // NOTE - should not need the "--" ... or this could be moved to AlienFile
- String[] lines = alienFile.getResult(false, "--", "/bin/ls", "-l", "--time-style=+%s",
- filename);
- // Should have 1 line with secs-since-epoch in 6th token
- if (lines.length == 1) {
- String[] toks = lines[0].split("\\s+");
- if (toks.length >= 6) {
- retVal = Long.valueOf(toks[5]) * 1000;
+ /*
+ * Get the file timestamp ... return 0 if file doesn't exist
+ */
+ public static long getFileTime(String user, File stateFile) {
+ if (stateFile.canRead()) {
+ return stateFile.lastModified();
+ }
+ // May be unreadable by DUCC or may not exist!
+ // Use duccling to get time in "date" style seconds
+ String[] cmd = { "/bin/ls", "-l", "--time-style=+%s", stateFile.getAbsolutePath() };
+ String result = DuccAsUser.execute(user, null, cmd);
+ String[] toks = result.split("\\s+");
+ // If file doesn't exist 6-th token will be part of an error msg
+ if (toks.length >= 6) {
+ try {
+ return Long.valueOf(toks[5]) * 1000;
+ } catch (NumberFormatException e) {
}
}
- WsLog.exit(cName, mName);
- return retVal;
- }
+ return 0;
- private static long getDomesticFileDate(String user, String filename) {
- String mName = "getDomesticFileDate";
- WsLog.enter(cName, mName);
- long retVal = 0;
- try {
- File file = new File(filename);
- retVal = file.lastModified();
- } catch (Exception e) {
- WsLog.trace(cName, mName, e);
- }
- WsLog.exit(cName, mName);
- return retVal;
}
/*
- * Returns null if file is missing or can't be read
+ * Read the Experiment.state file ... as the user if necessary
*/
- public static String getFileContents(String user, String filename) {
- String mName = "getFileContents";
- WsLog.enter(cName, mName);
- boolean canRead = ((new File(filename)).canRead());
- String retVal = canRead ? getDomesticFileContents(user, filename)
- : getAlienFileContents(user, filename);
- WsLog.exit(cName, mName);
- return retVal;
- }
-
- private static String getAlienFileContents(String user, String filename) {
- String mName = "getAlienFileContents";
- WsLog.enter(cName, mName);
- String retVal = null;
- try {
- AlienFile alienFile = new AlienFile(user, filename);
- retVal = alienFile.getString();
- } catch (Throwable t) {
- WsLog.trace(cName, mName, t);
- }
-
- WsLog.exit(cName, mName);
- return retVal;
- }
-
- private static String getDomesticFileContents(String user, String filename) {
- String mName = "getDomesticFileContents";
- WsLog.enter(cName, mName);
- String retVal = null;
- FileReader fr = null;
- BufferedReader br = null;
- StringReader sr = null;
+ public static String readFile(String user, File file) {
+ String mName = "readFile";
try {
- fr = new FileReader(filename);
- br = new BufferedReader(fr);
- StringBuffer sb = new StringBuffer();
- String line = br.readLine();
- while (line != null) {
- sb.append(line);
- line = br.readLine();
+ if (file.canRead()) {
+ byte[] bytes = Files.readAllBytes(file.toPath());
+ return new String(bytes);
+ } else {
+ AlienFile alienFile = new AlienFile(user, file.getAbsolutePath());
+ return alienFile.getString();
}
- retVal = sb.toString();
} catch (Exception e) {
- WsLog.debug(cName, mName, e);
- } finally {
- if (br != null) {
- closer(br);
- }
- if (fr != null) {
- closer(fr);
- }
- if (sr != null) {
- closer(sr);
- }
+ WsLog.error(cName, mName, "Failed to read file " + file.getAbsoluteFile());
+ WsLog.error(cName, mName, e);
+ return null;
}
- WsLog.exit(cName, mName);
- return retVal;
}
public static boolean launchJed(IExperiment experiment) {
@@ -249,8 +150,8 @@ public class ExperimentsRegistryUtilitie
"--description", "JED---" + dwj.getStandardInfo().getLogDirectory()
};
- // Update state file AFTER successfully restoring the JED AP from the DB
- if (!experiment.updateStateFile()) {
+ // Update state file with the user's umask AFTER successfully restoring the JED AP from the DB
+ if (!experiment.updateStateFile(dwj.getStandardInfo().getUmask())) {
return false;
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/IExperiment.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/IExperiment.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/IExperiment.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/IExperiment.java Fri Oct 25 21:22:04 2019
@@ -22,41 +22,36 @@ import java.util.ArrayList;
import org.apache.uima.ducc.common.utils.id.DuccId;
-public interface IExperiment extends Comparable<Object> {
- public void setId(String value);
-
- public String getId();
+public interface IExperiment extends Comparable<IExperiment> {
public String getUser();
public String getDirectory();
- public Task[] getTasks();
-
- public ArrayList<String> getJobIds();
+ public ArrayList<Task> getTasks();
public String getStartDate();
- public long getStartTime();
-
public boolean isActive();
public Jed.Status getStatus();
- public void setFileDate(long value);
-
public long getFileDate();
- public int getVersion();
-
public boolean isStale();
- // Set the DuccId of the AP that launched the experiment
- public void setJedDuccId(DuccId duccId);
-
- // Return the DuccId of the AP that launched the experiment, or null
+ /*
+ * Return the DuccId of the AP that launched the experiment, or null
+ */
public DuccId getJedDuccId();
- // Update the Experiment.state file indicating the tasks to be rerun
- public boolean updateStateFile();
+ /*
+ * Update the DuccId of the AP that launched the experiment
+ */
+ public void updateJedId(DuccId duccId);
+
+ /*
+ * Update the Experiment.state file indicating the tasks to be rerun
+ */
+ public boolean updateStateFile(String umask);
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Task.java
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Task.java?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Task.java (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/xd/Task.java Fri Oct 25 21:22:04 2019
@@ -24,7 +24,7 @@ import com.google.gson.annotations.Expos
* The "exposed" fields are those exposed and saved by JED in the Experiment.state file
* See TaskState in the com.ibm.watsonx.framework.jed project
*/
-public class Task {
+public class Task implements Comparable<Task> {
@Expose
public String pathId;
@@ -57,4 +57,9 @@ public class Task {
public long[] duccId;
public boolean rerun = false; // Note - this is NOT in the Experiment.state file
+
+ @Override
+ public int compareTo(Task that) {
+ return this.taskId - that.taskId; // So can sort tasks for display
+ }
}
Modified: uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.local.js
URL: http://svn.apache.org/viewvc/uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.local.js?rev=1868959&r1=1868958&r2=1868959&view=diff
==============================================================================
--- uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.local.js (original)
+++ uima/uima-ducc/trunk/uima-ducc-web/src/main/webapp/root/js/ducc.local.js Fri Oct 25 21:22:04 2019
@@ -252,10 +252,10 @@ function ducc_update_page_local(type)
}
}
-function ducc_toggle_task_state(expid, taskid)
+function ducc_toggle_task_state(taskid)
{
try {
- ducc_load_data("experiment-details", "?id="+expid+"&taskid="+taskid);
+ ducc_load_data("experiment-details", location.search+"&taskid="+taskid);
}
catch(err) {
ducc_error("ducc_toggle_task_state",err);
@@ -273,14 +273,14 @@ function ducc_restart_experiment()
}
}
-function ducc_terminate_experiment(id)
+function ducc_terminate_experiment(directory)
{
try {
$.jGrowl(" Pending termination...");
$.ajax(
{
type: 'POST',
- url : "/ducc-servlet/experiment-cancel-request"+"?id="+id,
+ url : "/ducc-servlet/experiment-cancel-request"+"?dir="+directory,
success : function (data)
{
$.jGrowl(data, { life: 6000 });
@@ -295,12 +295,12 @@ function ducc_terminate_experiment(id)
return false;
}
-function ducc_confirm_terminate_experiment(id,directory)
+function ducc_confirm_terminate_experiment(directory)
{
try {
var result=confirm("Terminate experiment "+directory+"?");
if (result==true) {
- ducc_terminate_experiment(id);
+ ducc_terminate_experiment(directory);
}
}
catch(err) {