You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/07/17 23:06:42 UTC
[87/88] [abbrv] oodt git commit: Fix for OODT-526: Add base
functionality in Curator for Task List persistence (mjoyce, mattmann)
Fix for OODT-526: Add base functionality in Curator for Task List persistence (mjoyce, mattmann)
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/208dde7c
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/208dde7c
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/208dde7c
Branch: refs/heads/feature/zookeeper-config
Commit: 208dde7c1c1ccc613614660d8bb4577e411e8c6b
Parents: 2ff8689
Author: Chris Mattmann <ma...@apache.org>
Authored: Mon Jul 17 14:39:07 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Mon Jul 17 14:39:07 2017 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../cas/curation/service/IngestionResource.java | 146 ++++++++++++++++++-
.../org/apache/oodt/cas/curation/HomePage.html | 2 +-
.../apache/oodt/cas/curation/HomePage_edrn.html | 2 +-
.../curation/workbench/Workbench.js | 4 +
5 files changed, 149 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d4cb042..081d501 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,8 @@ Apache OODT Change Log
Release 1.1 - Current Development
+* OODT-526 Add base functionality in Curator for Task List persistence (mjoyce, mattmann)
+
* OODT-525 Export IngestTaskList from Curator (mjoyce, mattmann)
* OODT-304 Add a Getter method to get the file handler configuration (Michael Cayanan via mattmann)
http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java
----------------------------------------------------------------------
diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java
index 154ddb3..fce9ec1 100644
--- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java
+++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/IngestionResource.java
@@ -17,6 +17,7 @@
package org.apache.oodt.cas.curation.service;
+import org.apache.oodt.cas.curation.structs.ExtractorConfig;
//OODT imports
import org.apache.oodt.cas.curation.structs.IngestionTask;
import org.apache.oodt.cas.curation.util.DateUtils;
@@ -33,12 +34,15 @@ import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.UUID;
import java.util.Vector;
@@ -50,7 +54,9 @@ import javax.xml.parsers.ParserConfigurationException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
import org.apache.xml.serialize.OutputFormat;
import org.apache.xml.serialize.XMLSerializer;
@@ -89,7 +95,7 @@ public class IngestionResource extends CurationService {
private static final String RESP_SUCCESS = "success";
private IngestionTaskList taskList;
-
+
private String taskListSaveLocPath;
public IngestionResource() {
@@ -130,7 +136,16 @@ public class IngestionResource extends CurationService {
newTask.setPolicy(policy);
newTask.setProductType(productType);
newTask.setStatus(IngestionTask.NOT_STARTED);
- return this.taskList.addIngestionTask(newTask);
+ String addedTaskID = this.taskList.addIngestionTask(newTask);
+ saveTaskListState();
+ return addedTaskID;
+ }
+
+ @GET
+ @Path("init")
+ @Produces("text/plain")
+ public void initTaskList() {
+ loadSavedTaskListState();
}
@GET
@@ -138,6 +153,7 @@ public class IngestionResource extends CurationService {
@Produces("text/plain")
public void removeTask(@QueryParam("taskId") String ingestTaskId) {
this.taskList.removeIngestionTask(ingestTaskId);
+ saveTaskListState();
}
@GET
@@ -195,7 +211,10 @@ public class IngestionResource extends CurationService {
task.setStatus(IngestionTask.FINISHED);
}
- return this.encodeIngestResponseAsHTML(true, null);
+ String response = this.encodeIngestResponseAsHTML(true, null);
+ saveTaskListState();
+ return response;
+
}
private String encodeTaskListAsHTML(List<IngestionTask> taskList) {
@@ -320,6 +339,113 @@ public class IngestionResource extends CurationService {
}
}
+ private void saveTaskListState() {
+ this.taskList.exportTaskListAsXMLToFile(this.taskListSaveLocPath);
+ }
+
+ private void loadSavedTaskListState() {
+ File state = new File(this.taskListSaveLocPath);
+ if (state.exists())
+ parseXMLState(state);
+ }
+
+ private void parseXMLState(File stateFile) {
+ try {
+ Document document = buildXMLDocument(stateFile);
+ rebuildTaskList(document);
+ } catch (Exception e) {
+ LOG.log(Level.WARNING,
+ "parseXMLState: Unable to process saved TaskList state");
+ }
+ }
+
+ private Document buildXMLDocument(File stateFile)
+ throws ParserConfigurationException, SAXException, IOException {
+
+ DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
+ .newInstance();
+ DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
+ Document document = docBuilder.parse(stateFile);
+ document.getDocumentElement().normalize();
+ return document;
+ }
+
+ private void rebuildTaskList(Document document)
+ throws FileNotFoundException, IOException, ParseException {
+
+ NodeList tasks = document.getElementsByTagName("Task");
+
+ for (int i = 0; i < tasks.getLength(); i++) {
+ Element taskElement = (Element) tasks.item(i);
+
+ String id = getNodeValue(taskElement, "ID");
+ String dateString = getNodeValue(taskElement, "CreateDate");
+ List<String> newFileList = getTaskFileList(taskElement);
+ String policy = getNodeValue(taskElement, "Policy");
+ String productType = getNodeValue(taskElement, "ProductType");
+ String status = getNodeValue(taskElement, "Status");
+ String[] extConfParams = getTaskExtractorConfigParams(taskElement);
+
+ IngestionTask newTask = createTaskFromParameters(id, dateString,
+ newFileList, policy, productType, status, extConfParams);
+
+ this.taskList.addIngestionTaskWithoutIdGen(newTask);
+ }
+ }
+
+ private List<String> getTaskFileList(Element parent) {
+ NodeList files = parent.getElementsByTagName("File");
+ List<String> newFileList = new Vector<String>();
+
+ for (int i = 0; i < files.getLength(); i++) {
+ Element fileNode = (Element) files.item(i);
+ newFileList.add(fileNode.getFirstChild().getNodeValue());
+ }
+
+ return newFileList;
+ }
+
+ private String[] getTaskExtractorConfigParams(Element parent) {
+ String[] params = new String[2];
+
+ NodeList extData = parent.getElementsByTagName("ExtractorConfig");
+ Element extDataElm = (Element) extData.item(0);
+ params[0] = extDataElm.getElementsByTagName("UploadPath").item(0)
+ .getFirstChild().getNodeValue();
+ params[1] = extDataElm.getElementsByTagName("ID").item(0).getFirstChild()
+ .getNodeValue();
+
+ return params;
+ }
+
+ private IngestionTask createTaskFromParameters(String id, String dateString,
+ List<String> fileList, String policy, String productType, String status,
+ final String[] extConfParams)
+ throws FileNotFoundException, IOException, ParseException {
+
+ Date date = new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy",
+ Locale.ENGLISH).parse(dateString);
+ ExtractorConfig extConf = ExtractorConfigReader
+ .readFromDirectory(new File(extConfParams[0]), extConfParams[1]);
+
+ IngestionTask newTask = new IngestionTask();
+ newTask.setId(id);
+ newTask.setCreateDate(date);
+ newTask.setFileList(fileList);
+ newTask.setPolicy(policy);
+ newTask.setProductType(productType);
+ newTask.setStatus(status);
+ newTask.setExtConf(extConf);
+
+ return newTask;
+ }
+
+ private String getNodeValue(Element element, String tag) {
+ NodeList nodeList = element.getElementsByTagName(tag).item(0)
+ .getChildNodes();
+ return ((Node) nodeList.item(0)).getNodeValue();
+ }
+
class IngestionTaskList {
private ConcurrentHashMap<String, IngestionTask> taskMap;
@@ -328,6 +454,10 @@ public class IngestionResource extends CurationService {
this.taskMap = new ConcurrentHashMap<String, IngestionTask>();
}
+ public int getNumberOfTasks() {
+ return taskMap.size();
+ }
+
public synchronized String addIngestionTask(IngestionTask task) {
this.provideTaskId(task);
taskMap.put(task.getId(), task);
@@ -338,6 +468,12 @@ public class IngestionResource extends CurationService {
taskMap.remove(taskId);
}
+ public synchronized String addIngestionTaskWithoutIdGen(
+ IngestionTask task) {
+ taskMap.put(task.getId(), task);
+ return task.getId();
+ }
+
public IngestionTask getIngestionTaskById(String taskId) {
return taskMap.get(taskId);
}
@@ -360,7 +496,7 @@ public class IngestionResource extends CurationService {
return taskList;
}
- //TODO: write method to load ingest task list from XML file
+ // TODO: write method to load ingest task list from XML file
public void exportTaskListAsXMLToFile(String fileName) {
try {
Document xmlDocument = generateXMLDocument();
http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html
----------------------------------------------------------------------
diff --git a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html
index a0d8911..cba9178 100644
--- a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html
+++ b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage.html
@@ -27,7 +27,7 @@
</wicket:remove>
</head>
-<body>
+<body onload="initIngestTaskList();refreshIngestTaskList();registerIngestionTaskListener();">
<div id="page">
<div id="cas-header">
<div id="cas-logo-left"><!-- cas logo lhs --></div>
http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html
----------------------------------------------------------------------
diff --git a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html
index e017835..a1f7875 100644
--- a/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html
+++ b/curator/webapp/src/main/java/org/apache/oodt/cas/curation/HomePage_edrn.html
@@ -20,7 +20,7 @@
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
</head>
-<body>
+<body onload="initIngestTaskList();refreshIngestTaskList();registerIngestionTaskListener();">
<div id="page">
<div id="ncibanner">
http://git-wip-us.apache.org/repos/asf/oodt/blob/208dde7c/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js
----------------------------------------------------------------------
diff --git a/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js b/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js
index 3d0601f..65f658e 100644
--- a/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js
+++ b/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/curation/workbench/Workbench.js
@@ -320,6 +320,10 @@ function createIngestionTaskRow(data) {
$('#ingestionTaskListItems > tbody').replace(data);
}
+function initIngestTaskList() {
+ $.get('./services/ingest/init');
+}
+
function removeIngestionTask(id) {
$.get('./services/ingest/remove',
{ 'taskId' : id },