You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/02/18 23:39:52 UTC
[6/6] oodt git commit: update files for new curator
update files for new curator
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/a47b088a
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/a47b088a
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/a47b088a
Branch: refs/heads/staging-curator
Commit: a47b088a8dcc42677d6596692d84c429e5a607c6
Parents: b1c5560
Author: Tom Barber <to...@analytical-labs.com>
Authored: Sat Feb 18 23:39:13 2017 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Sat Feb 18 23:39:13 2017 +0000
----------------------------------------------------------------------
catalog/pom.xml | 10 +-
.../commons/validation/DirectoryValidator.java | 31 +
.../commons/validation/ValidationElement.java | 60 ++
.../commons/validation/ValidationOutput.java | 62 ++
core/pom.xml | 43 +-
crawler/pom.xml | 10 +-
curator2/pom.xml | 182 ++++
.../curation/configuration/Configuration.java | 98 ++
.../oodt/cas/curation/directory/Directory.java | 15 +
.../curation/directory/DirectoryListing.java | 98 ++
.../cas/curation/directory/FileDirectory.java | 40 +
.../oodt/cas/curation/ingest/IngestBackend.java | 182 ++++
.../cas/curation/ingest/IngestException.java | 18 +
.../oodt/cas/curation/ingest/InputStruct.java | 19 +
.../oodt/cas/curation/ingest/OutputStruct.java | 18 +
.../metadata/FlatDirMetadataHandler.java | 63 ++
.../cas/curation/metadata/MetadataBackend.java | 178 ++++
.../cas/curation/metadata/MetadataHandler.java | 33 +
.../cas/curation/rest/DirectoryBackend.java | 111 +++
.../curation/rest/ExceptionResponseHandler.java | 52 +
.../oodt/cas/curation/rest/IngestRest.java | 111 +++
.../oodt/cas/curation/rest/MetadataRest.java | 139 +++
.../oodt/cas/curation/rest/UploadBackend.java | 79 ++
.../oodt/cas/curation/rest/ValidationRest.java | 65 ++
.../cas/curation/service/CurationService.java | 171 ++++
.../curation/service/CurationServiceConfig.java | 227 +++++
.../cas/curation/service/DirectoryResource.java | 138 +++
.../cas/curation/service/IngestionResource.java | 352 +++++++
.../cas/curation/service/MetadataResource.java | 970 +++++++++++++++++++
.../cas/curation/service/PolicyResource.java | 310 ++++++
.../cas/curation/service/SystemResource.java | 69 ++
.../curation/servlet/CuratorConfMetKeys.java | 56 ++
.../cas/curation/servlet/CuratorServlet.java | 28 +
.../cas/curation/structs/ExtractorConfig.java | 78 ++
.../cas/curation/structs/IngestionTask.java | 168 ++++
.../curation/structs/IngestionTaskStatus.java | 36 +
.../curation/util/CurationXmlStructFactory.java | 114 +++
.../oodt/cas/curation/util/DateUtils.java | 59 ++
.../curation/util/ExtractorConfigReader.java | 58 ++
.../curation/util/ExtractorConfigWriter.java | 51 +
.../apache/oodt/cas/curation/util/SSOUtils.java | 60 ++
.../curation/validation/ValidationBackend.java | 128 +++
.../validation/ValidationException.java | 21 +
.../site/resources/images/basic_extractor.jpg | Bin 0 -> 53280 bytes
.../src/site/resources/images/basic_filemgr.jpg | Bin 0 -> 72096 bytes
.../src/site/resources/images/basic_ingest.jpg | Bin 0 -> 48213 bytes
.../src/site/resources/images/basic_login.jpg | Bin 0 -> 26935 bytes
.../src/site/resources/images/basic_page.jpg | Bin 0 -> 26115 bytes
.../src/site/resources/images/basic_staging.jpg | Bin 0 -> 27100 bytes
.../src/site/resources/images/cas-curator.jpg | Bin 0 -> 27536 bytes
.../src/site/resources/images/cas-curator.psd | Bin 0 -> 1003720 bytes
.../src/site/resources/media/Bach-SuiteNo2.mp3 | Bin 0 -> 4781080 bytes
curator2/src/site/site.xml | 32 +
curator2/src/site/xdoc/api/index.xml | 105 ++
curator2/src/site/xdoc/development/maven.xml | 175 ++++
curator2/src/site/xdoc/user/advanced.xml | 56 ++
curator2/src/site/xdoc/user/basic.xml | 690 +++++++++++++
filemgr/pom.xml | 26 +-
.../oodt/cas/filemgr/catalog/Catalog.java | 2 +-
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 534 +++++-----
.../oodt/cas/filemgr/catalog/SpacerCatalog.java | 312 ++++++
.../filemgr/catalog/SpacerCatalogFactory.java | 62 ++
.../catalog/solr/DefaultProductSerializer.java | 42 +-
.../cas/filemgr/catalog/solr/SolrCatalog.java | 46 +-
.../cas/filemgr/catalog/solr/SolrClient.java | 92 +-
.../datatransfer/LocalDataTransferer.java | 49 +-
.../oodt/cas/filemgr/structs/Element.java | 24 +-
.../filemgr/system/CommonsXmlRpcTransport.java | 183 ++++
.../filemgr/system/XmlRpcFileManagerClient.java | 66 +-
.../oodt/cas/filemgr/tools/SolrIndexer.java | 55 +-
.../cas/filemgr/util/XmlRpcStructFactory.java | 2 +-
.../oodt/cas/filemgr/util/XmlStructFactory.java | 11 +-
.../cli/action/TestLuceneQueryCliAction.java | 8 +-
filemgr/src/test/resources/filemgr.properties | 5 +-
.../MetadataProvidedMetExtractor.java | 32 +
pom.xml | 2 +
profile/pom.xml | 13 +-
protocol/http/pom.xml | 10 +-
resource/pom.xml | 10 +-
.../resource/system/CommonsXmlRpcTransport.java | 181 ++++
.../system/XmlRpcResourceManagerClient.java | 79 +-
sso/pom.xml | 13 +-
.../oodt/security/sso/opensso/SSOProxy.java | 134 ++-
workflow/pom.xml | 10 +-
84 files changed, 7375 insertions(+), 497 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index a957174..7c79eab 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -76,8 +76,14 @@
</exclusions>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java b/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java
new file mode 100644
index 0000000..f62a520
--- /dev/null
+++ b/commons/src/main/java/org/apache/oodt/commons/validation/DirectoryValidator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.commons.validation;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ * Interface for Directory Validation inside CAS Curator
+ *
+ * @author tbarber
+ */
+public interface DirectoryValidator {
+
+ ValidationOutput validate(File f, Map<String,String> stagingpath);
+
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java
new file mode 100644
index 0000000..3bed81d
--- /dev/null
+++ b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationElement.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.commons.validation;
+
+/**
+ * @author tbarber
+ */
+public class ValidationElement {
+
+ String field;
+ boolean valid;
+ String message;
+
+ public ValidationElement() {
+ }
+
+ public ValidationElement(String field, boolean valid, String message) {
+ this.field = field;
+ this.valid = valid;
+ this.message = message;
+ }
+
+ public String getField() {
+ return field;
+ }
+
+ public void setField(String field) {
+ this.field = field;
+ }
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public void setValid(boolean valid) {
+ this.valid = valid;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java
new file mode 100644
index 0000000..92fb3be
--- /dev/null
+++ b/commons/src/main/java/org/apache/oodt/commons/validation/ValidationOutput.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.commons.validation;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Validation Response for serialization back to UI.
+ *
+ * @author tbarber
+ */
+public class ValidationOutput {
+
+ String validationpath;
+ List<ValidationElement> validationelements = new ArrayList<ValidationElement>();
+ boolean valid = true;
+
+ public ValidationOutput() {
+ }
+
+ public String getPath() {
+ return validationpath;
+ }
+
+ public void setPath(String path) {
+ this.validationpath = path;
+ }
+
+ public List<ValidationElement> getValidationelements() {
+ return validationelements;
+ }
+
+ public String getValidationpath() {
+ return validationpath;
+ }
+
+ public void addValidationResult(String field, boolean valid, String message){
+ ValidationElement validationElement = new ValidationElement(field, valid, message);
+ this.validationelements.add(validationElement);
+ if(!valid){
+ this.valid=false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index ad4c77f..35f3a70 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,11 +70,6 @@ the License.
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
@@ -123,14 +118,9 @@ the License.
<version>1.2.1</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
- </dependency>
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>1.4</version>
+ <version>2.4</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
@@ -140,7 +130,7 @@ the License.
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
- <version>1.0.3</version>
+ <version>1.2</version>
</dependency>
<dependency>
<groupId>commons-pool</groupId>
@@ -156,6 +146,12 @@ the License.
<groupId>edu.ucar</groupId>
<artifactId>netcdf4</artifactId>
<version>4.5.5</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>edu.ucar</groupId>
@@ -281,7 +277,13 @@ the License.
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2.5</version>
+ <version>4.5.2</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
@@ -406,7 +408,12 @@ the License.
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>1.3.0</version>
+ <version>6.2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>6.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
@@ -506,14 +513,14 @@ the License.
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
-<!-- <dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j</artifactId>
+ <artifactId>slf4j-log4j12</artifactId>
<version>1.7.12</version>
- </dependency>-->
+ </dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <artifactId>jcl-over-slf4j</artifactId>
<version>1.7.12</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/crawler/pom.xml
----------------------------------------------------------------------
diff --git a/crawler/pom.xml b/crawler/pom.xml
index 1fa731e..2bde05b 100644
--- a/crawler/pom.xml
+++ b/crawler/pom.xml
@@ -70,8 +70,14 @@ the License.
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/pom.xml
----------------------------------------------------------------------
diff --git a/curator2/pom.xml b/curator2/pom.xml
new file mode 100644
index 0000000..fa2017e
--- /dev/null
+++ b/curator2/pom.xml
@@ -0,0 +1,182 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more contributor
+license agreements. See the NOTICE.txt file distributed with this work for
+additional information regarding copyright ownership. The ASF licenses this
+file to you under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-core</artifactId>
+ <version>0.13-SNAPSHOT</version>
+ <relativePath>../core/pom.xml</relativePath>
+ </parent>
+ <artifactId>cas-curator-engine</artifactId>
+ <packaging>jar</packaging>
+ <name>CAS Curation Engine</name>
+ <description>A web application for managing policy for products and files and metadata that have been ingested via the CAS component.</description>
+ <build>
+ <plugins />
+ </build>
+ <profiles>
+ <profile>
+ <id>audit</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>rat-maven-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/src/MIT-License.txt</exclude>
+ <exclude>**/media/Bach-SuiteNo2.mp3</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>verify</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+ <repositories>
+ <repository>
+ <id>ibiblio</id>
+ <name>Ibiblio Repository</name>
+ <layout>default</layout>
+ <url>http://mirrors.ibiblio.org/pub/mirrors/maven2</url>
+ <releases>
+ <enabled>true</enabled>
+ <updatePolicy>always</updatePolicy>
+ <checksumPolicy>warn</checksumPolicy>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>maven2-repository.dev.java.net</id>
+ <name>Java.net Repository for Maven</name>
+ <url>http://download.java.net/maven/2/</url>
+ <layout>default</layout>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>1.17</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-commons</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <version>1.2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.3</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ <version>3.2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <version>2.4</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-filemgr</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-sso</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-crawler</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>tika</artifactId>
+ <groupId>org.apache.tika</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-metadata</artifactId>
+ <version>${project.parent.version}</version>
+ <type>jar</type>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.json-lib</groupId>
+ <artifactId>json-lib</artifactId>
+ <version>2.3</version>
+ <classifier>jdk15</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cxf</groupId>
+ <artifactId>cxf-rt-frontend-jaxrs</artifactId>
+ <version>3.1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cxf</groupId>
+ <artifactId>cxf-rt-transports-http</artifactId>
+ <version>3.1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cxf</groupId>
+ <artifactId>cxf-rt-transports-http-jetty</artifactId>
+ <version>3.1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.cxf</groupId>
+ <artifactId>cxf-rt-management</artifactId>
+ <version>3.1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.3.1</version>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java b/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java
new file mode 100644
index 0000000..d828e5d
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/configuration/Configuration.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.curation.configuration;
+
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.servlet.ServletContext;
+
+import org.apache.oodt.cas.metadata.util.PathUtils;
+
+/**
+ * Singleton to hold configuration for the curation
+ * @author starchmd
+ */
+public class Configuration {
+ //Config names
+ public static final String FILLER_METDATA_KEY = "fill";
+ public static final String UPLOAD_AREA_CONFIG = "org.apache.oodt.cas.curator.upload.area";
+ public static final String STAGING_AREA_CONFIG = "org.apache.oodt.cas.curator.staging.area";
+ public static final String ARCHIVE_AREA_CONFIG = "org.apache.oodt.cas.curator.archive.area";
+ public static final String METADATA_AREA_CONFIG = "org.apache.oodt.cas.curator.metadata.area";
+ public static final String EXTRACTOR_AREA_CONFIG = "org.apache.oodt.cas.curator.extractor.area";
+ public static final String FILEMANAGER_URL_CONFIG = "org.apache.oodt.cas.curator.filemanager.url";
+ public static final String FILEMANAGER_PROP_CONFIG = "org.apache.oodt.cas.curator.filemanager.prop";
+ public static final String DIRECTORYBACKEND_VALIDATOR = "org.apache.oodt.cas.curator.directory.validator";
+ public static final String POLICY_UPLOAD_PATH = "org.apache.oodt.cas.curator.dataDefinition.uploadPath";
+
+ //Stores the configuration object as a Properties object
+ private static Properties config = new Properties();
+
+ /**
+ * Loads a configuration given a servlet context
+ * @param context - servlet context
+ */
+ @SuppressWarnings("unchecked")
+ public static void loadConfiguration(ServletContext context) {
+ //TODO: make sure all above values are loaded
+ Enumeration<String> inits = context.getInitParameterNames();
+ while(inits.hasMoreElements()) {
+ String key = inits.nextElement();
+ set(key,context.getInitParameter(key));
+ }
+ }
+ /**
+ * Accessor to wrap properties object implementation
+ * @param key - configuration key to access
+ * @return value of configuration
+ */
+ public static String get(String key) {
+ String ret = config.getProperty(key);
+ return ret;
+ }
+ /**
+ * Get key with replacement of env vars
+ * @param key - key to get
+ * @return value with [VAR] replaced
+ */
+ public static String getWithReplacement(String key) {
+ String ret = PathUtils.replaceEnvVariables(get(key));
+ return ret;
+ }
+ /**
+ * Modifier for global configuration, private to prevent tampering
+ * @param key - key to modify
+ * @param val - value to modify
+ */
+ private static void set(String key,String val) {
+ config.setProperty(key, val);
+ }
+
+ public static Map<String,String> getAllProperties() {
+ Enumeration e = config.propertyNames();
+ Map<String,String> generatedproperties = new HashMap<String, String>();
+ while (e.hasMoreElements()) {
+ String key = (String) e.nextElement();
+ generatedproperties.put(key, getWithReplacement(key));
+ }
+ return generatedproperties;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java
new file mode 100644
index 0000000..fd4e3a2
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/Directory.java
@@ -0,0 +1,15 @@
+package org.apache.oodt.cas.curation.directory;
+
+/**
+ * Anything that can be a directory listed by curator. Allows implementation of
+ * different backends to the staging area (not just file system).
+ *
+ * @author starchmd
+ */
+public interface Directory {
+ /**
+ * Perform a listing of this directory.
+ * @return - directory listing
+ */
+ public DirectoryListing list() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java
new file mode 100644
index 0000000..3c8e0f1
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/DirectoryListing.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.curation.directory;
+
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.commons.validation.DirectoryValidator;
+import org.apache.oodt.commons.validation.ValidationOutput;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * A directory listing object
+ *
+ * @author starchmd
+ */
+public class DirectoryListing {
+ //Types of directory objects
+ public enum Type {
+ DIRECTORY,
+ OBJECT
+ }
+ public static final String ROOT_NAME = "Root";
+ //Attributes of node
+ DirectoryListing.Type type;
+ String name;
+ String path;
+ ValidationOutput validation;
+
+ //Children listings (only valid for directory types)
+ List<DirectoryListing> children = new LinkedList<DirectoryListing>();
+ /**
+ * Get a directory listing
+ * @param type - type of listing
+ * @param name - name of object
+ */
+ public DirectoryListing(DirectoryListing.Type type,String name,String path, ValidationOutput validation) {
+ this.name = name;
+ this.type = type;
+ this.path = path;
+ this.children = (type == DirectoryListing.Type.DIRECTORY) ? new LinkedList<DirectoryListing>() : null;
+ this.validation = validation;
+ }
+ /**
+ * Create a directory listing
+ * @param paths - list of file paths
+ * @param validator
+ * @return top-level directory listing object
+ */
+ public static DirectoryListing lisingFromFileObjects(Collection<File> paths, File root,
+ DirectoryValidator validator) {
+ //Shallow copy and sort
+ List<File> copy = new LinkedList<File>(paths);
+ Collections.sort(copy);
+ //Create a stack to hold directories (implementation details)
+ LinkedList<DirectoryListing> stack = new LinkedList<DirectoryListing>();
+ stack.addLast( (root != null && root.isDirectory()) ?
+ new DirectoryListing(DirectoryListing.Type.DIRECTORY,root.getPath(),root.getPath(),
+ validator != null ? validator.validate(root, Configuration.getAllProperties()):null) :
+ new DirectoryListing(DirectoryListing.Type.DIRECTORY,ROOT_NAME,"", null));
+ for (File file : paths) {
+ if (file.equals(root))
+ continue;
+ //Remove all directories off stack until file starts with last's path
+ while (!file.getPath().startsWith(stack.peekLast().path))
+ stack.removeLast();
+ //Get type and name of this file path and create dl object
+ DirectoryListing.Type type = file.isDirectory() ? DirectoryListing.Type.DIRECTORY : DirectoryListing.Type.OBJECT;
+ DirectoryListing dl = new DirectoryListing(type,file.getName(),file.getPath(),
+ validator != null ? validator.validate(file, Configuration.getAllProperties()) :
+ null);
+
+ //Add to last's children
+ stack.peekLast().children.add(dl);
+ if (type == DirectoryListing.Type.DIRECTORY) {
+ stack.addLast(dl);
+ }
+ }
+ return stack.peekFirst();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java
new file mode 100644
index 0000000..ce2d3f6
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/directory/FileDirectory.java
@@ -0,0 +1,40 @@
+package org.apache.oodt.cas.curation.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.oodt.commons.validation.DirectoryValidator;
+
+/**
+ * A file-system based directory for listing into the curator.
+ *
+ * @author starchmd
+ */
+public class FileDirectory implements Directory {
+
+ String directory = null;
+ private DirectoryValidator validator;
+ /**
+ * Build the object around a set directory
+ * @param directory
+ */
+ public FileDirectory(String directory, DirectoryValidator validator) {
+ this.directory = directory;
+ this.validator = validator;
+ }
+ /* (non-Javadoc)
+ * @see org.apache.oodt.cas.curation.directory.Directory#list()
+ */
+ @Override
+ public DirectoryListing list() throws IOException {
+ Collection<File> listing = FileUtils.listFilesAndDirs(new File(directory),TrueFileFilter.INSTANCE,TrueFileFilter.INSTANCE);
+ return DirectoryListing.lisingFromFileObjects(listing,listing.iterator().next(), validator);
+ }
+
+ public void setValidator(DirectoryValidator validator) {
+ this.validator = validator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java
new file mode 100644
index 0000000..c72603e
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestBackend.java
@@ -0,0 +1,182 @@
+package org.apache.oodt.cas.curation.ingest;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.net.URLEncoder;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.cas.curation.ingest.InputStruct.InputEntry;
+import org.apache.oodt.cas.curation.metadata.FlatDirMetadataHandler;
+import org.apache.oodt.cas.curation.rest.IngestRest;
+import org.apache.oodt.cas.filemgr.ingest.Ingester;
+import org.apache.oodt.cas.filemgr.ingest.StdIngester;
+import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * Backend for the ingest service
+ * @author starchmd
+ */
+public class IngestBackend {
+ private static final String DATA_TRANSFER_SERVICE = "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory";
+ private static final String IN_PROGRESS = "IN PROGRESS";
+ private static final String DONE = "DONE";
+ private static final Logger LOG = Logger.getLogger(IngestRest.class.getName());
+
+ private ConcurrentHashMap<String,List<InputEntry>> current = new ConcurrentHashMap<String,List<InputEntry>>();
+
+ private XmlRpcFileManagerClient client = null;
+ private Ingester ingester = null;
+ private URL url = null;
+ /**
+ * Setup this backend
+ * @throws IngestException
+ */
+ public IngestBackend() throws IngestException {
+ try {
+ this.url = new URL(Configuration.getWithReplacement(Configuration.FILEMANAGER_URL_CONFIG));
+ this.ingester = new StdIngester(DATA_TRANSFER_SERVICE);
+ LOG.log(Level.INFO,"Connecting to File Manager at:"+this.url.toString());
+ this.client = new XmlRpcFileManagerClient(this.url);
+ } catch(Exception e) {
+ LOG.log(Level.WARNING,"Error: problem constructing backend: "+e);
+ throw new IngestException("Error: problem setting up ingest backend.",e);
+ }
+ }
+ /**
+ * Ingest based on input
+ * @param input - input struct
+ * @param user - user to isolate requests
+ */
+ public void ingest(InputStruct input,String user) {
+ if (!current.containsKey(user)) {
+ current.put(user, new LinkedList<InputEntry>());
+ }
+ for (InputStruct.InputEntry entry : input.entries) {
+ current.get(user).add(entry);
+ }
+ for (InputStruct.InputEntry entry : input.entries) {
+ try {
+ ingest(entry.file,user);
+ } catch(IngestException e) {
+ entry.error = e;
+ }
+ }
+ }
+ /**
+ * Ingests a single file
+ * @param file - file to ingest
+ * @param user - user to isolate requests
+ * @throws IngestException - error on ingestion
+ */
+ private void ingest(String file, String user) throws IngestException {
+ try {
+ file = URLDecoder.decode(file);
+ File full = null;
+ if (!file.startsWith("/")) {
+ String parent = new File(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG)).getParent();
+ full = new File(parent,file);
+ } else {
+ full = new File(file);
+ }
+ FlatDirMetadataHandler handler = new FlatDirMetadataHandler();
+ Metadata meta = handler.get(file,user);
+ System.out.println("File: "+file+" URL: "+this.url+" Full: "+full.getAbsoluteFile()+" Metadata: "+meta);
+ ingester.ingest(this.url, full.getAbsoluteFile(), meta);
+ //Remove metadata file after successful ingest
+ handler.remove(file,user);
+ org.apache.commons.io.FileUtils.deleteQuietly(full);
+ LOG.log(Level.FINE,"Checking if directory can be removed:"+full.getParent());
+ File p = full.getParentFile();
+ String test = Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG).trim();
+
+ if(test.endsWith("/")){
+ test=test.substring(0, test.length()-1);;
+ }
+ while(p!=null && !p.getAbsolutePath().equals(test))
+ if(p.exists() && p.isDirectory()){
+
+ if(p.list().length==0){
+ FileUtils.deleteDirectory(p);
+ }
+ p = p.getParentFile();
+ }
+ } catch(Exception e) {
+ LOG.log(Level.WARNING,"Error: failed ingesting product: "+e);
+ throw new IngestException("Error: problem while ingesting",e);
+ }
+ }
+ /**
+ * Check the status of the currently ingested items
+ * @return output struct
+ */
+ public OutputStruct status(String user) throws IngestException {
+ try {
+ //List<InputStruct.InputEntry> torm = new LinkedList<InputStruct.InputEntry>();
+ List<OutputStruct.OutputEntry> ret = new LinkedList<OutputStruct.OutputEntry>();
+ if (this.current.get(user) != null) {
+ for (InputStruct.InputEntry entry : current.get(user)) {
+ OutputStruct.OutputEntry temp = new OutputStruct.OutputEntry();
+ temp.file = entry.file;
+ temp.pname = entry.pname;
+ temp.timestamp = entry.timestamp;
+ try {
+
+ if (entry.error != null) {
+ //torm.add(entry);
+ temp.status = "Error: " + entry.error.getMessage();
+ } else if (client.hasProduct(entry.pname)) {
+ //torm.add(entry);
+ temp.product = client.getProductByName(entry.pname).getProductId();
+ temp.status = DONE;
+ } else {
+ temp.status = IN_PROGRESS;
+ }
+ ret.add(temp);
+ }
+ catch (CatalogException e){
+ LOG.log(Level.WARNING,"Error: failed fetching product: "+e.getLocalizedMessage());
+ }
+ }
+ }
+ //for (InputStruct.InputEntry entry : torm) {
+ // this.current.get(user).remove(entry);
+ //}
+ //if (this.current.get(user).isEmpty()) {
+ // this.current.remove(user);
+ //}
+ OutputStruct out = new OutputStruct();
+ out.status = ret;
+ return out;
+ } catch (Exception e) {
+ LOG.log(Level.WARNING,"Error: failed checking status: "+e);
+ throw new IngestException("Error: failed to check status",e);
+ }
+ }
+ /**
+ * Clears any errors registered in this system
+ */
+ public void clearErrors(String user) {
+ List<InputStruct.InputEntry> torm = new LinkedList<InputStruct.InputEntry>();
+ if (this.current.get(user) == null) {
+ return;
+ }
+ for (InputStruct.InputEntry entry : this.current.get(user)) {
+ torm.add(entry);
+ }
+ for (InputStruct.InputEntry entry : torm) {
+ this.current.get(user).remove(entry);
+ }
+ if (this.current.get(user).isEmpty()) {
+ this.current.remove(user);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java
new file mode 100644
index 0000000..55043f2
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/IngestException.java
@@ -0,0 +1,18 @@
+package org.apache.oodt.cas.curation.ingest;
+
+/**
+ * A validation backend exception
+ *
+ * @author starchmd
+ */
+public class IngestException extends Exception {
+ private static final long serialVersionUID = -7714024388845498231L;
+ /**
+ * ctor
+ * @param message - message
+ * @param cause - cause of error
+ */
+ public IngestException(String message,Exception cause) {
+ super(message,cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java
new file mode 100644
index 0000000..42644b2
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/InputStruct.java
@@ -0,0 +1,19 @@
+package org.apache.oodt.cas.curation.ingest;
+
+import java.util.List;
+
+/**
+ * Class to represent input for ingest
+ * @author starchmd
+ */
+public class InputStruct {
+ //public String id;
+ public List<InputStruct.InputEntry> entries;
+ public static class InputEntry {
+ public String pname;
+ public String file;
+ public long size;
+ public long timestamp;
+ public Exception error = null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java
new file mode 100644
index 0000000..e8c6543
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/ingest/OutputStruct.java
@@ -0,0 +1,18 @@
+package org.apache.oodt.cas.curation.ingest;
+
+import java.util.List;
+
+/**
+ * Class representing output
+ * @author selina
+ */
+public class OutputStruct {
+ public List<OutputStruct.OutputEntry> status;
+ public static class OutputEntry {
+ public String file;
+ public String product;
+ public String pname;
+ public String status;
+ public long timestamp;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java
new file mode 100644
index 0000000..c0cb9f4
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/FlatDirMetadataHandler.java
@@ -0,0 +1,63 @@
+/**
+ *
+ */
+package org.apache.oodt.cas.curation.metadata;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.SerializableMetadata;
+
+/**
+ * A metadata handler that uses a directory and fills it with flat-files.
+ * @author starchmd
+ */
+public class FlatDirMetadataHandler implements MetadataHandler {
+ public static final String ENCODING = "UTF-8";
+
+ /* (non-Javadoc)
+ * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#get(java.lang.String)
+ */
+ @Override
+ public Metadata get(String file,String user) throws InstantiationException, FileNotFoundException, IOException {
+ SerializableMetadata met = new SerializableMetadata(ENCODING,false);
+ met.loadMetadataFromXmlStream(new FileInputStream(FlatDirMetadataHandler.getLocation(file,user)));
+ return met;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#set(java.lang.String, org.apache.oodt.cas.metadata.Metadata)
+ */
+ @Override
+ public void set(String file,String user, Metadata metadata) throws FileNotFoundException, IOException {
+ SerializableMetadata ser = new SerializableMetadata(metadata);
+ ser.writeMetadataToXmlStream(new FileOutputStream(FlatDirMetadataHandler.getLocation(file,user)));
+ }
+ /*
+ * (non-Javadoc)
+ * @see org.apache.oodt.cas.curation.metadata.MetadataHandler#remove(java.lang.String)
+ */
+ @Override
+ public void remove(String file,String user) throws FileNotFoundException, IOException {
+ File metfile = FlatDirMetadataHandler.getLocation(file,user);
+ if (metfile.exists()) {
+ if (!metfile.delete()) {
+ throw new IOException("Failed to delete: "+metfile.getAbsolutePath());
+ }
+ }
+ }
+ /**
+ * Gets the flat-dir location for the file's metadata
+ * @param file - file object
+ * @param user - user isolation id
+ * @return file representing location of the flat-file
+ */
+ private static File getLocation(String file,String user) {
+ return new File(Configuration.getWithReplacement(Configuration.METADATA_AREA_CONFIG),user.replace(File.separatorChar, '_')+"-"+file.replace(File.separatorChar, '_')+".met");
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java
new file mode 100644
index 0000000..6bdeb59
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataBackend.java
@@ -0,0 +1,178 @@
+package org.apache.oodt.cas.curation.metadata;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.cas.curation.metadata.FlatDirMetadataHandler;
+import org.apache.oodt.cas.curation.metadata.MetadataHandler;
+import org.apache.oodt.cas.curation.structs.ExtractorConfig;
+import org.apache.oodt.cas.curation.util.ExtractorConfigReader;
+import org.apache.oodt.cas.metadata.MetExtractor;
+import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
+import org.apache.oodt.cas.metadata.extractors.MetadataProvidedMetExtractor;
+import org.apache.oodt.cas.metadata.util.GenericMetadataObjectFactory;
+
+/**
+ * A backend that persists the working metadata entry from the front end.
+ * Also handles metadata extraction, and extractor listing.
+ *
+ * @author starchmd
+ */
+public class MetadataBackend {
+
+ private static final Logger LOG = Logger.getLogger(MetadataBackend.class.getName());
+ private MetadataHandler handler;
+ private final Map<String,ExtractorConfig> extractors = new HashMap<String,ExtractorConfig>();
+ /**
+ * Construct a directory backend with hard-coded directories
+ */
+ public MetadataBackend() {
+ handler = new FlatDirMetadataHandler();
+ }
+ /**
+ * Gets persisted metadata object, and runs extractor if extractor is specified
+ * @param file - file to get metadata from
+ * @param user - user used to prevent cross-talk
+ * @param extractor - if specified, this extractor will be run and replace existing metadata
+ * @return newly constructed metadat object
+ */
+ public Metadata getMetadata(String file, String user, String extractor) throws Exception {
+ LOG.info("Getting metadata for: "+file+" and extractor: "+extractor);
+ Metadata met = null;
+ try {
+ System.out.println("Getting metadata for file: "+file +" with extractor: "+extractor);
+ met = handler.get(file,user);
+ } catch(Exception e) {
+ met = new Metadata();
+ }
+ //If extractor is specified, then its metadata is considered "correct" and previous metadata is used only to fill filler
+ if (extractors.containsKey(extractor)) {
+ System.out.println("Merging");
+ met = extractMergeAndPresist(file,user,extractor,met);
+ }
+ return met;
+ }
+ /**
+ * Persists the metadata for a given file and runs extractor, if specified
+ * @param file - file to specify metadata for
+ * @param user - user used to prevent cross-talk
+ * @param extractor - optional extractor to run
+ * @param metadata - new metadata for file
+ * @return metadata after optional extraction
+ */
+ public Metadata putMetadata(String file, String user, String extractor,Metadata metadata) throws Exception {
+ LOG.info("Putting metadata for: "+file+" and extractor: "+extractor);
+ fineLogMetadata("Put metadata:",metadata);
+ handler.set(file, user, metadata);
+ if (extractors.containsKey(extractor)) {
+ metadata = extractMergeAndPresist(file,user,extractor,metadata);
+ }
+ return metadata;
+ }
+ /**
+ * Deletes the metadata for a given file
+ * @param file - file to specify metadata for
+ * @param user - user used to prevent cross-talk
+ */
+ public void deleteMetadata(String file,String user) throws Exception {
+ LOG.info("Deleting metadata for: "+file);
+ handler.remove(file,user);
+ }
+
+ /**
+ * Returns the list of extractors
+ * @return - list of extractors
+ */
+ public Collection<String> getExtractors() throws Exception {
+ loadMetadataExtractors();
+ return this.extractors.keySet();
+ }
+ /**
+ * Loads the metadata extractors
+ */
+ protected void loadMetadataExtractors() {
+ File directory = new File(Configuration.getWithReplacement(Configuration.EXTRACTOR_AREA_CONFIG));
+ //Load only sub-directories of the extractor config area
+ FilenameFilter filter = new FilenameFilter() {
+ @Override
+ public boolean accept(File current, String name) {
+ return new File(current, name).isDirectory();
+ }
+ };
+ String[] subdirs = directory.list(filter);
+ for (String id : subdirs != null?subdirs:new String[]{}) {
+ try {
+ extractors.put(id,ExtractorConfigReader.readFromDirectory(directory, id));
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Failed to load extractor confugred at:"+new File(directory,id).toString()+" Exception:", e);
+ }
+ }
+ }
+ /**
+ * In both set and get the extractor must be run, results merged, and whole metadata persisted
+ * @param file - file associated with this run
+ * @param user - user used to prevent cross-talk
+ * @param extractor - extractor to run
+ * @param met - metadata object
+ * @return metadata object after merging
+ * @throws Exception
+ */
+ protected Metadata extractMergeAndPresist(String file,String user,String extractor, Metadata met) throws Exception {
+ LOG.info("Running "+extractor+" extractoe and presisting metadat for:"+file);
+ System.out.println("Running Extractor");
+ Metadata extracted = this.runExtractor(file, extractor,met);
+ System.out.println("Replace Metadata");
+ met.replaceMetadata(extracted);
+ fineLogMetadata("Merged metadata:",met);
+ //Persist newly extracted metadata
+ System.out.println("Persisting metadata");
+ handler.set(file, user, met);
+ return met;
+ }
+ /**
+ * If possible, logs metadata values finely
+ */
+ private void fineLogMetadata(String message, Metadata metadata) {
+ if (LOG.isLoggable(Level.FINE)) {
+ for (String key : metadata.getAllKeys()) {
+ message += "\n\t"+key+": "+metadata.getMetadata(key);
+ }
+ LOG.fine(message);
+ }
+ }
+ /**
+ * Runs a metadata extractor on given file
+ * @param file - file to extract metadata from
+ * @param id - id of the extractor to use
+ * @param metadata - metadata to attach to extraction, if possible
+ * @return metadata extracted
+ * @throws MetExtractionException - exception thrown when failure to extract metadata
+ */
+ protected Metadata runExtractor(String file, String id, Metadata metadata) throws MetExtractionException {
+ System.out.println("Here1");
+ String parent = new File(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG)).getParent();
+ File full = new File(parent,file);
+ //Get extractor
+ System.out.println("Here2");
+ ExtractorConfig config = extractors.get(id);
+ System.out.println("Here3");
+ MetExtractor metExtractor = GenericMetadataObjectFactory.getMetExtractorFromClassName(config.getClassName());
+ System.out.println("Here4");
+ metExtractor.setConfigFile(config.getConfigFiles().get(0));
+ System.out.println("Here5");
+ if (metExtractor instanceof MetadataProvidedMetExtractor) {
+ ((MetadataProvidedMetExtractor)metExtractor).attachMetadata(metadata);
+ }
+ System.out.println("Here6");
+ return metExtractor.extractMetadata(full.getAbsolutePath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java
new file mode 100644
index 0000000..05bd592
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/metadata/MetadataHandler.java
@@ -0,0 +1,33 @@
+package org.apache.oodt.cas.curation.metadata;
+
+import org.apache.oodt.cas.metadata.Metadata;
+
+/**
+ * A specific handler for metadata temporary storage. This does not handle the final cataloging,
+ * but acts as a temporaty storage of metadata that the user is curating.
+ *
+ * @author starchmd
+ */
+public interface MetadataHandler {
+ /**
+ * Gets the metadata for a file or null if none exists
+ * @param file - file the metadata describes
+ * @param user - user id to isolate work
+ * @return OODT metadata object
+ * @throws Exception
+ */
+ public Metadata get(String file,String user) throws Exception;
+ /**
+ * Sets the metadata for a file
+ * @param file - file the metadata describes
+ * @param user - user id to isolate work
+ * @param metadata - metadata object to set
+ */
+ public void set(String file,String user,Metadata metadata) throws Exception;
+ /**
+ * Remove metadata storage for file
+ * @param file - file the metadata describes
+ * @param user - user id to isolate work
+ */
+ public void remove(String file,String user) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java
new file mode 100644
index 0000000..3e0c6e5
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/DirectoryBackend.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.curation.rest;
+
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.cas.curation.directory.Directory;
+import org.apache.oodt.cas.curation.directory.FileDirectory;
+import org.apache.oodt.commons.validation.DirectoryValidator;
+
+import com.google.gson.Gson;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+
+/**
+ * A directory backend that returns JSON object representing a directory structure.
+ * Will also list available directory backends.
+ *
+ * @author starchmd
+ */
+@Path("directory")
+public class DirectoryBackend {
+ private DirectoryValidator validator;
+ Map<String,Directory> types = new HashMap<String,Directory>();
+ Gson gson = new Gson();
+ /**
+ * Construct a directory backend with hard-coded directories
+ */
+ public DirectoryBackend() {
+
+ }
+ @GET
+ @Produces("application/json")
+ /**
+ * Get the types of directory backends.
+ */
+ public String getDirectoryTypes() {
+ //TODO: update this loading code to be user-configured and be fed off of "type"
+ if (types.get("files") == null)
+ bootstrapValidator();
+ types.put("files", new FileDirectory(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG),
+ validator));
+ return gson.toJson(types.keySet());
+ }
+
+ @GET
+ @Produces("application/json")
+ @Path("{type}")
+ /**
+ * Returns the listing of the given directory type
+ * @param type - type of directory to list
+ */
+ public String list(@PathParam("type") String type) throws Exception {
+ //TODO: update this loading code to be user-configured and be fed off of "type"
+ if (types.get("files") == null)
+ bootstrapValidator();
+ types.put("files", new FileDirectory(Configuration.getWithReplacement(Configuration.STAGING_AREA_CONFIG),
+ validator));
+ return gson.toJson(types.get(type).list());
+ }
+
+ /**
+ * Initialise the validator engine as defined in web.xml.
+ */
+ private void bootstrapValidator(){
+ if(validator==null) {
+ String vclass = Configuration.getWithReplacement(Configuration.DIRECTORYBACKEND_VALIDATOR);
+ if (vclass != null && !vclass.equals("")) {
+ Class<?> clazz = null;
+ try {
+ clazz = Class.forName(vclass);
+
+ Constructor<?> constructor = clazz.getConstructor();
+ this.validator = (DirectoryValidator) constructor.newInstance();
+
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ } catch (NoSuchMethodException e) {
+ e.printStackTrace();
+ } catch (InstantiationException e) {
+ e.printStackTrace();
+ } catch (IllegalAccessException e) {
+ e.printStackTrace();
+ } catch (InvocationTargetException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java
new file mode 100644
index 0000000..edfd898
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ExceptionResponseHandler.java
@@ -0,0 +1,52 @@
+package org.apache.oodt.cas.curation.rest;
+
+import javax.ws.rs.core.Response;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * Helper class to build response for thrown exception
+ *
+ * @author starchmd
+ */
+public class ExceptionResponseHandler {
+ /*
+ * Static initializer to get GSON converter
+ */
+ private static Gson gson;
+ {
+ gson = new GsonBuilder().create();
+ }
+ /**
+ * Recurse down an exception causality tree
+ * @param e -exception
+ * @return exception info regarding this exception
+ */
+ public static ExceptionInfo recurseExceptionTree(Throwable e) {
+ ExceptionInfo info = new ExceptionInfo();
+ info.name = e.getClass().getName();
+ info.message = e.getLocalizedMessage();
+ info.stack = e.getStackTrace();
+ info.cause = recurseExceptionTree(e.getCause());
+ return info;
+ }
+ /**
+ * Return a response that wraps an exception
+ * @return
+ */
+ public static Response BuildExceptionResponse(Exception e) {
+ ExceptionInfo info = recurseExceptionTree(e);
+ return Response.serverError().entity(gson.toJson(info)).build();
+ }
+ /**
+ * Information regarding an exception
+ * @author starchmd
+ */
+ static class ExceptionInfo {
+ String name;
+ String message;
+ ExceptionInfo cause;
+ StackTraceElement[] stack;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java
new file mode 100644
index 0000000..6efb563
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/IngestRest.java
@@ -0,0 +1,111 @@
+package org.apache.oodt.cas.curation.rest;
+
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.Produces;
+import javax.ws.rs.GET;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.oodt.cas.curation.ingest.IngestBackend;
+import org.apache.oodt.cas.curation.ingest.IngestException;
+import org.apache.oodt.cas.curation.ingest.InputStruct;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * A backend that uses put to initiate ingest
+ *
+ * @author starchmd
+ */
+@Path("ingest")
+public class IngestRest {
+ private static final Logger LOG = Logger.getLogger(IngestRest.class.getName());
+ //GSON serialization object using Google GSON
+ private Gson gson = new GsonBuilder().create();
+ private IngestBackend backend = null;
+ /**
+ * Construct a directory backend with hard-coded directories
+ */
+ public IngestRest() {}
+ @PUT
+ @Consumes("application/json")
+ /**
+ * Runs the ingest
+ * @param user - user to isolate requests
+ * @param extractor - optional extractor to run
+ * @param json - new json for file
+ */
+ public Response ingest(@QueryParam("user") String user,@QueryParam("extractor") String extractor,String json) {
+ try {
+ setup();
+ InputStruct input = gson.fromJson(json, InputStruct.class);
+ List<String> files = new LinkedList<String>();
+ for (InputStruct.InputEntry entry : input.entries) {
+ files.add(entry.file);
+ }
+ LOG.log(Level.INFO, "Ingesting files: "+StringUtils.join(files,","));
+ this.backend.ingest(input,user);
+ return Response.ok().build();
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e);
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ @GET
+ @Produces("application/json")
+ /**
+ * Get ingest status
+ * @return list of statuses
+ */
+ public Response status(@QueryParam("user") String user) {
+ try {
+ setup();
+ LOG.log(Level.INFO, "Reading current status");
+ return Response.ok().entity(gson.toJson(this.backend.status(user))).build();
+ } catch(Exception e) {
+ LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e);
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ @DELETE
+ /**
+ * Delete errors
+ * @return response object
+ */
+ public Response remove(@QueryParam("user") String user) {
+ try {
+ setup();
+ LOG.log(Level.INFO, "Deleteing current errors");
+ this.backend.clearErrors(user);
+ return Response.ok().build();
+ } catch(Exception e) {
+ LOG.log(Level.SEVERE,"Exception occured calling ingest REST endpoint", e);
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ /**
+ * Setup the backend
+ * @throws IngestException
+ */
+ public synchronized void setup() throws IngestException {
+ if (this.backend != null)
+ return;
+ LOG.log(Level.INFO, "Setting up ingest backend");
+ this.backend = new IngestBackend();
+ }
+
+ public IngestBackend getBackend() {
+ return backend;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java
new file mode 100644
index 0000000..8c65a42
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/MetadataRest.java
@@ -0,0 +1,139 @@
+package org.apache.oodt.cas.curation.rest;
+
+import java.util.logging.Logger;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+
+import org.apache.oodt.cas.curation.metadata.MetadataBackend;
+import org.apache.oodt.cas.metadata.Metadata;
+
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.FieldAttributes;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+/**
+ * A backend that puts/gets the metadata of a given file.
+ *
+ * @author starchmd
+ */
+@Path("metadata")
+public class MetadataRest {
+
+ private static final Logger LOG = Logger.getLogger(MetadataRest.class.getName());
+ //GSON serialization object using Google GSON
+ private Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy(){
+ @Override
+ public boolean shouldSkipClass(Class<?> clazz) {
+ return false;
+ }
+ @Override
+ public boolean shouldSkipField(FieldAttributes field) {
+ return field.getName().equals("parent");
+ }}).create();
+ private MetadataBackend backend;
+
+ /**
+ * Construct a directory backend with hard-coded directories
+ */
+ public MetadataRest() {}
+ @GET
+ @Produces("application/json")
+ @Path("{file:.+}")
+ /**
+ * Gets the metadata as JSON, refreshes using an extractor
+ * @param file - file to get metadata from
+ * @param extractor - if specified, this extractor will be run and replace existing metadata
+ */
+ public Response getMetadata(@PathParam("file") String file,@QueryParam("user") String user,@QueryParam("extractor") String extractor) {
+ try {
+ if (file.equals("dev/null"))
+ {
+ file = "/dev/null";
+ }
+ this.setup();
+ System.out.println("Getting metadata for file: "+file);
+ Metadata met = this.backend.getMetadata(file,(user==null)?"":user, extractor);
+ return Response.ok().entity(gson.toJson(met)).build();
+ } catch (Exception e) {
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+
+ @PUT
+ @Consumes("application/json")
+ @Produces("application/json")
+ @Path("{file:.+}")
+ /**
+ * Sets the metadata for a given file
+ * @param file - file to specify metadata for
+ * @param extractor - optional extractor to run
+ * @param json - new json for file
+ */
+ public Response putMetadata(@PathParam("file") String file,@QueryParam("user") String user,@QueryParam("extractor") String extractor,String json) {
+ try {
+ if (file.equals("dev/null"))
+ {
+ file = "/dev/null";
+ }
+ Metadata input = gson.fromJson(json, Metadata.class);
+ this.setup();
+ Metadata met = this.backend.putMetadata(file,(user==null)?"":user, extractor, input);
+ return Response.ok().entity(gson.toJson(met)).build();
+ } catch(Exception e) {
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+
+ }
+ }
+ @DELETE
+ @Path("{file:.+}")
+ /**
+ * Deletes the metadata for a given file
+ * @param file - file to specify metadata for
+ */
+ public Response deleteMetadata(@PathParam("file") String file,@QueryParam("user") String user) {
+ try {
+ if (file.equals("dev/null"))
+ {
+ file = "/dev/null";
+ }
+ this.setup();
+ this.backend.deleteMetadata(file,(user==null)?"":user);
+ return Response.ok().build();
+ } catch(Exception e) {
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ @GET
+ @Produces("application/json")
+ @Path("extractors")
+ /**
+ * Returns the list of extractors
+ * @return - list of extractors
+ */
+ public Response getExtractors() {
+ try {
+ this.setup();
+ return Response.ok().entity(gson.toJson(this.backend.getExtractors())).build();
+ } catch (Exception e) {
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ /**
+ * Setup the backend to call
+ */
+ private synchronized void setup() {
+ if (this.backend != null)
+ return;
+ LOG.info("Setting up metadata backend for metadata rest service");
+ this.backend = new MetadataBackend();
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java
new file mode 100644
index 0000000..07f3633
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/UploadBackend.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.curation.rest;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.logging.Level;
+
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.cxf.jaxrs.ext.multipart.Attachment;
+import org.apache.cxf.jaxrs.ext.multipart.MultipartBody;
+import org.apache.oodt.cas.curation.configuration.Configuration;
+
+/**
+ * A RESTful backend allowing for the user to upload files
+ * to the CAS-Curator.
+ *
+ * @author starchmd
+ */
+@Path("upload")
+public class UploadBackend {
+ /**
+ * Construct using configuration to determine upload area
+ */
+ public UploadBackend() {}
+
+ @POST
+ @Path("file")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ /**
+ * Handles the uploading of the data, by copying all data out of the input stream.
+ * @param mbody - body of the multipart post
+ * @return - OK response on success
+ */
+ public Response upload(MultipartBody mbody) {
+ try {
+ List<Attachment> attachments = mbody.getAllAttachments();
+ for (Attachment attachment : attachments) {
+ String filename = attachment.getContentDisposition().getParameter("filename");
+ try {
+ InputStream in = attachment.getDataHandler().getInputStream();
+ OutputStream os = new FileOutputStream(new File(Configuration.getWithReplacement(Configuration.UPLOAD_AREA_CONFIG),filename));
+ IOUtils.copy(in,os);
+ in.close();
+ os.close();
+ } catch(IOException e) {
+ throw new IOException("Failed uploading file:",e);
+ }
+ }
+ return Response.ok().build();
+ } catch(Exception e) {
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java
new file mode 100644
index 0000000..0f83d2d
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/rest/ValidationRest.java
@@ -0,0 +1,65 @@
+package org.apache.oodt.cas.curation.rest;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Response;
+
+import org.apache.oodt.cas.curation.configuration.Configuration;
+import org.apache.oodt.cas.curation.validation.ValidationBackend;
+import org.apache.oodt.cas.curation.validation.ValidationException;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+
+/**
+ * A backend that loads and prepares a validation layer from the
+ * filemanager.
+ *
+ * @author starchmd
+ */
+@Path("validation")
+public class ValidationRest {
+ private static final Logger LOG = Logger.getLogger(ValidationBackend.class.getName());
+ private Gson gson = new GsonBuilder().create();
+ ValidationBackend backend = null;
+
+ /**
+ * Empty REST service constructor. (Configuration not available yet.
+ */
+ public ValidationRest(){}
+
+ @GET
+ @Produces("application/json")
+ /**
+ * Return information pertaining to validation
+ * @return - OK response on success
+ */
+ public Response information() {
+ try {
+ setup();
+ return Response.ok().entity(gson.toJson(this.backend.getValidation())).build();
+ } catch(Exception e) {
+ LOG.log(Level.SEVERE,"Validation exception occured calling validation/info REST endpoint", e);
+ return ExceptionResponseHandler.BuildExceptionResponse(e);
+ }
+ }
+ /**
+ * Setup this service
+ */
+ public synchronized void setup() throws ValidationException {
+ if (this.backend != null)
+ return;
+ try {
+ LOG.log(Level.INFO, "Setting up validation backend");
+ this.backend = new ValidationBackend(Configuration.getWithReplacement(Configuration.FILEMANAGER_PROP_CONFIG));
+ } catch(ValidationException ve) {
+ LOG.log(Level.SEVERE,"Validation exception occured setting up validation REST service", ve);
+ throw ve;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/a47b088a/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java
----------------------------------------------------------------------
diff --git a/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java b/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java
new file mode 100644
index 0000000..903d65d
--- /dev/null
+++ b/curator2/src/main/java/org/apache/oodt/cas/curation/service/CurationService.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.oodt.cas.curation.service;
+
+//OODT imports
+import org.apache.oodt.cas.curation.servlet.CuratorConfMetKeys;
+import org.apache.oodt.cas.curation.util.SSOUtils;
+import org.apache.oodt.security.sso.SingleSignOn;
+
+//JDK imports
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Logger;
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.UriInfo;
+
+//JAX-RS imports
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+
+//APACHE imports
+import org.apache.commons.lang.StringUtils;
+
+/**
+ *
+ *
+ * The Main Curation Web Service responsible for providing basic tools and
+ * functionality for performing ingestion into the CAS filemgr, and management
+ * of the staging area for ingestion.
+ *
+ * @author pramirez
+ * @author mattmann
+ * @version $Revision$
+ *
+ */
+public class CurationService extends HttpServlet implements CuratorConfMetKeys {
+
+ private static final long serialVersionUID = -5370697580594691669L;
+
+ private static final Logger LOG = Logger.getLogger(CurationService.class
+ .getName());
+
+ protected static CurationServiceConfig config = null;
+
+ protected final static String FORMAT_JSON = "json";
+
+ protected final static String FORMAT_HTML = "html";
+
+ protected final static String UNKNOWN_OUT_FORMAT = "Unsupported Output Format!";
+
+ protected SingleSignOn sso;
+
+ /**
+ * Default Constructor.
+ */
+ public CurationService() {
+ }
+
+ @Override
+ public void init(ServletConfig conf) throws ServletException {
+ super.init(conf);
+ this.config = CurationServiceConfig.getInstance(conf);
+ }
+
+ /**
+ * Gets the staging area as a JSON formatted response object.
+ *
+ * @param base
+ * The base directory to read files from, should be the staging area
+ * root.
+ * @param path
+ * The particular child path within the staging area.
+ * @param showFiles
+ * Whether or not to show {@link File#isFile()} files or not.
+ * @return A String representation formatting using
+ * {@link JSONObject#toString()}.
+ */
+ public String getDirectoryAreaAsJSON(String base, String path,
+ boolean showFiles) {
+ String startingPath = (base + "/" + path);
+ startingPath = StringUtils.replace(startingPath, "source", "/");
+ String f[] = getFilesInDirectory(startingPath, showFiles);
+
+ List<Map<String, Object>> items = new ArrayList<Map<String, Object>>();
+ for (int i = 0; i < f.length; i++) {
+ Map<String, Object> entry = new HashMap<String, Object>();
+ String children[] = getFilesInDirectory(startingPath + "/" + f[i],
+ showFiles);
+ entry.put("text", f[i]);
+ entry.put("id", path + "/" + f[i]);
+ entry.put("expanded", false);
+ entry.put("hasChildren", children != null && (children.length > 0));
+ entry.put("isFile", new File(startingPath + "/" + f[i]).isFile());
+ items.add(entry);
+ }
+
+ return JSONArray.fromObject(items).toString();
+ }
+
+ protected String[] getFilesInDirectory(String directory,
+ final boolean showFiles) {
+ File dir = new File(directory);
+ FilenameFilter filter = new FilenameFilter() {
+
+ public boolean accept(File dir, String name) {
+
+ return !name.startsWith(".")
+ && (showFiles || !new File(dir, name).isFile());
+ }
+ };
+ return dir.list(filter);
+ }
+
+ protected void sendRedirect(String page, UriInfo uriInfo,
+ HttpServletResponse response) throws IOException {
+ response.sendRedirect(uriInfo.getBaseUriBuilder().path("../" + page)
+ .build().toASCIIString());
+ }
+
+ protected String cleansePath(String path) throws UnsupportedEncodingException {
+ String newPath = path;
+ if (newPath.startsWith("/")) {
+ newPath = path.substring(1);
+ }
+ newPath = URLDecoder.decode(newPath, "UTF-8");
+ return newPath;
+
+ }
+
+ /**
+ * Configures the web context persistence layer for the CAS SSO so that all
+ * services ({@link HttpServlet}s) that extend this implementation get the
+ * ability to configure an SSO object for free, essentially.
+ *
+ * @param req
+ * The HTTP request object.
+ * @param res
+ * The HTTP response object.
+ */
+ protected void configureSingleSignOn(HttpServletRequest req,
+ HttpServletResponse res) {
+ this.sso = SSOUtils.getWebSingleSignOn(CurationService.config, req, res);
+ }
+}