You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2018/05/18 16:11:14 UTC
[01/11] oodt git commit: update doap file
Repository: oodt
Updated Branches:
refs/heads/development 84af13a93 -> ec25440d9
update doap file
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/3cac96e4
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/3cac96e4
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/3cac96e4
Branch: refs/heads/development
Commit: 3cac96e4d51c19e0fa2467be3cb66350bc2dfe01
Parents: 84af13a
Author: Tom Barber <to...@jpl.nasa.gov>
Authored: Fri Apr 13 11:10:43 2018 +0100
Committer: Tom Barber <to...@jpl.nasa.gov>
Committed: Fri Apr 13 11:10:43 2018 +0100
----------------------------------------------------------------------
doap_oodt.rdf | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 132 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/3cac96e4/doap_oodt.rdf
----------------------------------------------------------------------
diff --git a/doap_oodt.rdf b/doap_oodt.rdf
new file mode 100644
index 0000000..5ed6790
--- /dev/null
+++ b/doap_oodt.rdf
@@ -0,0 +1,132 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl"?>
+<rdf:RDF xml:lang="en"
+ xmlns="http://usefulinc.com/ns/doap#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:asfext="http://projects.apache.org/ns/asfext#"
+ xmlns:foaf="http://xmlns.com/foaf/0.1/">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+ <Project rdf:about="http://oodt.apache.org">
+ <created>2012-04-14</created>
+ <license rdf:resource="http://usefulinc.com/doap/licenses/asl20" />
+ <name>Apache OODT</name>
+ <homepage rdf:resource="http://oodt.apache.org" />
+ <asfext:pmc rdf:resource="http://oodt.apache.org" />
+ <shortdesc>Apache OODT software provides metadata for middleware (and vice versa):
+- Transparent access to distributed resources
+- Data discovery and query optimization
+- Distributed processing and virtual archives
+.</shortdesc>
+ <description>Apache OODT software is component based, and offers a software architecture beyond simple science applications.</description>
+ <bug-database rdf:resource="https://issues.apache.org/jira/browse/OODT" />
+ <mailing-list rdf:resource="http://oodt.apache.org/contact.html" />
+ <download-page rdf:resource="http://oodt.apache.org/download.html" />
+ <programming-language>Java</programming-language>
+ <category rdf:resource="http://projects.apache.org/category/big-data" />
+ <release>
+ <Version>
+ <name>Apache OODT 0.10</name>
+ <created>2015-09-13</created>
+ <revision>0.10</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.9</name>
+ <created>2015-06-06</created>
+ <revision>0.9</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.8.1</name>
+ <created>2015-01-07</created>
+ <revision>0.8.1</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.8</name>
+ <created>2014-12-01</created>
+ <revision>0.8</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.7</name>
+ <created>2014-09-01</created>
+ <revision>0.7</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.6</name>
+ <created>2013-07-01</created>
+ <revision>0.6</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.5</name>
+ <created>2012-10-01</created>
+ <revision>0.5</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.4</name>
+ <created>2012-06-01</created>
+ <revision>0.4</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.3</name>
+ <created>2011-06-01</created>
+ <revision>0.3</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.2</name>
+ <created>2011-02-08</created>
+ <revision>0.2</revision>
+ </Version>
+ </release>
+ <release>
+ <Version>
+ <name>Apache OODT 0.1-incubating</name>
+ <created>2010-11-09</created>
+ <revision>0.1-incubating</revision>
+ </Version>
+ </release>
+ <repository>
+ <GitRepository>
+ <location rdf:resource="https://git-wip-us.apache.org/repos/asf/oodt.git"/>
+ <browse rdf:resource="https://git-wip-us.apache.org/repos/asf/oodt.git"/>
+ </GitRepository>
+ </repository>
+ <maintainer>
+ <foaf:Person>
+ <foaf:name>Apache OODT PMC</foaf:name>
+ <foaf:mbox rdf:resource="mailto:dev@oodt.apache.org"/>
+ </foaf:Person>
+ </maintainer>
+ </Project>
+</rdf:RDF>
+
[04/11] oodt git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/oodt
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/oodt
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/19e51621
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/19e51621
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/19e51621
Branch: refs/heads/development
Commit: 19e51621268fc52b0355b8203a9dc42d61dd8ce4
Parents: 3fcd424 3cac96e
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Apr 18 18:47:05 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed Apr 18 18:47:05 2018 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 +
doap_oodt.rdf | 132 +++++++++++++++++++
.../filemgr/src/main/resources/bin/filemgr | 4 +-
.../resmgr/src/main/resources/bin/resmgr | 3 +-
.../workflow/src/main/resources/bin/wmgr | 3 +-
5 files changed, 140 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[10/11] oodt git commit: Completely removed XmlRpcFileManager from
all the unnecessary locations. Replaced with
RpcCommunicationsFactory.createClient()
Posted by ma...@apache.org.
Completely removed XmlRpcFileManager from all the unnecessary locations.
Replaced with RpcCommunicationsFactory.createClient()
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/2d19275c
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/2d19275c
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/2d19275c
Branch: refs/heads/development
Commit: 2d19275c1f142d4d42f9765aad8904ebbe7e5a12
Parents: dc79494
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Thu May 17 18:04:53 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu May 17 18:04:53 2018 +0530
----------------------------------------------------------------------
.../FilemgrUniquenessCheckComparator.java | 10 +-
.../curation/service/CurationServiceConfig.java | 22 +-
.../cas/curation/service/MetadataResource.java | 317 +++++++++----------
.../cas/curation/service/PolicyResource.java | 9 +-
.../cli/action/DeleteProductByIdCliAction.java | 1 -
.../cli/action/FileManagerCliAction.java | 22 +-
.../cli/action/IngestProductCliAction.java | 1 -
.../datatransfer/LocalDataTransferer.java | 11 +-
.../cas/filemgr/metadata/ProductMetKeys.java | 9 +-
.../oodt/cas/filemgr/tools/DeleteProduct.java | 37 ++-
.../oodt/cas/filemgr/tools/ExpImpCatalog.java | 1 -
.../tools/MetadataBasedProductMover.java | 1 -
.../oodt/cas/filemgr/tools/QueryTool.java | 29 +-
.../oodt/cas/filemgr/tools/SolrIndexer.java | 35 +-
.../org/apache/oodt/cas/pge/util/XmlHelper.java | 21 +-
.../oodt/cas/product/CASProductHandler.java | 1 -
.../cas/product/data/DataDeliveryServlet.java | 33 +-
.../product/data/DatasetDeliveryServlet.java | 3 +-
.../jaxrs/services/CasProductJaxrsService.java | 1 -
.../oodt/cas/product/rdf/RDFDatasetServlet.java | 1 -
.../oodt/cas/product/rdf/RDFProductServlet.java | 1 -
.../oodt/cas/product/rss/RSSProductServlet.java | 34 +-
.../product/rss/RSSProductTransferServlet.java | 1 -
23 files changed, 298 insertions(+), 303 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java
index 95512c1..f0bd4dc 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/comparator/FilemgrUniquenessCheckComparator.java
@@ -23,7 +23,7 @@ import java.io.File;
import java.net.URL;
//OODT imports
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.exceptions.PreconditionComparatorException;
import org.apache.oodt.cas.metadata.preconditions.PreConditionComparator;
@@ -41,17 +41,15 @@ import org.springframework.beans.factory.annotation.Required;
* product exists before generating metadata for it.
* </p>.
*/
-public class FilemgrUniquenessCheckComparator extends
- PreConditionComparator<Boolean> {
+public class FilemgrUniquenessCheckComparator extends PreConditionComparator<Boolean> {
private String filemgrUrl;
@Override
protected int performCheck(File product, Boolean compareItem)
throws PreconditionComparatorException {
- try {
- boolean returnVal = new XmlRpcFileManagerClient(new URL(
- this.filemgrUrl)).hasProduct(product.getName());
+ try (FileManagerClient fmClient= RpcCommunicationFactory.createClient(new URL(filemgrUrl))){
+ boolean returnVal = fmClient.hasProduct(product.getName());
return Boolean.valueOf(returnVal).compareTo(compareItem);
} catch (Exception e) {
throw new PreconditionComparatorException(
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java
----------------------------------------------------------------------
diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java
index 1fbfb9b..70f24d7 100644
--- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java
+++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/CurationServiceConfig.java
@@ -20,7 +20,8 @@ package org.apache.oodt.cas.curation.service;
//OODT imports
import org.apache.oodt.cas.curation.metadata.CuratorConfMetKeys;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -57,14 +58,13 @@ import javax.servlet.ServletContext;
*
*/
public class CurationServiceConfig implements CuratorConfMetKeys {
- private static CurationServiceConfig instance;
- private final Map<String, String> parameters = new ConcurrentHashMap<String, String>();
-
- private XmlRpcFileManagerClient fmClient = null;
private static final Logger LOG = Logger
.getLogger(CurationServiceConfig.class.getName());
+ private static CurationServiceConfig instance;
+ private final Map<String, String> parameters = new ConcurrentHashMap<String, String>();
+
/**
* Gets a singleton static instance of the global
* {@link CurationServiceConfig} for the CAS Curator Webapp.
@@ -121,12 +121,12 @@ public class CurationServiceConfig implements CuratorConfMetKeys {
/**
*
- * @return The full {@link XmlRpcFileManagerClient} built from the CAS curator
+ * @return The full {@link FileManagerClient} built from the CAS curator
* property <code>filemgr.url</code>.
*/
- public XmlRpcFileManagerClient getFileManagerClient() {
+ public FileManagerClient getFileManagerClient() {
try {
- return new XmlRpcFileManagerClient(new URL(this.getFileMgrURL()));
+ return RpcCommunicationFactory.createClient(new URL(this.getFileMgrURL()));
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
return null;
@@ -189,12 +189,6 @@ public class CurationServiceConfig implements CuratorConfMetKeys {
// Note that the constructor is private
private CurationServiceConfig(ServletConfig conf) {
readContextParams(conf.getServletContext());
- try {
- fmClient = new XmlRpcFileManagerClient(new URL(this.getFileMgrURL()));
- } catch (Exception e) {
- LOG.log(Level.SEVERE, e.getMessage());
- LOG.log(Level.WARNING, "Unable to build CurationServiceConfig: Message: " + e.getMessage());
- }
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java
----------------------------------------------------------------------
diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java
index e9e5953..92901c2 100644
--- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java
+++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/MetadataResource.java
@@ -20,6 +20,9 @@ package org.apache.oodt.cas.curation.service;
//JDK imports
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+import net.sf.json.JSONSerializer;
import org.apache.oodt.cas.curation.structs.ExtractorConfig;
import org.apache.oodt.cas.curation.util.CurationXmlStructFactory;
import org.apache.oodt.cas.curation.util.ExtractorConfigReader;
@@ -33,7 +36,7 @@ import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.filemgr.validation.XMLValidationLayer;
import org.apache.oodt.cas.metadata.MetExtractor;
@@ -41,13 +44,16 @@ import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
import org.apache.oodt.cas.metadata.util.GenericMetadataObjectFactory;
-
-import net.sf.json.JSONArray;
-import net.sf.json.JSONObject;
-import net.sf.json.JSONSerializer;
-
import org.springframework.util.StringUtils;
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriInfo;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
@@ -59,47 +65,28 @@ import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.FormParam;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
@Path("metadata")
/**
- *
+ *
* A web-service endpoint for dealing with CAS {@link Metadata} object.
- *
+ *
* @author pramirez
* @version $Id$
*/
public class MetadataResource extends CurationService {
-
+
@Context
UriInfo uriInfo;
-
- @Context
+
+ @Context
private ServletContext context;
private static final long serialVersionUID = 1930946924218765724L;
@@ -109,20 +96,20 @@ public class MetadataResource extends CurationService {
public static final String CATALOG = "catalog";
public static final String PRODUCT_TYPE = "productType";
-
+
public static final String UPDATE = "update";
-
+
public static final String DELETE = "delete";
private static Logger LOG = Logger.getLogger(MetadataResource.class.getName());
// single instance of CAS catalog shared among all requests
private Catalog catalog = null;
-
+
public MetadataResource(){
-
+
}
public MetadataResource(@Context ServletContext context) {
-
+
}
@GET
@@ -135,19 +122,19 @@ public class MetadataResource extends CurationService {
@Context HttpServletRequest req, @Context HttpServletResponse res) {
Metadata metadata;
-
+
try {
metadata = this.getStagingMetadata(id, configId, overwrite);
} catch (Exception e) {
return "<div class=\"error\">" + e.getMessage() + "</div>";
}
-
+
if (FORMAT_HTML.equals(format)) {
return this.getMetadataAsHTML(metadata);
}
return this.getMetadataAsJSON(metadata).toString();
}
-
+
@GET
@Path("extractor/config")
@Produces("text/plain")
@@ -179,14 +166,14 @@ public class MetadataResource extends CurationService {
}
return html.toString();
}
-
+
protected String getExtractorConfigIdsAsJSON(String[] configIds) {
// TODO: Support JSON
return "Not Implemented...";
}
/**
- *
+ *
* @param id
* Relative path from staging root to product. The met extension will
* be added to this id to look up and see if a met file exists with
@@ -229,7 +216,7 @@ public class MetadataResource extends CurationService {
}
/**
- *
+ *
* @param id
* Relative path from staging root to the product
* @param config
@@ -246,7 +233,7 @@ public class MetadataResource extends CurationService {
.getStagingAreaPath()
+ "/" + id);
}
-
+
@GET
@Path(CATALOG)
@Produces("text/plain")
@@ -259,9 +246,8 @@ public class MetadataResource extends CurationService {
Metadata metadata;
String productId = id.substring(id.lastIndexOf("/") + 1);
- try {
- prod = CurationService.config.getFileManagerClient().getProductById(
- productId);
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) {
+ prod = fmClient.getProductById(productId);
metadata = this.getCatalogMetadata(prod);
} catch (Exception e) {
return "<div class=\"error\">" + e.getMessage() + "</div>";
@@ -279,15 +265,14 @@ public class MetadataResource extends CurationService {
@Produces("text/plain")
public String setCatalogMetadata(MultivaluedMap<String, String> formParams,
@FormParam("id") String id) {
-
+
Product prod;
Metadata metadata = this.getMetadataFromMap(formParams);
-
+
String productId = id.substring(id.lastIndexOf("/") + 1);
- try {
- prod = CurationService.config.getFileManagerClient().getProductById(
- productId);
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) {
+ prod = fmClient.getProductById(productId);
this.updateCatalogMetadata(prod, metadata);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
@@ -296,7 +281,7 @@ public class MetadataResource extends CurationService {
return this.getMetadataAsHTML(metadata);
}
-
+
@GET
@Path(PRODUCT_TYPE)
@Produces("text/plain")
@@ -321,7 +306,7 @@ public class MetadataResource extends CurationService {
}
return this.getMetadataAsJSON(metadata).toString();
}
-
+
@POST
@Path(PRODUCT_TYPE)
@@ -361,7 +346,7 @@ public class MetadataResource extends CurationService {
return "Staging met info";
}
-
+
private JSONObject getMetadataAsJSON(Metadata metadata) {
return JSONObject.fromObject(metadata.getMap());
@@ -379,10 +364,10 @@ public class MetadataResource extends CurationService {
return metadata;
}
-
+
private Metadata getMetadataFromMap(MultivaluedMap<String, String> formParams) {
Metadata metadata = new Metadata();
-
+
for (Map.Entry<String, List<String>> entry : formParams.entrySet()) {
if (entry.getKey().startsWith("metadata.")) {
String newKey = entry.getKey().substring(entry.getKey().indexOf('.') + 1);
@@ -391,10 +376,10 @@ public class MetadataResource extends CurationService {
}
}
}
-
+
return metadata;
}
-
+
protected String getMetadataAsHTML(Metadata metadata) {
if (metadata == null) {
return "<table></table>";
@@ -423,12 +408,12 @@ public class MetadataResource extends CurationService {
return html.toString();
}
-
+
/**
* Reads a {@link Metadata} object from a String representation of a .met
* {@link File}.
- *
+ *
* @param file
* The full path to the .met {@link File}.
* @return The read-in CAS {@link Metadata} object.
@@ -454,7 +439,7 @@ public class MetadataResource extends CurationService {
/**
* Retrieves the cataloged {@link Metadata} associated with a {@link Product}.
- *
+ *
* @param product
* The {@link Product} to obtain cataloged {@link Metadata} for.
* @return The cataloged {@link Metadata} for {@link Product}.
@@ -463,14 +448,18 @@ public class MetadataResource extends CurationService {
* {@link Catalog}.
*/
public Metadata getCatalogMetadata(Product product) throws CatalogException {
- return CurationService.config.getFileManagerClient().getMetadata(
- CurationService.config.getFileManagerClient().getProductById(
- product.getProductId()));
+ try( FileManagerClient fmClient = CurationService.config.getFileManagerClient()){
+ return fmClient.getMetadata(fmClient.getProductById(product.getProductId()));
+ } catch (IOException e){
+ LOG.severe("Error occurred when fetching catalog metadata for product: " +
+ product.getProductName() + " :" + e.getMessage());
+ return null;
+ }
}
/**
* Writes a CAS {@link Metadata} {@link File} using the given identifier.
- *
+ *
* @param id
* The identifier of the .met file to write.
* @param metadata
@@ -488,12 +477,12 @@ public class MetadataResource extends CurationService {
serMet.writeMetadataToXmlStream(new FileOutputStream(new File(config
.getMetAreaPath(), id + config.getMetExtension())));
}
-
+
/**
- * Method to update the catalog metadata for a given product.
- * All current metadata fields will be preserved,
+ * Method to update the catalog metadata for a given product.
+ * All current metadata fields will be preserved,
* except those specified in the HTTP POST request as 'metadata.<field_name>=<field_value>'.
- *
+ *
* @param id
* identifier of CAS product - either 'id' or 'name' must be specified
* @param name
@@ -507,23 +496,17 @@ public class MetadataResource extends CurationService {
@Path(UPDATE)
@Consumes("application/x-www-form-urlencoded")
@Produces("text/plain")
- public String updateMetadata(MultivaluedMap<String, String> formParams,
- @FormParam("id") String id,
- @FormParam("name") String name,
+ public String updateMetadata(MultivaluedMap<String, String> formParams,
+ @FormParam("id") String id,
+ @FormParam("name") String name,
@DefaultValue("true") @FormParam("replace") boolean replace,
@DefaultValue("false") @FormParam("remove") boolean remove) {
-
+
// new metadata from HTTP POST request
Metadata newMetadata = this.getMetadataFromMap(formParams);
-
- // client for interacting with remote File Manager
- XmlRpcFileManagerClient fmClient = CurationService.config.getFileManagerClient();
-
// empty metadata
Metadata metadata;
-
- try {
-
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) {
// retrieve product from catalog
Product product;
if (StringUtils.hasText(id)) {
@@ -534,7 +517,7 @@ public class MetadataResource extends CurationService {
} else {
throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified");
}
-
+
// retrieve existing metadata
metadata = fmClient.getMetadata(product);
@@ -543,44 +526,41 @@ public class MetadataResource extends CurationService {
metadata.removeMetadata("reference_data_store");
metadata.removeMetadata("reference_fileSize");
metadata.removeMetadata("reference_mimeType");
-
+
// merge new and existing metadata
metadata.addMetadata(newMetadata);
-
+
// replace metadata values for keys specified in HTTP request (not others)
if (replace) {
for (String key : newMetadata.getAllKeys()) {
metadata.replaceMetadata(key, newMetadata.getAllMetadata(key));
}
}
-
+
// remove metadata tags
if (remove) {
for (String key : newMetadata.getAllKeys()) {
metadata.removeMetadata(key);
}
}
-
+
// insert old and new metadata
fmClient.updateMetadata(product, metadata);
-
+
// return product id to downstream processors
return "id="+product.getProductId();
-
+
} catch (Exception e) {
-
LOG.log(Level.SEVERE, e.getMessage());
// return error message
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
-
}
-
}
/**
* Updates the cataloged {@link Metadata} for a {@link Product} in the CAS
* File Manager.
- *
+ *
* @param product
* The {@link Product} to update {@link Metadata} for.
* @throws CatalogException
@@ -598,7 +578,7 @@ public class MetadataResource extends CurationService {
is.close();
}
Catalog catalog = this.getCatalog();
-
+
Metadata oldMetadata = catalog.getMetadata(product);
List<Reference> references = catalog.getProductReferences(product);
Product newProduct = new Product(product.getProductName(), product
@@ -617,7 +597,7 @@ public class MetadataResource extends CurationService {
/**
* Method to delete a specific product from the catalog
- *
+ *
* @param id
* identifier of CAS product - either 'id' or 'name' must be specified
* @param name
@@ -629,20 +609,20 @@ public class MetadataResource extends CurationService {
@Consumes("application/x-www-form-urlencoded")
@Produces("text/plain")
public String deleteCatalogMetadata(
- @FormParam("id") String id,
+ @FormParam("id") String id,
@FormParam("name") String name) {
- try {
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) {
// retrieve product from catalog
Product product;
- if (StringUtils.hasText(id)) {
- id = id.substring(id.lastIndexOf("/") + 1);
- product = CurationService.config.getFileManagerClient().getProductById(id);
- } else if (StringUtils.hasText(name)) {
- product = CurationService.config.getFileManagerClient().getProductByName(name);
- } else {
- throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified");
- }
+ if (StringUtils.hasText(id)) {
+ id = id.substring(id.lastIndexOf("/") + 1);
+ product = fmClient.getProductById(id);
+ } else if (StringUtils.hasText(name)) {
+ product = fmClient.getProductByName(name);
+ } else {
+ throw new Exception("Either the HTTP parameter 'id' or the HTTP parameter 'name' must be specified");
+ }
// remove product from catalog
this.deleteCatalogProduct(product);
@@ -651,17 +631,16 @@ public class MetadataResource extends CurationService {
return "id="+product.getProductId();
} catch (Exception e) {
-
LOG.log(Level.SEVERE, e.getMessage());
// return error message
throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR);
}
- }
+ }
/**
* Deletes a given product from the catalog
- *
+ *
* @param product
* The {@link Product} to delete
* @throws FileNotFoundException
@@ -669,10 +648,14 @@ public class MetadataResource extends CurationService {
* @throws CatalogException
* If any error occurs during this delete operation.
*/
- public void deleteCatalogProduct(Product product)
- throws CatalogException {
- CurationService.config.getFileManagerClient().removeProduct(product);
- }
+ public void deleteCatalogProduct(Product product) throws CatalogException {
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()){
+ fmClient.removeProduct(product);
+ } catch (IOException e) {
+ LOG.severe(String.format("Couldn't detele product - %s : %s", product.getProductName(), e.getMessage()));
+ throw new CatalogException("Unable to delete product", e);
+ }
+ }
private Metadata getProductTypeMetadataForPolicy(String policy,
String productTypeName) throws MalformedURLException,
@@ -685,34 +668,38 @@ public class MetadataResource extends CurationService {
XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays
.asList(policies));
ProductType productType = repMgr.getProductTypeByName(productTypeName);
-
- return productType.getTypeMetadata();
- }
-
- private Metadata writeProductTypeMetadata(String policy,
- String productTypeName, Metadata metadata)
- throws MalformedURLException, InstantiationException, RepositoryManagerException, UnsupportedEncodingException,
- CurationException {
- String rootPolicyPath = this.cleanse(CurationService.config
- .getPolicyUploadPath());
- String policyPath = new File(rootPolicyPath + policy).toURL()
- .toExternalForm();
- String[] policies = { policyPath };
- XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays
- .asList(policies));
- ProductType productType = repMgr.getProductTypeByName(productTypeName);
- productType.setTypeMetadata(metadata);
-
- CurationXmlStructFactory.writeProductTypeXmlDocument(repMgr
- .getProductTypes(), rootPolicyPath + policy + "/product-types.xml");
-
- // refresh the config on the fm end
- CurationService.config.getFileManagerClient().refreshConfigAndPolicy();
-
return productType.getTypeMetadata();
}
-
+
+ private Metadata writeProductTypeMetadata(String policy,
+ String productTypeName, Metadata metadata)
+ throws MalformedURLException, InstantiationException, RepositoryManagerException, UnsupportedEncodingException,
+ CurationException {
+ String rootPolicyPath = this.cleanse(CurationService.config
+ .getPolicyUploadPath());
+ String policyPath = new File(rootPolicyPath + policy).toURL()
+ .toExternalForm();
+ String[] policies = {policyPath};
+ XMLRepositoryManager repMgr = new XMLRepositoryManager(Arrays
+ .asList(policies));
+
+ ProductType productType = repMgr.getProductTypeByName(productTypeName);
+ productType.setTypeMetadata(metadata);
+
+ CurationXmlStructFactory.writeProductTypeXmlDocument(repMgr
+ .getProductTypes(), rootPolicyPath + policy + "/product-types.xml");
+
+ // refresh the config on the fm end
+ try (FileManagerClient fmClient = CurationService.config.getFileManagerClient()) {
+ fmClient.refreshConfigAndPolicy();
+ } catch (IOException e) {
+ LOG.severe(String.format("Unable to refresh config and policy: %s", e.getMessage()));
+ }
+
+ return productType.getTypeMetadata();
+ }
+
private String cleanse(String origPath) {
String retStr = origPath;
if (!retStr.endsWith("/")) {
@@ -720,10 +707,10 @@ public class MetadataResource extends CurationService {
}
return retStr;
}
-
+
// Method to instantiate the CAS catalog, if not done already.
private synchronized Catalog getCatalog() {
-
+
if (catalog==null) {
String catalogFactoryClass = this.context.getInitParameter(CATALOG_FACTORY_CLASS);
// preserve backward compatibility
@@ -732,15 +719,15 @@ public class MetadataResource extends CurationService {
}
catalog = GenericFileManagerObjectFactory.getCatalogServiceFromFactory(catalogFactoryClass);
}
-
+
return catalog;
}
-
+
@DELETE
@Path(PRODUCT_TYPE+"/remove")
@Produces("text/plain")
public boolean removeProductType(
- @FormParam("policy") String policy,
+ @FormParam("policy") String policy,
@FormParam("id") String id) {
XMLRepositoryManager xmlRepo = getRepo(policy);
try {
@@ -752,7 +739,7 @@ public class MetadataResource extends CurationService {
return false;
}
}
-
+
@GET
@Path(PRODUCT_TYPE+"/parentmap")
@Produces("text/plain")
@@ -761,13 +748,13 @@ public class MetadataResource extends CurationService {
XMLValidationLayer vLayer = getValidationLayer(policy);
return JSONSerializer.toJSON(vLayer.getSubToSuperMap()).toString();
}
-
+
@POST
@Path(PRODUCT_TYPE+"/parent/add")
@Produces("text/plain")
public boolean addParentForProductType(
- @FormParam("policy") String policy,
- @FormParam("id") String id,
+ @FormParam("policy") String policy,
+ @FormParam("id") String id,
@FormParam("parentId") String parentId) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -780,13 +767,13 @@ public class MetadataResource extends CurationService {
}
return false;
}
-
+
@DELETE
@Path(PRODUCT_TYPE+"/parent/remove")
@Produces("text/plain")
public boolean removeParentForProductType(
- @FormParam("policy") String policy,
- @FormParam("id") String id)
+ @FormParam("policy") String policy,
+ @FormParam("id") String id)
throws ValidationLayerException {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -799,13 +786,13 @@ public class MetadataResource extends CurationService {
}
return false;
}
-
+
@POST
@Path(PRODUCT_TYPE+"/elements/add")
@Produces("text/plain")
public boolean addElementsForProductType(
- @FormParam("policy") String policy,
- @FormParam("id") String id,
+ @FormParam("policy") String policy,
+ @FormParam("id") String id,
@FormParam("elementIds") String elementIds) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -825,13 +812,13 @@ public class MetadataResource extends CurationService {
}
return false;
}
-
+
@GET
@Path(PRODUCT_TYPE+"/elements")
@Produces("text/plain")
public String getElementsForProductType(
- @FormParam("policy") String policy,
- @FormParam("id") String id,
+ @FormParam("policy") String policy,
+ @FormParam("id") String id,
@FormParam("direct") boolean direct) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -847,12 +834,12 @@ public class MetadataResource extends CurationService {
}
return null;
}
-
+
@DELETE
@Path(PRODUCT_TYPE+"/elements/remove/all")
@Produces("text/plain")
public boolean removeAllElementsForProductType(
- @FormParam("policy") String policy,
+ @FormParam("policy") String policy,
@FormParam("id") String id) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -874,8 +861,8 @@ public class MetadataResource extends CurationService {
@Path(PRODUCT_TYPE+"/elements/remove")
@Produces("text/plain")
public boolean removeElementsForProductType(
- @FormParam("policy") String policy,
- @FormParam("id") String id,
+ @FormParam("policy") String policy,
+ @FormParam("id") String id,
@FormParam("elementIds") String elementIds) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -896,12 +883,12 @@ public class MetadataResource extends CurationService {
}
return false;
}
-
+
@GET
@Path(PRODUCT_TYPE+"/typeswithelement/{elementId}")
@Produces("text/plain")
public String getProductTypeIdsHavingElement(
- @FormParam("policy") String policy,
+ @FormParam("policy") String policy,
@PathParam("elementId") String elementId) {
XMLValidationLayer vLayer = getValidationLayer(policy);
XMLRepositoryManager xmlRepo = getRepo(policy);
@@ -919,19 +906,19 @@ public class MetadataResource extends CurationService {
}
return JSONSerializer.toJSON(typeids).toString();
}
-
-
+
+
/*
* Private helper functions
*/
- private void removeUnusedElements(List<Element> elements,
- XMLRepositoryManager xmlRepo, XMLValidationLayer vLayer)
+ private void removeUnusedElements(List<Element> elements,
+ XMLRepositoryManager xmlRepo, XMLValidationLayer vLayer)
throws ValidationLayerException, RepositoryManagerException {
// Remove Elements that aren't used in any product type
List<ProductType> ptypelist = xmlRepo.getProductTypes();
ConcurrentHashMap<String, Boolean> usedElementIds = new ConcurrentHashMap<String, Boolean>();
for(ProductType ptype: ptypelist) {
- List<Element> ptypeElements =
+ List<Element> ptypeElements =
vLayer.getElements(ptype);
for(Element el: ptypeElements) {
usedElementIds.put(el.getElementId(), true);
@@ -942,7 +929,7 @@ public class MetadataResource extends CurationService {
vLayer.removeElement(el);
}
}
- }
+ }
private XMLRepositoryManager getRepo(String policy) {
XMLRepositoryManager xmlRepo = null;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java
----------------------------------------------------------------------
diff --git a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java
index bde259b..c3bd174 100644
--- a/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java
+++ b/curator/services/src/main/java/org/apache/oodt/cas/curation/service/PolicyResource.java
@@ -28,6 +28,7 @@ import org.apache.oodt.cas.filemgr.structs.Query;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import net.sf.json.JSONObject;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import java.io.File;
import java.io.FilenameFilter;
@@ -116,11 +117,9 @@ public class PolicyResource extends CurationService {
ProductType productType;
ProductPage page;
- try {
- productType = config.getFileManagerClient().getProductTypeByName(
- productTypeName);
- page = config.getFileManagerClient().pagedQuery(new Query(),
- productType, pageNum);
+ try (FileManagerClient fmClient = config.getFileManagerClient()){
+ productType = fmClient.getProductTypeByName(productTypeName);
+ page = fmClient.pagedQuery(new Query(), productType, pageNum);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Unable to obtain products for product type: ["
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java
index a202a0c..03eb54e 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/DeleteProductByIdCliAction.java
@@ -23,7 +23,6 @@ import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import java.io.IOException;
import java.net.MalformedURLException;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java
index c3e9032..8630e3d 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/FileManagerCliAction.java
@@ -17,18 +17,20 @@
package org.apache.oodt.cas.filemgr.cli.action;
//JDK imports
-import java.net.MalformedURLException;
-import java.net.URL;
-//Apache imports
import org.apache.commons.lang.Validate;
-
-//OODT imports
import org.apache.oodt.cas.cli.action.CmdLineAction;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+//Apache imports
+//OODT imports
+
/**
* Base {@link CmdLineAction} for File Manager.
*
@@ -42,8 +44,7 @@ public abstract class FileManagerCliAction extends CmdLineAction {
return System.getProperty("org.apache.oodt.cas.filemgr.url");
}
- protected FileManagerClient getClient()
- throws MalformedURLException, ConnectionException {
+ protected FileManagerClient getClient() throws MalformedURLException, ConnectionException {
Validate.notNull(getUrl(), "Must specify url");
if (client != null) {
@@ -56,4 +57,11 @@ public abstract class FileManagerCliAction extends CmdLineAction {
public void setClient(FileManagerClient client) {
this.client = client;
}
+
+ @Override
+ public void finalize() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
index 4bc326b..e0ec33e 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/IngestProductCliAction.java
@@ -23,7 +23,6 @@ import org.apache.oodt.cas.cli.exception.CmdLineActionException;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
index b6f2beb..06eb300 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/LocalDataTransferer.java
@@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.tika.mime.MimeTypeException;
@@ -84,8 +83,7 @@ public class LocalDataTransferer implements DataTransfer {
public void setFileManagerUrl(URL url) {
try {
client = RpcCommunicationFactory.createClient(url);
- LOG.log(Level.INFO, "Local Data Transfer to: ["
- + client.getFileManagerUrl().toString() + "] enabled");
+ LOG.log(Level.INFO, "Local Data Transfer to: [" + client.getFileManagerUrl().toString() + "] enabled");
} catch (ConnectionException e) {
LOG.log(Level.SEVERE, e.getMessage());
}
@@ -439,4 +437,11 @@ public class LocalDataTransferer implements DataTransfer {
+ p.getProductId() + "]: Message: " + e.getMessage());
}
}
+
+ @Override
+ public void finalize() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java
index 558c6eb..cfd8377 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/metadata/ProductMetKeys.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -20,10 +20,11 @@ package org.apache.oodt.cas.filemgr.metadata;
/**
*
- * Met key field names used to augment {@link Product} {@link Metadata} in
- * {@link org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient#getMetadata(org.apache.oodt.cas.filemgr.structs.Product)}
+ * Met key field names used to augment {@link org.apache.oodt.cas.filemgr.structs.Product}
+ * {@link org.apache.oodt.cas.metadata.Metadata} in
+ * {@link org.apache.oodt.cas.filemgr.system.FileManagerClient#getMetadata(org.apache.oodt.cas.filemgr.structs.Product)}
* and in
- * {@link org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient#getReducedMetadata(org.apache.oodt.cas.filemgr.structs.Product, java.util.List)}
+ * {@link org.apache.oodt.cas.filemgr.system.FileManagerClient#getReducedMetadata(org.apache.oodt.cas.filemgr.structs.Product, java.util.List)}
* .
*
* @see http://issues.apache.org/jira/browse/OODT-72
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
index 654c85a..772a5ee 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
@@ -18,13 +18,14 @@
package org.apache.oodt.cas.filemgr.tools;
//OODT imports
-import org.apache.oodt.cas.filemgr.structs.Reference;
+
import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
-//JDK imports
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
@@ -32,10 +33,12 @@ import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import java.util.List;
import java.util.Vector;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//JDK imports
/**
* @author woollard
@@ -53,22 +56,19 @@ public class DeleteProduct {
private static final Logger LOG = Logger
.getLogger(DeleteProduct.class.getName());
- /* our File Manager client */
- private XmlRpcFileManagerClient client = null;
-
+ private FileManagerClient client;
/* whether or not we should commit our deletions */
private boolean commit = true;
public DeleteProduct(String fileManagerUrl, boolean commit) {
+ this.commit = commit;
+
try {
- client = new XmlRpcFileManagerClient(new URL(fileManagerUrl));
+ client = RpcCommunicationFactory.createClient(new URL(fileManagerUrl));
} catch (Exception e) {
- LOG.log(Level.SEVERE, "Unable to create file manager client: Message: "
- + e.getMessage() + ": errors to follow");
+ LOG.severe("Unable to create client: " + e.getMessage());
}
- this.commit = commit;
-
if (!this.commit) {
LOG.log(Level.INFO, "Commit disabled.");
} else {
@@ -230,4 +230,15 @@ public class DeleteProduct {
return prodIds;
}
+
+ /**
+ * Not the best place to do this. But, no other option at the moment. Mandatory to close the client once done.
+ * @throws IOException
+ */
+ @Override
+ public void finalize() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java
index e64e560..c2db95f 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/ExpImpCatalog.java
@@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java
index 989dbd7..64e2c67 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataBasedProductMover.java
@@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index 27e37d2..cf065e4 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -32,10 +32,11 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.QueryFormulationException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.SqlParser;
-import java.net.MalformedURLException;
+import java.io.IOException;
import java.net.URL;
import java.util.List;
import java.util.Vector;
@@ -55,7 +56,7 @@ public final class QueryTool {
private static String freeTextBlock = "__FREE__";
- private XmlRpcFileManagerClient client = null;
+ private FileManagerClient client = null;
private enum QueryType { LUCENE, SQL }
@@ -64,7 +65,7 @@ public final class QueryTool {
public QueryTool(URL fmUrl) throws InstantiationException {
try {
- client = new XmlRpcFileManagerClient(fmUrl);
+ client = RpcCommunicationFactory.createClient(fmUrl);
} catch (ConnectionException e) {
throw new InstantiationException(e.getMessage());
}
@@ -151,8 +152,8 @@ public final class QueryTool {
}
public static void main(String[] args)
- throws MalformedURLException, InstantiationException, CatalogException, QueryFormulationException,
- ConnectionException {
+ throws IOException, InstantiationException, CatalogException, QueryFormulationException,
+ ConnectionException {
String usage = "Usage: QueryTool [options] \n"
+ "options: \n"
+ "--url <fm url> \n"
@@ -241,17 +242,19 @@ public final class QueryTool {
return prodTypes;
}
- private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl)
- throws MalformedURLException, CatalogException, ConnectionException, QueryFormulationException {
+ private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl)
+ throws IOException, CatalogException, ConnectionException, QueryFormulationException {
ComplexQuery complexQuery = SqlParser.parseSqlQuery(query);
complexQuery.setSortByMetKey(sortBy);
complexQuery.setToStringResultFormat(outputFormat);
- List<QueryResult> results = new XmlRpcFileManagerClient(new URL(filemgrUrl)).complexQuery(complexQuery);
- StringBuilder returnString = new StringBuilder("");
- for (QueryResult qr : results) {
- returnString.append(qr.toString()).append(delimiter);
+ try(FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(filemgrUrl))){
+ List<QueryResult> results = fmClient.complexQuery(complexQuery);
+ StringBuilder returnString = new StringBuilder("");
+ for (QueryResult qr : results) {
+ returnString.append(qr.toString()).append(delimiter);
+ }
+ return returnString.substring(0, returnString.length() - delimiter.length());
}
- return returnString.substring(0, returnString.length() - delimiter.length());
}
private static void exit(String msg) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 16ca7f1..9361ead 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -34,7 +34,8 @@ import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -67,6 +68,9 @@ import java.util.logging.Logger;
* specific documentation.
*/
public class SolrIndexer {
+
+ private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
+
private final static String SOLR_INDEXER_CONFIG = "SOLR_INDEXER_CONFIG";
private final static String SOLR_URL = "solr.url";
private final static String FILEMGR_URL = "filemgr.url";
@@ -77,7 +81,6 @@ public class SolrIndexer {
private final SolrServer server;
private String fmUrl;
private String solrUrl;
- private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
private final static SimpleDateFormat solrFormat = new SimpleDateFormat(
"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
@@ -245,9 +248,7 @@ public class SolrIndexer {
*/
public void indexProductTypes(boolean delete) {
LOG.info("Indexing product types...");
- try {
- XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) {
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -290,6 +291,8 @@ public class SolrIndexer {
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
+ e.getMessage());
+ } catch (IOException e) {
+ LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage()));
}
LOG.info("Finished indexing product types.");
}
@@ -297,7 +300,7 @@ public class SolrIndexer {
/**
* Suppresses exception that occurred with older file managers.
*/
- private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) {
+ private ProductPage safeFirstPage(FileManagerClient fmClient, ProductType type) {
ProductPage page = null;
try {
page = fmClient.getFirstPage(type);
@@ -320,9 +323,7 @@ public class SolrIndexer {
*/
public void indexAll(boolean delete) {
LOG.info("Indexing products...");
- try {
- XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) {
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -358,6 +359,8 @@ public class SolrIndexer {
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
+ e.getMessage());
+ } catch (IOException e) {
+ LOG.severe(String.format("Error occurred when indexing: %s", e.getMessage()));
}
}
@@ -374,9 +377,7 @@ public class SolrIndexer {
public void indexProduct(String productId)
throws SolrServerException {
LOG.info("Attempting to index product: " + productId);
- try {
- XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) {
Product product = fmClient.getProductById(productId);
Metadata productMetadata = fmClient.getMetadata(product);
indexProduct(product.getProductId(), productMetadata, product
@@ -390,6 +391,8 @@ public class SolrIndexer {
+ e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
+ } catch (IOException e) {
+ LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage()));
}
}
@@ -409,8 +412,7 @@ public class SolrIndexer {
public void indexProductByName(String productName, boolean delete) throws SolrServerException {
LOG.info("Attempting to index product: " + productName);
- try {
-
+ try (FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl))) {
// Try to delete product by name
// Note: the standard field "CAS.ProductName" must be mapped to some Solr field in file indexer.properties
if (delete) {
@@ -425,9 +427,6 @@ public class SolrIndexer {
LOG.warning("Could not delete product: "+productName+" from Solr index");
}
}
-
- XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
Product product = fmClient.getProductByName(productName);
Metadata productMetadata = fmClient.getMetadata(product);
// NOTE: delete (by id) is now false
@@ -442,6 +441,8 @@ public class SolrIndexer {
+ e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
+ } catch (IOException e) {
+ LOG.severe(String.format("Error occurred when indexing product types: %s", e.getMessage()));
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
----------------------------------------------------------------------
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
index 8c53aff..9f01d99 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
@@ -16,7 +16,10 @@
*/
package org.apache.oodt.cas.pge.util;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.util.QueryUtils;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.SqlParser;
@@ -28,11 +31,6 @@ import org.apache.oodt.cas.pge.config.OutputDir;
import org.apache.oodt.cas.pge.config.RegExprOutputFiles;
import org.apache.oodt.cas.pge.exceptions.PGEException;
import org.apache.oodt.commons.xml.XMLUtils;
-
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -442,20 +440,15 @@ public class XmlHelper {
public static String fillIn(String value, Metadata inputMetadata,
boolean envReplaceRecur) throws PGEException {
- try {
+ try (FileManagerClient fmClient=RpcCommunicationFactory.createClient(
+ new URL(inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL.getName())))){
while ((value = PathUtils
.doDynamicReplacement(value, inputMetadata)).contains("[")
&& envReplaceRecur) {
}
if (value.toUpperCase().matches(
"^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$")) {
- value = QueryUtils
- .getQueryResultsAsString(new XmlRpcFileManagerClient(
- new URL(inputMetadata
- .getMetadata(QUERY_FILE_MANAGER_URL
- .getName())))
- .complexQuery(SqlParser
- .parseSqlQueryMethod(value)));
+ value = QueryUtils.getQueryResultsAsString(fmClient.complexQuery(SqlParser.parseSqlQueryMethod(value)));
}
return value;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java
index 4e4d19c..00ef50f 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/CASProductHandler.java
@@ -25,7 +25,6 @@ import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.product.data.DataUtils;
import org.apache.oodt.product.LargeProductQueryHandler;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
index 62a9b25..282053b 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
@@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -59,18 +58,30 @@ public class DataDeliveryServlet extends HttpServlet implements
public static final int INT = 512;
+ /** Client i/f to filemgr server. */
+ private FileManagerClient client;
+
+ /** our log stream */
+ private static final Logger LOG = Logger.getLogger(DataDeliveryServlet.class
+ .getName());
+
+ /** our working dir path. */
+ private String workingDirPath;
+
+ /** serial version UID. */
+ private static final long serialVersionUID = -955613407495060869L;
+
/** {@inheritDoc} */
public void init(ServletConfig config) throws ServletException {
super.init(config);
try {
String fileMgrURL;
try {
- fileMgrURL = PathUtils.replaceEnvVariables(config.getServletContext().getInitParameter(
- "filemgr.url") );
+ fileMgrURL = PathUtils.replaceEnvVariables(config.getServletContext().getInitParameter("filemgr.url") );
} catch (Exception e) {
throw new ServletException("Failed to get filemgr url : " + e.getMessage(), e);
}
- client = new XmlRpcFileManagerClient(new URL(fileMgrURL));
+ client = RpcCommunicationFactory.createClient(new URL(fileMgrURL));
} catch (MalformedURLException ex) {
throw new ServletException(ex);
} catch (ConnectionException ex) {
@@ -265,18 +276,4 @@ public class DataDeliveryServlet extends HttpServlet implements
in.close();
out.close();
}
-
- /** Client i/f to filemgr server. */
- private FileManagerClient client;
-
- /** our log stream */
- private static final Logger LOG = Logger.getLogger(DataDeliveryServlet.class
- .getName());
-
- /** our working dir path. */
- private String workingDirPath;
-
- /** serial version UID. */
- private static final long serialVersionUID = -955613407495060869L;
-
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java
index 00c89c1..9685e36 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DatasetDeliveryServlet.java
@@ -24,7 +24,6 @@ import org.apache.oodt.cas.filemgr.structs.ProductPage;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -247,7 +246,7 @@ public class DatasetDeliveryServlet extends HttpServlet implements
} catch (Exception e) {
throw new ServletException("Failed to get filemgr url : " + e.getMessage(), e);
}
- client = new XmlRpcFileManagerClient(new URL(fileMgrURL));
+ client = RpcCommunicationFactory.createClient(new URL(fileMgrURL));
} catch (MalformedURLException ex) {
throw new ServletException(ex);
} catch (ConnectionException ex) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java
index f8d32d3..ddb98de 100755
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/jaxrs/services/CasProductJaxrsService.java
@@ -22,7 +22,6 @@ import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.product.exceptions.CasProductException;
import org.apache.oodt.cas.product.jaxrs.exceptions.BadRequestException;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java
index 03fb813..e15faf4 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFDatasetServlet.java
@@ -26,7 +26,6 @@ import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.commons.xml.XMLUtils;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java
index df1eeac..f32211a 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rdf/RDFProductServlet.java
@@ -29,7 +29,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
index 94772d3..4a2636c 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
@@ -20,6 +20,7 @@ package org.apache.oodt.cas.product.rss;
//OODT imports
+
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.Reference;
@@ -27,7 +28,7 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.commons.util.DateConvert;
@@ -35,17 +36,6 @@ import org.apache.oodt.commons.xml.XMLUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-//JDK imports
-import java.io.FileNotFoundException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@@ -58,6 +48,18 @@ import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+//JDK imports
/**
* A Servlet that supports the <a
@@ -317,7 +319,7 @@ public class RSSProductServlet extends HttpServlet {
private void getFileManager(String fileManagerUrl) {
try {
- fm = new XmlRpcFileManagerClient(new URL(fileManagerUrl));
+ fm = RpcCommunicationFactory.createClient(new URL(fileManagerUrl));
} catch (MalformedURLException e) {
LOG.log(Level.SEVERE,
"Unable to initialize file manager url in RSS Servlet: [url="
@@ -329,4 +331,10 @@ public class RSSProductServlet extends HttpServlet {
}
}
+ @Override
+ public void finalize() throws IOException {
+ if (fm != null) {
+ fm.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/2d19275c/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java
----------------------------------------------------------------------
diff --git a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java
index 9aaecaf..44eb712 100644
--- a/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java
+++ b/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductTransferServlet.java
@@ -23,7 +23,6 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
[02/11] oodt git commit: Updated startup scripts of resmgr and
workflowmanager to use AvroRpc implementations.
Posted by ma...@apache.org.
Updated startup scripts of resmgr and workflowmanager to use AvroRpc implementations.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ecfe0a78
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ecfe0a78
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ecfe0a78
Branch: refs/heads/development
Commit: ecfe0a78849c928d904eeb129cfd7d24f9600afc
Parents: 9feacbb
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Apr 18 18:45:47 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed Apr 18 18:45:47 2018 +0530
----------------------------------------------------------------------
resource/src/main/bin/resmgr | 1 -
.../resource/system/ResourceManagerMain.java | 6 +++-
.../system/rpc/ResourceManagerFactory.java | 18 ++++++++++
resource/src/main/resources/resource.properties | 4 +++
.../workflow/system/WorkflowManagerStarter.java | 20 +++++++----
.../system/rpc/RpcCommunicationFactory.java | 38 +++++++++-----------
workflow/src/main/resources/workflow.properties | 6 ++--
7 files changed, 58 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/resource/src/main/bin/resmgr
----------------------------------------------------------------------
diff --git a/resource/src/main/bin/resmgr b/resource/src/main/bin/resmgr
index f80cd90..1ca4978 100644
--- a/resource/src/main/bin/resmgr
+++ b/resource/src/main/bin/resmgr
@@ -51,7 +51,6 @@ case "$1" in
$JAVA_HOME/bin/java -Djava.ext.dirs=${CAS_RESMGR_HOME}/lib \
-Djava.util.logging.config.file=${CAS_RESMGR_HOME}/etc/logging.properties \
-Dorg.apache.oodt.cas.resource.properties=${CAS_RESMGR_PROPS} \
- -Dresmgr.manager=org.apache.oodt.cas.resource.system.AvroRpcResourceManager \
org.apache.oodt.cas.resource.system.ResourceManagerMain --portNum $SERVER_PORT &
echo $! >${RUN_HOME}/cas.resmgr.pid
echo "OK"
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManagerMain.java
----------------------------------------------------------------------
diff --git a/resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManagerMain.java b/resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManagerMain.java
index 162d64e..ad82d06 100644
--- a/resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManagerMain.java
+++ b/resource/src/main/java/org/apache/oodt/cas/resource/system/ResourceManagerMain.java
@@ -43,6 +43,7 @@ public class ResourceManagerMain {
final ResourceManager manager = ResourceManagerFactory.getResourceManager(portNum);
manager.startUp();
+ logger.info("Resource manager started at port: {}", portNum);
Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
@@ -53,7 +54,10 @@ public class ResourceManagerMain {
for (; ; ) {
try {
Thread.currentThread().join();
- } catch (InterruptedException ignore) {
+ } catch (InterruptedException e) {
+ logger.error("Main thread interrupted. Exiting: {}", e.getMessage());
+ manager.shutdown();
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java
----------------------------------------------------------------------
diff --git a/resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java b/resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java
index 9287f6e..24f56b9 100644
--- a/resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java
+++ b/resource/src/main/java/org/apache/oodt/cas/resource/system/rpc/ResourceManagerFactory.java
@@ -22,6 +22,8 @@ import org.apache.oodt.cas.resource.system.ResourceManagerClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.FileInputStream;
import java.lang.reflect.Constructor;
import java.net.URL;
@@ -29,7 +31,22 @@ public class ResourceManagerFactory {
private static final Logger logger = LoggerFactory.getLogger(ResourceManagerFactory.class);
+ private static void loadProperties() {
+ // set up the configuration, if there is any
+ if (System.getProperty("org.apache.oodt.cas.resource.properties") != null) {
+ String configFile = System.getProperty("org.apache.oodt.cas.resource.properties");
+
+ logger.info("Loading File Manager Configuration Properties from: [{}]", configFile);
+ try {
+ System.getProperties().load(new FileInputStream(new File(configFile)));
+ } catch (Exception e) {
+ logger.error("Error loading configuration properties from: [{}]", configFile);
+ }
+ }
+ }
+
public static ResourceManager getResourceManager(int port) throws Exception {
+ loadProperties();
String resourceManagerClass = System.getProperty("resmgr.manager",
"org.apache.oodt.cas.resource.system.AvroRpcResourceManager");
@@ -48,6 +65,7 @@ public class ResourceManagerFactory {
}
public static ResourceManagerClient getResourceManagerClient(URL url) throws Exception {
+ loadProperties();
String resMgrClientClass = System.getProperty("resmgr.manager.client",
"org.apache.oodt.cas.resource.system.AvroRpcResourceManagerClient");
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/resource/src/main/resources/resource.properties
----------------------------------------------------------------------
diff --git a/resource/src/main/resources/resource.properties b/resource/src/main/resources/resource.properties
index a467949..bf6a420 100644
--- a/resource/src/main/resources/resource.properties
+++ b/resource/src/main/resources/resource.properties
@@ -16,6 +16,10 @@
#
# Properties required to configure the Resource Manager
+# Client and server classes to be used as resource managers
+resmgr.manager=org.apache.oodt.cas.resource.system.AvroRpcResourceManager
+resmgr.manager.client=org.apache.oodt.cas.resource.system.AvroRpcResourceManagerClient
+
# resource spark master
resource.runner.spark.host = mesos://<ip>:5050
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerStarter.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerStarter.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerStarter.java
index 04d5831..775574d 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerStarter.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerStarter.java
@@ -58,14 +58,20 @@ public class WorkflowManagerStarter {
}
loadProperties();
- WorkflowManager manager = RpcCommunicationFactory.createServer(portNum);
+ final WorkflowManager manager = RpcCommunicationFactory.createServer(portNum);
- for (;;)
- try {
- Thread.currentThread().join();
- } catch (InterruptedException ignore) {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ manager.shutdown();
}
- }
-
+ });
+ try {
+ Thread.currentThread().join();
+ } catch (InterruptedException e) {
+ LOG.log(Level.SEVERE, String.format("Interrupted while executing: %s", e.getMessage()));
+ manager.shutdown();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java
index 2a8e471..f0b89ac 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/rpc/RpcCommunicationFactory.java
@@ -19,6 +19,8 @@ package org.apache.oodt.cas.workflow.system.rpc;
import org.apache.oodt.cas.workflow.system.WorkflowManager;
import org.apache.oodt.cas.workflow.system.WorkflowManagerClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
@@ -27,13 +29,15 @@ import java.util.Properties;
/**
* @author radu
- *
+ * <p>
* Create instaces server/client for {@link WorkflowManager} and {@link WorkflowManagerClient}
*/
public class RpcCommunicationFactory {
- private static String getRpcServerClassName(){
+ private static final Logger logger = LoggerFactory.getLogger(RpcCommunicationFactory.class);
+
+ private static String getRpcServerClassName() {
InputStream prpFileStream = RpcCommunicationFactory.class.getResourceAsStream("/workflow.properties");
Properties properties = new Properties();
try {
@@ -45,49 +49,39 @@ public class RpcCommunicationFactory {
"org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory");
}
- private static String getRpcClientClassName(){
+ private static String getRpcClientClassName() {
InputStream prpFileStream = RpcCommunicationFactory.class.getResourceAsStream("/workflow.properties");
Properties properties = new Properties();
try {
properties.load(prpFileStream);
} catch (IOException e) {
- e.printStackTrace();
+ logger.error("Unable to load properties", e);
}
return properties.getProperty("workflow.client.factory",
"org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory");
}
- public static WorkflowManager createServer(int port){
+ public static WorkflowManager createServer(int port) {
try {
WorkflowManagerFactory workflowManagerFactory =
(WorkflowManagerFactory) Class.forName(getRpcServerClassName()).newInstance();
workflowManagerFactory.setPort(port);
return workflowManagerFactory.createServer();
- } catch (InstantiationException e) {
- e.printStackTrace();
- } catch (IllegalAccessException e) {
- e.printStackTrace();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
+ } catch (Exception e) {
+ logger.error("Error creating server", e);
+ throw new IllegalStateException("Unable to create server", e);
}
- return null;
}
- public static WorkflowManagerClient createClient(URL url){
+ public static WorkflowManagerClient createClient(URL url) {
try {
WorkflowManagerFactory workflowManagerFactory =
(WorkflowManagerFactory) Class.forName(getRpcClientClassName()).newInstance();
workflowManagerFactory.setUrl(url);
return workflowManagerFactory.createClient();
- } catch (InstantiationException e) {
- e.printStackTrace();
- } catch (IllegalAccessException e) {
- e.printStackTrace();
- } catch (ClassNotFoundException e) {
- e.printStackTrace();
+ } catch (Exception e) {
+ logger.error("Unable to create client", e);
+ throw new IllegalStateException("Unable to create client", e);
}
- return null;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ecfe0a78/workflow/src/main/resources/workflow.properties
----------------------------------------------------------------------
diff --git a/workflow/src/main/resources/workflow.properties b/workflow/src/main/resources/workflow.properties
index 912a186..328d991 100644
--- a/workflow/src/main/resources/workflow.properties
+++ b/workflow/src/main/resources/workflow.properties
@@ -16,10 +16,8 @@
# Properties required to configure the Workflow Manager
#Rpc workflow communication class
-workflow.server.factory = org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory
-workflow.client.factory = org.apache.oodt.cas.workflow.system.rpc.XmlRpcWorkflowManagerFactory
-#workflow.server.factory = org.apache.oodt.cas.workflow.system.AvroRpcWorkflowManagerFactory
-#workflow.client.factory = org.apache.oodt.cas.workflow.system.AvroRpcWorkflowManagerFactory
+workflow.server.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory
+workflow.client.factory = org.apache.oodt.cas.workflow.system.rpc.AvroRpcWorkflowManagerFactory
# workflow repository factory
workflow.repo.factory = org.apache.oodt.cas.workflow.repository.XMLWorkflowRepositoryFactory
[05/11] oodt git commit: OODT-975 Upgrade Tomcat in radix
Posted by ma...@apache.org.
OODT-975 Upgrade Tomcat in radix
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/40ed14e4
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/40ed14e4
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/40ed14e4
Branch: refs/heads/development
Commit: 40ed14e48510b9e6993caaa97c7d69bad9ff99a7
Parents: 3cac96e
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Tue Apr 24 14:51:10 2018 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Tue Apr 24 14:51:10 2018 -0700
----------------------------------------------------------------------
.../resources/archetype-resources/distribution/pom.xml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/40ed14e4/mvn/archetypes/radix/src/main/resources/archetype-resources/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/mvn/archetypes/radix/src/main/resources/archetype-resources/distribution/pom.xml b/mvn/archetypes/radix/src/main/resources/archetype-resources/distribution/pom.xml
index d2de984..7a1ec25 100644
--- a/mvn/archetypes/radix/src/main/resources/archetype-resources/distribution/pom.xml
+++ b/mvn/archetypes/radix/src/main/resources/archetype-resources/distribution/pom.xml
@@ -30,7 +30,7 @@
<name>Distribution (RADiX Distro of OODT)</name>
<packaging>pom</packaging>
<properties>
- <tomcat.version>5.5.23</tomcat.version>
+ <tomcat.version>9.0.7</tomcat.version>
</properties>
<profiles>
<profile>
@@ -123,8 +123,8 @@
<configuration>
<artifactItems>
<artifactItem>
- <groupId>tomcat</groupId>
- <artifactId>apache-tomcat</artifactId>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat</artifactId>
<version>${tomcat.version}</version>
<type>zip</type>
<overWrite>false</overWrite>
@@ -278,8 +278,8 @@
<configuration>
<artifactItems>
<artifactItem>
- <groupId>tomcat</groupId>
- <artifactId>apache-tomcat</artifactId>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat</artifactId>
<version>${tomcat.version}</version>
<type>zip</type>
<overWrite>false</overWrite>
[08/11] oodt git commit: [OODT-978] Fixed crawler launcher hanging
issue in file manager avro version by adding a close() method to the
FileManagerClient
Posted by ma...@apache.org.
[OODT-978] Fixed crawler launcher hanging issue in file manager avro version
by adding a close() method to the FileManagerClient
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/1c49a9bb
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/1c49a9bb
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/1c49a9bb
Branch: refs/heads/development
Commit: 1c49a9bbbc8e7c95bd23106c3dc58560c5af7488
Parents: 19e5162
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed May 16 15:56:29 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed May 16 15:56:29 2018 +0530
----------------------------------------------------------------------
.../apache/oodt/cas/crawl/CrawlerLauncher.java | 4 ++-
.../apache/oodt/cas/crawl/ProductCrawler.java | 19 ++++++++++--
.../cli/action/CrawlerLauncherCliAction.java | 5 +++-
.../oodt/cas/filemgr/ingest/CachedIngester.java | 13 ++++++--
.../cas/filemgr/ingest/CmdLineIngester.java | 1 +
.../oodt/cas/filemgr/ingest/Ingester.java | 3 +-
.../oodt/cas/filemgr/ingest/StdIngester.java | 31 +++++++++++++-------
.../filemgr/system/AvroFileManagerClient.java | 17 +++++++----
.../cas/filemgr/system/FileManagerClient.java | 5 +++-
.../filemgr/system/XmlRpcFileManagerClient.java | 5 ++++
.../cli/action/DummyFileManagerClient.java | 5 ++++
11 files changed, 81 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
index 9fd682c..b47c7e2 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/CrawlerLauncher.java
@@ -18,13 +18,14 @@ package org.apache.oodt.cas.crawl;
//JDK imports
import java.io.IOException;
+import java.util.Set;
//OODT imports
import org.apache.oodt.cas.cli.CmdLineUtility;
/**
* A command line interface to the new Spring enabled crawler.
- *
+ *
* @author bfoster (Brian Foster)
* @version $Revision$
* @since OODT-190
@@ -34,5 +35,6 @@ public class CrawlerLauncher {
public static void main(String[] args) throws IOException {
CmdLineUtility cmdLineUtility = new CmdLineUtility();
cmdLineUtility.run(args);
+ System.out.println("Exiting crawler launcher");
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
index 3f5d254..494b2c2 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/ProductCrawler.java
@@ -17,8 +17,6 @@
package org.apache.oodt.cas.crawl;
import com.google.common.annotations.VisibleForTesting;
-
-//OODT imports
import org.apache.oodt.cas.crawl.action.CrawlerAction;
import org.apache.oodt.cas.crawl.action.CrawlerActionRepo;
import org.apache.oodt.cas.crawl.config.ProductCrawlerBean;
@@ -29,6 +27,7 @@ import org.apache.oodt.cas.metadata.Metadata;
import java.io.File;
import java.io.FileFilter;
+import java.io.IOException;
import java.net.URL;
import java.util.Collections;
import java.util.List;
@@ -37,6 +36,8 @@ import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
+//OODT imports
+
/**
* An abstract base class for Product Crawling. This class provides methods to
@@ -71,11 +72,15 @@ public abstract class ProductCrawler extends ProductCrawlerBean {
protected Ingester ingester;
public void crawl() {
+ LOG.info("Crawl operation invoked");
crawl(new File(getProductPath()));
}
public void crawl(File dirRoot) {
- // Reset ingest status.
+ LOG.info(String.format("Start crawling dir: %s", dirRoot));
+
+ // Reset ingest status.a
+
ingestStatus.clear();
// Load actions.
@@ -114,6 +119,8 @@ public abstract class ProductCrawler extends ProductCrawlerBean {
}
}
}
+
+ LOG.info(String.format("Finished crawling dir: %s", dirRoot));
}
public IngestStatus handleFile(File product) {
@@ -363,4 +370,10 @@ public abstract class ProductCrawler extends ProductCrawlerBean {
}
return allSucceeded;
}
+
+ public void shutdown() throws IOException {
+ if (ingester != null) {
+ ingester.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
index cce8ea8..b7b70ef 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/cli/action/CrawlerLauncherCliAction.java
@@ -27,7 +27,7 @@ import org.springframework.context.support.FileSystemXmlApplicationContext;
/**
* A {@link CmdLineAction} which is responsible for launching crawlers.
- *
+ *
* @author bfoster (Brian Foster)
*/
public class CrawlerLauncherCliAction extends CmdLineAction {
@@ -54,9 +54,12 @@ public class CrawlerLauncherCliAction extends CmdLineAction {
if (pc.getDaemonPort() != -1 && pc.getDaemonWait() != -1) {
new CrawlDaemon(pc.getDaemonWait(), pc, pc.getDaemonPort())
.startCrawling();
+ printer.println("Finished crawler daemon");
} else {
pc.crawl();
+ printer.println("Finished crawling");
}
+ pc.shutdown();
} catch (Exception e) {
throw new CmdLineActionException("Failed to launch crawler : "
+ e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
index 8e96be1..52e5c9f 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CachedIngester.java
@@ -25,6 +25,7 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import java.io.File;
import java.io.FileInputStream;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
import java.net.URL;
@@ -39,7 +40,7 @@ import java.util.logging.Logger;
*
* <p>
* An extension of the {@link StdIngester} that uses a {@link Cache} to keep
- * track of {@link Product} ingestion status. If the existing {@link Cache} used
+ * track of {@link org.apache.oodt.cas.filemgr.structs.Product} ingestion status. If the existing {@link Cache} used
* is already sync'ed to the requested File Manager (specified by the
* <code>fmUrl</code> parameter in {@link #hasProduct(URL, File)} or
* {@link #hasProduct(URL, String)}), then the {@link Cache} will simply return
@@ -58,7 +59,7 @@ public class CachedIngester extends StdIngester {
/**
* @param transferService
* The underlying data transfer service to use to ingest
- * {@link Product}s.
+ * {@link org.apache.oodt.cas.filemgr.structs.Product}s.
* @param cacheServiceFactory
* The {@link CacheFactory} to use to construct this
* {@link Ingester}'s {@link Cache}.
@@ -94,7 +95,7 @@ public class CachedIngester extends StdIngester {
*
* @param transferService
* The underlying data transfer service to use to ingest
- * {@link Product}s.
+ * {@link org.apache.oodt.cas.filemgr.structs.Product}s.
* @param cache
* The {@link Cache} that this {@link Ingester} will use.
* @throws InstantiationException
@@ -167,4 +168,10 @@ public class CachedIngester extends StdIngester {
+ e.getMessage());
}
}
+
+ @Override
+ public void close() throws IOException {
+ cache.clear();
+ super.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
index 59d0736..8d6807b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/CmdLineIngester.java
@@ -128,6 +128,7 @@ public class CmdLineIngester extends StdIngester {
System.out.println("Result: " + productID);
}
+ ingester.close();
}
private static List<String> readProdFilesFromStdin() {
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
index b12dfd4..8e19273 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/Ingester.java
@@ -24,6 +24,7 @@ import org.apache.oodt.cas.metadata.MetExtractor;
import org.apache.oodt.cas.metadata.Metadata;
//JDK imports
+import java.io.Closeable;
import java.io.File;
import java.net.URL;
import java.util.List;
@@ -37,7 +38,7 @@ import java.util.List;
* An interface for ingesting {@link Product}s
* </p>.
*/
-public interface Ingester {
+public interface Ingester extends Closeable {
/**
* Ingests a {@link Product} to the file manager service object identified
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
index 2421873..0195bb6 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/ingest/StdIngester.java
@@ -18,6 +18,7 @@
package org.apache.oodt.cas.filemgr.ingest;
//OODT imports
+
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
@@ -26,15 +27,15 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.IngestException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
+import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.oodt.cas.metadata.MetExtractor;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
-//JDK imports
import java.io.File;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
@@ -42,26 +43,28 @@ import java.util.Vector;
import java.util.logging.Level;
import java.util.logging.Logger;
+//JDK imports
+
/**
* @author mattmann
* @author bfoster
* @version $Revision$
- *
+ *
* <p>
* An implementation of the {@link Ingster} interface that uses the following
* pieces of {@link Metadata} information to determine how to ingest a
* {@link Product}:
- *
+ *
* <ul>
* <li>Filename - The name of the Product file to ingest.</li>
* <li>ProductType - The type of the Product to ingest.</li>
* <li>FileLocation - A full path pointer to directory containing the Product
* file to ingest.</li>
* </ul>
- *
+ *
* The {@link Ingester} supports overriding certain {@link Product} properties,
* including:
- *
+ *
* <ul>
* <li>Specification of <code>ProductStructure</code> parameter that will
* tell the {@link Ingester} whether or not the {@link Product} is a directory
@@ -90,7 +93,7 @@ public class StdIngester implements Ingester, CoreMetKeys {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.io.File, org.apache.oodt.cas.metadata.MetExtractor,
* java.io.File)
@@ -111,7 +114,7 @@ public class StdIngester implements Ingester, CoreMetKeys {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.util.List, org.apache.oodt.cas.metadata.MetExtractor, java.io.File)
*/
@@ -143,7 +146,7 @@ public class StdIngester implements Ingester, CoreMetKeys {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#ingest(java.net.URL,
* java.io.File, org.apache.oodt.cas.metadata.Metadata)
*/
@@ -224,7 +227,7 @@ public class StdIngester implements Ingester, CoreMetKeys {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL,
* java.io.File)
*/
@@ -293,7 +296,7 @@ public class StdIngester implements Ingester, CoreMetKeys {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.ingest.Ingester#hasProduct(java.net.URL,
* java.lang.String)
*/
@@ -309,4 +312,10 @@ public class StdIngester implements Ingester, CoreMetKeys {
}
}
+ @Override
+ public void close() throws IOException {
+ if (fmClient != null) {
+ fmClient.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
index ca3839d..0b28bff 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/AvroFileManagerClient.java
@@ -88,7 +88,7 @@ public class AvroFileManagerClient implements FileManagerClient {
//setup the and start the client
try {
this.fileManagerUrl = url;
- InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(),this.fileManagerUrl.getPort());
+ InetSocketAddress inetSocketAddress = new InetSocketAddress(url.getHost(), this.fileManagerUrl.getPort());
this.client = new NettyTransceiver(inetSocketAddress, 40000L);
proxy = (AvroFileManager) SpecificRequestor.getClient(AvroFileManager.class, client);
} catch (IOException e) {
@@ -124,8 +124,7 @@ public class AvroFileManagerClient implements FileManagerClient {
try {
if (proxy != null) {
success = proxy.isAlive();
- }
- else return false;
+ } else return false;
} catch (AvroRemoteException e) {
LOG.log(Level.WARNING, "AvroRemoteException when connecting to filemgr: ["
+ this.fileManagerUrl + "]");
@@ -155,7 +154,7 @@ public class AvroFileManagerClient implements FileManagerClient {
success = proxy.removeProductTransferStatus(AvroTypeFactory.getAvroProduct(product));
} catch (AvroRemoteException e) {
throw new DataTransferException(e.getMessage());
- }
+ }
return success;
}
@@ -584,7 +583,7 @@ public class AvroFileManagerClient implements FileManagerClient {
@Override
public String ingestProduct(Product product, Metadata metadata,
- boolean clientTransfer) throws Exception {
+ boolean clientTransfer) throws Exception {
try {
// ingest product
String productId = this.proxy.ingestProduct(
@@ -594,7 +593,7 @@ public class AvroFileManagerClient implements FileManagerClient {
if (clientTransfer) {
LOG.log(Level.FINEST, "File Manager Client: clientTransfer enabled: transfering product ["
- + product.getProductName() + "]");
+ + product.getProductName() + "]");
// we need to transfer the product ourselves
// make sure we have the product ID
@@ -736,4 +735,10 @@ public class AvroFileManagerClient implements FileManagerClient {
this.dataTransfer = dataTransfer;
}
+ @Override
+ public void close() throws IOException {
+ if (client != null) {
+ client.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
index 8b1c54a..c0fa46a 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManagerClient.java
@@ -35,6 +35,8 @@ import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
import org.apache.oodt.cas.metadata.Metadata;
//JDK imports
+import java.io.Closeable;
+import java.io.IOException;
import java.net.URL;
import java.util.List;
@@ -45,7 +47,7 @@ import java.util.List;
* <p>Interface of client for FileManager RPC logic. All methods that are used for
* RPC transfer.</p>
*/
-public interface FileManagerClient {
+public interface FileManagerClient extends Closeable {
public boolean refreshConfigAndPolicy();
@@ -176,4 +178,5 @@ public interface FileManagerClient {
public void setDataTransfer(DataTransfer dataTransfer);
+ void close() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
index 3bd91fd..06c2726 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
@@ -1404,4 +1404,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
this.dataTransfer.setFileManagerUrl(this.fileManagerUrl);
}
+ @Override
+ public void close() throws IOException {
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/1c49a9bb/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
index 6a4c2bb..ebf42f0 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/cli/action/DummyFileManagerClient.java
@@ -285,4 +285,9 @@ public class DummyFileManagerClient implements FileManagerClient {
public void setDataTransfer(DataTransfer dataTransfer) {
}
+
+ @Override
+ public void close() throws IOException {
+
+ }
}
[06/11] oodt git commit: Remove
mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig and
mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej from SCM
Posted by ma...@apache.org.
Remove mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig and mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej from SCM
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/fed87b2b
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/fed87b2b
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/fed87b2b
Branch: refs/heads/development
Commit: fed87b2bccc9890b0f5e45883936cc31fcd12f8f
Parents: 3cac96e
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Mon Apr 30 09:59:26 2018 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Mon Apr 30 09:59:26 2018 -0700
----------------------------------------------------------------------
.../maven-cas-install-plugin.iml.orig | 48 --------------
.../maven-cas-install-plugin.iml.rej | 70 --------------------
2 files changed, 118 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/fed87b2b/mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig
----------------------------------------------------------------------
diff --git a/mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig b/mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig
deleted file mode 100755
index c78919b..0000000
--- a/mvn/plugins/cas-install/maven-cas-install-plugin.iml.orig
+++ /dev/null
@@ -1,48 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
- <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
- <output url="file://$MODULE_DIR$/target/classes" />
- <output-test url="file://$MODULE_DIR$/target/test-classes" />
- <content url="file://$MODULE_DIR$">
- <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
- <excludeFolder url="file://$MODULE_DIR$/target" />
- </content>
- <orderEntry type="library" name="Clover IDEA Plugin" level="application" />
- <orderEntry type="inheritedJdk" />
- <orderEntry type="sourceFolder" forTests="false" />
- <orderEntry type="library" name="Maven: org.apache.maven:maven-plugin-api:2.0" level="project" />
- <orderEntry type="module" module-name="cas-metadata" />
- <orderEntry type="module" module-name="oodt-commons" />
- <orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.1" level="project" />
- <orderEntry type="library" name="Maven: xml-apis:xml-apis:2.0.2" level="project" />
- <orderEntry type="library" name="Maven: commons-collections:commons-collections:2.1" level="project" />
- <orderEntry type="library" name="Maven: commons-pool:commons-pool:1.2" level="project" />
- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
- <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
- <orderEntry type="module" module-name="pcs-input" />
- <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.3" level="project" />
- <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
- <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.7" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-core:2.5.4" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-hibernate3:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
- <orderEntry type="library" name="Maven: org.hibernate:hibernate:3.2.5.ga" level="project" />
- <orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:1.2.3" level="project" />
- <orderEntry type="library" name="Maven: asm:asm-attrs:1.5.3" level="project" />
- <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
- <orderEntry type="library" name="Maven: antlr:antlr:2.7.6" level="project" />
- <orderEntry type="library" name="Maven: cglib:cglib:2.1_3" level="project" />
- <orderEntry type="library" name="Maven: asm:asm:1.5.3" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-beans:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-context:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-dao:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: com.google.guava:guava:10.0.1" level="project" />
- <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
- <orderEntry type="library" scope="TEST" name="Maven: junit:junit:3.8.1" level="project" />
- <orderEntry type="library" name="Maven: org.apache.ant:ant:1.7.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.7.0" level="project" />
- <orderEntry type="library" name="Maven: commons-io:commons-io:1.4" level="project" />
- </component>
-</module>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/fed87b2b/mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej
----------------------------------------------------------------------
diff --git a/mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej b/mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej
deleted file mode 100755
index 4eff02b..0000000
--- a/mvn/plugins/cas-install/maven-cas-install-plugin.iml.rej
+++ /dev/null
@@ -1,70 +0,0 @@
---- mvn/plugins/cas-install/maven-cas-install-plugin.iml
-+++ mvn/plugins/cas-install/maven-cas-install-plugin.iml
-@@ -9,13 +9,13 @@
- </content>
- <orderEntry type="inheritedJdk" />
- <orderEntry type="sourceFolder" forTests="false" />
-- <orderEntry type="library" name="Maven: xml-apis:xml-apis:2.0.2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.6" level="project" />
-+ <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.maven:maven-plugin-api:2.0" level="project" />
- <orderEntry type="module" module-name="cas-metadata" />
- <orderEntry type="module" module-name="oodt-commons" />
- <orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.1" level="project" />
-- <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
-+ <orderEntry type="library" name="Maven: xml-apis:xml-apis:2.0.2" level="project" />
- <orderEntry type="library" name="Maven: commons-collections:commons-collections:2.1" level="project" />
- <orderEntry type="library" name="Maven: commons-pool:commons-pool:1.2" level="project" />
- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
---- mvn/plugins/cas-install/maven-cas-install-plugin.iml
-+++ mvn/plugins/cas-install/maven-cas-install-plugin.iml
-@@ -1,49 +0,0 @@
--<?xml version="1.0" encoding="UTF-8"?>
--<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
-- <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
-- <output url="file://$MODULE_DIR$/target/classes" />
-- <output-test url="file://$MODULE_DIR$/target/test-classes" />
-- <content url="file://$MODULE_DIR$">
-- <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
-- <excludeFolder url="file://$MODULE_DIR$/target" />
-- </content>
-- <orderEntry type="inheritedJdk" />
-- <orderEntry type="sourceFolder" forTests="false" />
-- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.6" level="project" />
-- <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.0.b2" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.maven:maven-plugin-api:2.0" level="project" />
-- <orderEntry type="module" module-name="cas-metadata" />
-- <orderEntry type="module" module-name="oodt-commons" />
-- <orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.1" level="project" />
-- <orderEntry type="library" name="Maven: xml-apis:xml-apis:2.0.2" level="project" />
-- <orderEntry type="library" name="Maven: commons-collections:commons-collections:2.1" level="project" />
-- <orderEntry type="library" name="Maven: commons-pool:commons-pool:1.2" level="project" />
-- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
-- <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
-- <orderEntry type="module" module-name="pcs-input" />
-- <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.3" level="project" />
-- <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
-- <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.7" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-core:2.5.4" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-hibernate3:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
-- <orderEntry type="library" name="Maven: org.hibernate:hibernate:3.2.5.ga" level="project" />
-- <orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:1.2.3" level="project" />
-- <orderEntry type="library" name="Maven: asm:asm-attrs:1.5.3" level="project" />
-- <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
-- <orderEntry type="library" name="Maven: antlr:antlr:2.7.6" level="project" />
-- <orderEntry type="library" name="Maven: cglib:cglib:2.1_3" level="project" />
-- <orderEntry type="library" name="Maven: asm:asm:1.5.3" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-beans:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-context:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-dao:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: com.google.guava:guava:10.0.1" level="project" />
-- <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
-- <orderEntry type="library" scope="TEST" name="Maven: junit:junit:3.8.1" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.ant:ant:1.7.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.ant:ant-launcher:1.7.0" level="project" />
-- <orderEntry type="library" name="Maven: commons-io:commons-io:1.4" level="project" />
-- </component>
--</module>
\ No newline at end of file
[09/11] oodt git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/oodt
Posted by ma...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/oodt
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/dc794944
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/dc794944
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/dc794944
Branch: refs/heads/development
Commit: dc794944b7362d87821c42ae91de8c39df849de5
Parents: 1c49a9b f347506
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed May 16 16:00:16 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed May 16 16:00:16 2018 +0530
----------------------------------------------------------------------
.../archetype-resources/distribution/pom.xml | 10 +--
.../maven-cas-install-plugin.iml.orig | 48 --------------
.../maven-cas-install-plugin.iml.rej | 70 --------------------
3 files changed, 5 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
[03/11] oodt git commit: Merge remote-tracking branch 'origin/master'
Posted by ma...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/3fcd4249
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/3fcd4249
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/3fcd4249
Branch: refs/heads/development
Commit: 3fcd4249a146415e6925a2c839de2f5dbf31aeb3
Parents: ecfe0a7 8b4a3d4
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Apr 18 18:46:21 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed Apr 18 18:46:21 2018 +0530
----------------------------------------------------------------------
README.md | 149 ++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 100 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
[11/11] oodt git commit: Fixed TestPGETaskInstance test failure due
to null FM Url at XMLHelper.fillIn()
Posted by ma...@apache.org.
Fixed TestPGETaskInstance test failure due to null FM Url at XMLHelper.fillIn()
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ec25440d
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ec25440d
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ec25440d
Branch: refs/heads/development
Commit: ec25440d97f260ade41101351a8516191076493d
Parents: 2d19275
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Fri May 18 13:19:35 2018 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Fri May 18 13:19:35 2018 +0530
----------------------------------------------------------------------
.../apache/oodt/cas/pge/PGETaskInstance.java | 74 +++++++++-----
.../org/apache/oodt/cas/pge/util/XmlHelper.java | 24 +++--
.../oodt/cas/pge/TestPGETaskInstance.java | 100 ++++++++++---------
.../system/AvroRpcWorkflowManagerClient.java | 39 +++++---
.../workflow/system/WorkflowManagerClient.java | 4 +-
.../system/XmlRpcWorkflowManagerClient.java | 4 +
6 files changed, 145 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
----------------------------------------------------------------------
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java b/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
index 4f36890..2e266e7 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/PGETaskInstance.java
@@ -16,6 +16,9 @@
*/
package org.apache.oodt.cas.pge;
+import com.google.common.base.Splitter;
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
import org.apache.commons.lang.Validate;
import org.apache.oodt.cas.crawl.AutoDetectProductCrawler;
import org.apache.oodt.cas.crawl.ProductCrawler;
@@ -54,12 +57,6 @@ import org.apache.oodt.cas.workflow.system.rpc.RpcCommunicationFactory;
import org.apache.oodt.cas.workflow.util.ScriptFile;
import org.apache.oodt.commons.exceptions.CommonsException;
import org.apache.oodt.commons.exec.ExecUtils;
-import org.apache.xmlrpc.XmlRpcException;
-
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
import org.springframework.context.support.FileSystemXmlApplicationContext;
import java.io.File;
@@ -80,8 +77,14 @@ import java.util.logging.SimpleFormatter;
import java.util.regex.Pattern;
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.*;
-import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.*;
-import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.*;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.CONF_FILE_BUILD;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.CRAWLING;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.RUNNING_PGE;
+import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.STAGING_INPUT;
+import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createConfigFilePropertyAdder;
+import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createFileStager;
+import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createPgeConfigBuilder;
+import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.createSciPgeConfigFileWriter;
/**
@@ -94,24 +97,21 @@ import static org.apache.oodt.cas.pge.util.GenericPgeObjectFactory.*;
public class PGETaskInstance implements WorkflowTaskInstance {
protected Logger logger = Logger.getLogger(PGETaskInstance.class.getName());
- protected WorkflowManagerClient wm;
- protected String workflowInstId;
+ private WorkflowManagerClient wmClient;
+ private String workflowInstId;
protected PgeMetadata pgeMetadata;
protected PgeConfig pgeConfig;
protected PGETaskInstance() {}
@Override
- public void run(Metadata metadata, WorkflowTaskConfiguration config)
- throws WorkflowTaskInstanceException {
+ public void run(Metadata metadata, WorkflowTaskConfiguration config) throws WorkflowTaskInstanceException {
try {
// Initialize CAS-PGE.
pgeMetadata = createPgeMetadata(metadata, config);
pgeConfig = createPgeConfig();
runPropertyAdders();
- wm = createWorkflowManagerClient();
- workflowInstId = getWorkflowInstanceId();
- logger = createLogger(); // use workflow ID specific logger from now on
+ logger = createLogger(); // use workflow ID specific logger from now on
// Write out PgeMetadata.
dumpMetadataIfRequested();
@@ -128,7 +128,9 @@ public class PGETaskInstance implements WorkflowTaskInstance {
runPge();
// Ingest products.
- runIngestCrawler(createProductCrawler());
+ ProductCrawler productCrawler = createProductCrawler();
+ runIngestCrawler(productCrawler);
+ productCrawler.shutdown();
// Commit dynamic metadata.
updateDynamicMetadata();
@@ -141,7 +143,7 @@ public class PGETaskInstance implements WorkflowTaskInstance {
protected void updateStatus(String status) throws Exception {
logger.info("Updating status to workflow as [" + status + "]");
- if (!wm.updateWorkflowInstanceStatus(workflowInstId, status)) {
+ if (!getWorkflowManagerClient().updateWorkflowInstanceStatus(workflowInstId, status)) {
throw new PGEException(
"Failed to update workflow status : client returned false");
}
@@ -250,12 +252,15 @@ public class PGETaskInstance implements WorkflowTaskInstance {
pgeConfig.getPropertyAdderCustomArgs());
}
- protected WorkflowManagerClient createWorkflowManagerClient()
- throws MalformedURLException {
- String url = pgeMetadata.getMetadata(WORKFLOW_MANAGER_URL);
- logger.info("Creating WorkflowManager client for url [" + url + "]");
- Validate.notNull(url, "Must specify " + WORKFLOW_MANAGER_URL);
- return RpcCommunicationFactory.createClient(new URL(url));
+ protected WorkflowManagerClient getWorkflowManagerClient() throws MalformedURLException {
+ if (this.wmClient == null) {
+ String url = pgeMetadata.getMetadata(WORKFLOW_MANAGER_URL);
+ logger.info("Creating WorkflowManager client for url [" + url + "]");
+ Validate.notNull(url, "Must specify " + WORKFLOW_MANAGER_URL);
+ this.wmClient = RpcCommunicationFactory.createClient(new URL(url));
+ }
+
+ return this.wmClient;
}
protected String getWorkflowInstanceId() {
@@ -586,7 +591,26 @@ public class PGETaskInstance implements WorkflowTaskInstance {
protected void updateDynamicMetadata() throws Exception {
pgeMetadata.commitMarkedDynamicMetadataKeys();
- wm.updateMetadataForWorkflow(workflowInstId,
- pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
+ getWorkflowManagerClient()
+ .updateMetadataForWorkflow(workflowInstId, pgeMetadata.asMetadata(PgeMetadata.Type.DYNAMIC));
+ }
+
+ public String getWorkflowInstId() {
+ return workflowInstId;
+ }
+
+ public void setWorkflowInstId(String workflowInstId) {
+ this.workflowInstId = workflowInstId;
+ }
+
+ public void setWmClient(WorkflowManagerClient wmClient) {
+ this.wmClient = wmClient;
+ }
+
+ @Override
+ public void finalize() throws IOException {
+ if (wmClient != null) {
+ wmClient.close();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
----------------------------------------------------------------------
diff --git a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
index 9f01d99..a373af0 100644
--- a/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
+++ b/pge/src/main/java/org/apache/oodt/cas/pge/util/XmlHelper.java
@@ -37,6 +37,7 @@ import org.w3c.dom.NodeList;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
@@ -438,21 +439,26 @@ public class XmlHelper {
return fillIn(value, inputMetadata, true);
}
- public static String fillIn(String value, Metadata inputMetadata,
- boolean envReplaceRecur) throws PGEException {
- try (FileManagerClient fmClient=RpcCommunicationFactory.createClient(
- new URL(inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL.getName())))){
- while ((value = PathUtils
- .doDynamicReplacement(value, inputMetadata)).contains("[")
- && envReplaceRecur) {
+ public static String fillIn(String value, Metadata inputMetadata, boolean envReplaceRecur) throws PGEException {
+ FileManagerClient fmClient=null;
+ try {
+ while ((value = PathUtils.doDynamicReplacement(value, inputMetadata)).contains("[") && envReplaceRecur) {
}
- if (value.toUpperCase().matches(
- "^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$")) {
+
+ if (value.toUpperCase().matches("^\\s*SQL\\s*\\(.*\\)\\s*\\{.*\\}\\s*$")) {
+ fmClient = RpcCommunicationFactory
+ .createClient(new URL(inputMetadata.getMetadata(QUERY_FILE_MANAGER_URL.getName())));
value = QueryUtils.getQueryResultsAsString(fmClient.complexQuery(SqlParser.parseSqlQueryMethod(value)));
}
return value;
} catch (Exception e) {
throw new PGEException("Failed to parse value: " + value, e);
+ } finally {
+ if (fmClient != null) {
+ try {
+ fmClient.close();
+ } catch (IOException ignored) { }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
----------------------------------------------------------------------
diff --git a/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java b/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
index c888290..db60792 100644
--- a/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
+++ b/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
@@ -18,6 +18,9 @@ package org.apache.oodt.cas.pge;
//OODT static imports
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.oodt.cas.crawl.AutoDetectProductCrawler;
import org.apache.oodt.cas.crawl.ProductCrawler;
@@ -35,13 +38,8 @@ import org.apache.oodt.cas.pge.metadata.PgeTaskStatus;
import org.apache.oodt.cas.pge.writers.MockDynamicConfigFileWriter;
import org.apache.oodt.cas.workflow.metadata.CoreMetKeys;
import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
+import org.apache.oodt.cas.workflow.system.AvroRpcWorkflowManagerClient;
import org.apache.oodt.cas.workflow.system.WorkflowManagerClient;
-import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient;
import org.junit.After;
import org.junit.Test;
import org.w3c.dom.Document;
@@ -50,10 +48,13 @@ import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.StringReader;
+import java.net.URL;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -64,14 +65,18 @@ import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import static org.apache.oodt.cas.pge.metadata.PgeTaskMetKeys.*;
import static org.apache.oodt.cas.pge.metadata.PgeTaskStatus.CRAWLING;
-import static org.easymock.EasyMock.*;
-import static org.junit.Assert.*;
-import static org.junit.Assume.*;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
//JDK imports
//JUnit imports
//Apache imports
@@ -219,25 +224,24 @@ public class TestPGETaskInstance {
assertEquals(Level.SEVERE.getLocalizedName() + ": pge2 message1", messages.get(1));
}
- @Test
+ @Test
public void testUpdateStatus() throws Exception {
final Map<String, String> args = Maps.newHashMap();
- PGETaskInstance pgeTask = createTestInstance();
- pgeTask.wm = new XmlRpcWorkflowManagerClient(null) {
- @Override
- public boolean updateWorkflowInstanceStatus(String instanceId,
- String status) {
- args.put("InstanceId", instanceId);
- args.put("Status", status);
- return true;
- }
- };
- String instanceId = "Test ID";
- String status = PgeTaskStatus.CRAWLING.getWorkflowStatusName();
- pgeTask.workflowInstId = instanceId;
- pgeTask.updateStatus(status);
- assertEquals(instanceId, args.get("InstanceId"));
- assertEquals(status, args.get("Status"));
+ PGETaskInstance pgeTask = createTestInstance();
+ pgeTask.setWmClient(new AvroRpcWorkflowManagerClient(new URL("http://localhost:9001")) {
+ @Override
+ public boolean updateWorkflowInstanceStatus(String instanceId, String status) {
+ args.put("InstanceId", instanceId);
+ args.put("Status", status);
+ return true;
+ }
+ });
+ String instanceId = "Test ID";
+ String status = PgeTaskStatus.CRAWLING.getWorkflowStatusName();
+ pgeTask.setWorkflowInstId(instanceId);
+ pgeTask.updateStatus(status);
+ assertEquals(instanceId, args.get("InstanceId"));
+ assertEquals(status, args.get("Status"));
}
@Test
@@ -271,11 +275,8 @@ public class TestPGETaskInstance {
@Test
public void testCreateWorkflowManagerClient() throws Exception {
PGETaskInstance pgeTask = createTestInstance();
- pgeTask.pgeMetadata.replaceMetadata(WORKFLOW_MANAGER_URL,
- "http://localhost:8888");
- WorkflowManagerClient wmClient =
- pgeTask.createWorkflowManagerClient();
- assertNotNull(wmClient);
+ pgeTask.pgeMetadata.replaceMetadata(WORKFLOW_MANAGER_URL, "http://localhost:8888");
+ assertNotNull(pgeTask.getWorkflowManagerClient());
}
@Test
@@ -462,12 +463,13 @@ public class TestPGETaskInstance {
pgeTask.pgeConfig.addOuputDirAndExpressions(new OutputDir("/tmp/dir1", true));
pgeTask.pgeConfig.addOuputDirAndExpressions(new OutputDir("/tmp/dir2", true));
pgeTask.pgeMetadata.replaceMetadata(ATTEMPT_INGEST_ALL, Boolean.toString(true));
- pgeTask.workflowInstId = "WorkflowInstanceId";
+ pgeTask.setWorkflowInstId("WorkflowInstanceId");
- pgeTask.wm = createMock(WorkflowManagerClient.class);
- expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
- CRAWLING.getWorkflowStatusName())).andReturn(true);
- replay(pgeTask.wm);
+ pgeTask.setWmClient(createMock(WorkflowManagerClient.class));
+ expect(pgeTask.getWorkflowManagerClient()
+ .updateWorkflowInstanceStatus(pgeTask.getWorkflowInstId(), CRAWLING.getWorkflowStatusName())
+ ).andReturn(true);
+ replay(pgeTask.getWorkflowManagerClient());
AutoDetectProductCrawler pc = createMock(AutoDetectProductCrawler.class);
pc.crawl(new File("/tmp/dir1"));
@@ -477,14 +479,14 @@ public class TestPGETaskInstance {
pgeTask.runIngestCrawler(pc);
- verify(pgeTask.wm);
+ verify(pgeTask.getWorkflowManagerClient());
verify(pc);
// Case: UpdateStatus Fail
- pgeTask.wm = createMock(WorkflowManagerClient.class);
- expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
+ pgeTask.setWmClient(createMock(WorkflowManagerClient.class));
+ expect(pgeTask.getWorkflowManagerClient().updateWorkflowInstanceStatus(pgeTask.getWorkflowInstId(),
CRAWLING.getWorkflowStatusName())).andReturn(false);
- replay(pgeTask.wm);
+ replay(pgeTask.getWorkflowManagerClient());
pc = createMock(AutoDetectProductCrawler.class);
replay(pc);
@@ -494,14 +496,14 @@ public class TestPGETaskInstance {
fail("Should have thrown");
} catch (Exception e) { /* expect throw */ }
- verify(pgeTask.wm);
+ verify(pgeTask.getWorkflowManagerClient());
verify(pc);
// Case: UpdateStatus Success, VerifyIngest Fail
- pgeTask.wm = createMock(WorkflowManagerClient.class);
- expect(pgeTask.wm.updateWorkflowInstanceStatus(pgeTask.workflowInstId,
+ pgeTask.setWmClient(createMock(WorkflowManagerClient.class));
+ expect(pgeTask.getWorkflowManagerClient().updateWorkflowInstanceStatus(pgeTask.getWorkflowInstId(),
CRAWLING.getWorkflowStatusName())).andReturn(true);
- replay(pgeTask.wm);
+ replay(pgeTask.getWorkflowManagerClient());
pc = createMock(AutoDetectProductCrawler.class);
pc.crawl(new File("/tmp/dir1"));
@@ -529,7 +531,7 @@ public class TestPGETaskInstance {
fail("Should have thrown");
} catch (Exception e) { /* expect throw */ }
- verify(pgeTask.wm);
+ verify(pgeTask.getWorkflowManagerClient());
verify(pc);
}
@@ -634,7 +636,7 @@ public class TestPGETaskInstance {
private PGETaskInstance createTestInstance(String workflowInstId)
throws Exception {
PGETaskInstance pgeTask = new PGETaskInstance();
- pgeTask.workflowInstId = workflowInstId;
+ pgeTask.setWorkflowInstId(workflowInstId);
pgeTask.pgeMetadata = new PgeMetadata();
pgeTask.pgeMetadata.replaceMetadata(NAME, "TestPGE");
pgeTask.pgeConfig = new PgeConfig();
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManagerClient.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManagerClient.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManagerClient.java
index adc0bba..337c520 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManagerClient.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManagerClient.java
@@ -20,13 +20,12 @@ package org.apache.oodt.cas.workflow.system;
import org.apache.avro.ipc.NettyTransceiver;
import org.apache.avro.ipc.Transceiver;
import org.apache.avro.ipc.specific.SpecificRequestor;
-import org.apache.oodt.cas.cli.CmdLineUtility;
import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.oodt.cas.workflow.structs.WorkflowInstancePage;
-import org.apache.oodt.cas.workflow.structs.WorkflowInstance;
import org.apache.oodt.cas.workflow.structs.Workflow;
-import org.apache.oodt.cas.workflow.structs.WorkflowTask;
import org.apache.oodt.cas.workflow.structs.WorkflowCondition;
+import org.apache.oodt.cas.workflow.structs.WorkflowInstance;
+import org.apache.oodt.cas.workflow.structs.WorkflowInstancePage;
+import org.apache.oodt.cas.workflow.structs.WorkflowTask;
import org.apache.oodt.cas.workflow.util.AvroTypeFactory;
import java.io.IOException;
@@ -45,26 +44,20 @@ import java.util.logging.Logger;
*/
public class AvroRpcWorkflowManagerClient implements WorkflowManagerClient {
- private static Logger LOG = Logger
- .getLogger(AvroRpcWorkflowManagerClient.class.getName());
-
- Transceiver client;
-
- org.apache.oodt.cas.workflow.struct.avrotypes.WorkflowManager proxy;
-
- URL workflowManagerUrl;
+ private static Logger LOG = Logger.getLogger(AvroRpcWorkflowManagerClient.class.getName());
+ private Transceiver client;
+ private org.apache.oodt.cas.workflow.struct.avrotypes.WorkflowManager proxy;
+ private URL workflowManagerUrl;
public AvroRpcWorkflowManagerClient(URL url){
workflowManagerUrl = url;
try {
client = new NettyTransceiver(new InetSocketAddress(url.getHost(),url.getPort()));
- proxy = SpecificRequestor.getClient(org.apache.oodt.cas.workflow.struct.avrotypes.WorkflowManager.class, client);
-
+ proxy = SpecificRequestor.getClient(org.apache.oodt.cas.workflow.struct.avrotypes.WorkflowManager.class, client);
} catch (IOException e) {
- e.printStackTrace();
+ LOG.severe(String.format("Error occurred when creating client: %s", e.getMessage()));
}
-
}
@Override
@@ -251,4 +244,18 @@ public class AvroRpcWorkflowManagerClient implements WorkflowManagerClient {
}
}
+
+ @Override
+ public void close() throws IOException {
+ if (client != null) {
+ client.close();
+ client = null;
+ LOG.info("Closed workflow manager client: " + workflowManagerUrl.toString());
+ }
+ }
+
+ @Override
+ public void finalize() throws IOException {
+ close();
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerClient.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerClient.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerClient.java
index 9d821bc..ed0cfd1 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerClient.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/WorkflowManagerClient.java
@@ -24,6 +24,7 @@ import org.apache.oodt.cas.workflow.structs.Workflow;
import org.apache.oodt.cas.workflow.structs.WorkflowTask;
import org.apache.oodt.cas.workflow.structs.WorkflowCondition;
+import java.io.Closeable;
import java.net.URL;
import java.util.List;
import java.util.Vector;
@@ -36,10 +37,11 @@ import java.util.Vector;
* Base interface for client RPC implementation.
* </p>
*/
-public interface WorkflowManagerClient {
+public interface WorkflowManagerClient extends Closeable {
boolean refreshRepository()
throws Exception;
+
String executeDynamicWorkflow(List<String> taskIds, Metadata metadata)
throws Exception;
http://git-wip-us.apache.org/repos/asf/oodt/blob/ec25440d/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java
index 9c29cc4..f9bc6e6 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java
@@ -534,4 +534,8 @@ public class XmlRpcWorkflowManagerClient implements WorkflowManagerClient {
client = new XmlRpcClient(workflowManagerUrl);
}
+ @Override
+ public void close() throws IOException {
+
+ }
}
[07/11] oodt git commit: Merge branch 'OODT-975'
Posted by ma...@apache.org.
Merge branch 'OODT-975'
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/f3475068
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/f3475068
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/f3475068
Branch: refs/heads/development
Commit: f34750687ed1fe631d0601980f90c3ae2c9c201a
Parents: fed87b2 40ed14e
Author: Lewis John McGibbney <le...@gmail.com>
Authored: Tue May 1 10:39:30 2018 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Tue May 1 10:39:30 2018 -0700
----------------------------------------------------------------------
.../resources/archetype-resources/distribution/pom.xml | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------