You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/07/26 12:16:43 UTC
[1/5] oodt git commit: OODT-940 update solr assets
Repository: oodt
Updated Branches:
refs/heads/development 07d9030d3 -> ceadfcb2b
OODT-940 update solr assets
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/29fc6162
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/29fc6162
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/29fc6162
Branch: refs/heads/development
Commit: 29fc61620fb11360e4beae6216238e555f2afc35
Parents: e55f2cd
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Nov 10 15:59:24 2016 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Thu Nov 10 15:59:24 2016 +0000
----------------------------------------------------------------------
core/pom.xml | 4 +-
.../catalog/solr/DefaultProductSerializer.java | 310 +++++++++---------
.../cas/filemgr/catalog/solr/SolrCatalog.java | 324 ++++++++++---------
3 files changed, 334 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 6fafae6..3420597 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -382,12 +382,12 @@ the License.
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>1.3.0</version>
+ <version>6.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>1.3.0</version>
+ <version>6.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
index 72180e3..5b42a8b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
@@ -16,12 +16,14 @@
*/
package org.apache.oodt.cas.filemgr.catalog.solr;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.solr.client.solrj.util.ClientUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -44,21 +46,21 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
- * Default implementation of {@link ProductSerializer}
- * that transforms a CAS product into a single Solr document based on the following rules:
+ * Default implementation of {@link ProductSerializer}
+ * that transforms a CAS product into a single Solr document based on the following rules:
* o) the core product attributes are used to generate Solr fields starting with "CAS...."
* o) the product references are converted to Solr fields starting with "CAS.Reference..." or "CAS.RootReference..."
- * o) all other metadata fields are converted into Solr fields with the same name and number of values.
+ * o) all other metadata fields are converted into Solr fields with the same name and number of values.
* Note that the field multiplicity must be consistent with its definition in the Solr schema.xml.
- *
+ *
* This class generates all Solr documents in XML format.
- *
+ *
* @author Luca Cinquini
*
*/
public class DefaultProductSerializer implements ProductSerializer {
- private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName());
+ private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName());
/**
* {@inheritDoc}
*/
@@ -72,10 +74,10 @@ public class DefaultProductSerializer implements ProductSerializer {
*/
@Override
public List<String> serialize(Product product, boolean create) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
List<String> docs = new ArrayList<String>();
-
+
// add core product attributes to map
this.addKeyValueToMap(fields, Parameters.PRODUCT_ID, product.getProductId());
this.addKeyValueToMap(fields, Parameters.PRODUCT_NAME, product.getProductName());
@@ -84,117 +86,117 @@ public class DefaultProductSerializer implements ProductSerializer {
ProductType productType = product.getProductType();
if (productType!=null) {
this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_NAME, productType.getName());
- this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId());
}
if (create) {
// only insert date/time when product is first created
Date productDateTime = new Date(); // current datetime
this.addKeyValueToMap(fields, Parameters.PRODUCT_RECEIVED_TIME, Parameters.SOLR_DATE_TIME_FORMATTER.format(productDateTime));
}
-
+
// create new product: use Solr id == CAS id
- if (create) {
+ if (create) {
docs.add( this.generateInsertDocuments(product.getProductId(), fields) );
-
- // update existing product
- } else {
+
+ // update existing product
+ } else {
docs.addAll( this.generateUpdateDocuments(product.getProductId(), fields, true) ); // replace=true
-
+
}
-
+
return docs;
-
+
}
-
+
/**
* {@inheritDoc}
*/
public List<String> serialize(String productId, Reference rootReference, List<Reference> references, boolean replace) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
-
+
// product root reference
if (rootReference!=null) {
-
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, rootReference.getOrigReference());
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, rootReference.getDataStoreReference());
+
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(rootReference.getOrigReference()));
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(rootReference.getDataStoreReference()));
addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_FILESIZE, ""+rootReference.getFileSize());
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, rootReference.getMimeType().toString());
-
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(rootReference.getMimeType().toString()));
+
}
-
+
// all other product references
- // note that Solr will preserve the indexing order.
+ // note that Solr will preserve the indexing order.
for (Reference reference : references) {
-
- addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, reference.getOrigReference());
- addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, reference.getDataStoreReference());
+
+ addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(reference.getOrigReference()));
+ addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(reference.getDataStoreReference()));
addKeyValueToMap(fields, Parameters.REFERENCE_FILESIZE, ""+reference.getFileSize());
- addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, reference.getMimeType().toString());
-
+ addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(reference.getMimeType().toString()));
+
}
-
+
return generateUpdateDocuments(productId, fields, replace);
-
+
}
-
+
/**
* {@inheritDoc}
*/
public QueryResponse deserialize(String xml) throws CatalogException {
-
+
try {
-
+
QueryResponse queryResponse = new QueryResponse();
-
+
// parse XML into DOM
Document document = parseXml(xml);
-
+
// extract information from DOM to Product
Element response = document.getDocumentElement();
Node result = response.getElementsByTagName("result").item(0);
queryResponse.setNumFound( Integer.parseInt( ((Element)result).getAttribute("numFound") ) );
- queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) );
+ queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) );
NodeList docs = result.getChildNodes();
for (int i=0; i< docs.getLength(); i++) {
Node node = docs.item(i);
if (node.getNodeName().equals("doc")) {
- Element doc = (Element)node;
+ Element doc = (Element)node;
CompleteProduct cp = this.deserialize(doc);
queryResponse.getCompleteProducts().add(cp);
}
- }
+ }
return queryResponse;
-
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage(), e);
}
-
+
}
-
+
/**
* {@inheritDoc}
*/
public List<String> serialize(String productId, Metadata metadata, boolean replace) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
-
+
for (String key : metadata.getKeys()) {
if (! (key.startsWith(Parameters.NS) // skip metadata keys starting with reserved namespace
- || Parameters.PRODUCT_TYPE_NAME.contains(key)
- // skip 'ProductType' as already stored as 'CAS.ProductTypeName'
- || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure'
+ //|| Parameters.PRODUCT_TYPE_NAME.contains(key)
+ // skip 'ProductType' as already stored as 'CAS.ProductTypeName'
+ || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure'
for (String value : metadata.getAllMetadata(key)) {
- this.addKeyValueToMap(fields, key, value);
+ this.addKeyValueToMap(fields, key, StringEscapeUtils.escapeXml(value));
}
}
}
-
+
return this.generateUpdateDocuments(productId, fields, replace);
-
+
}
-
+
/**
* Method to add a (key, value) to a multi-valued map with appropriate checks.
* @param map
@@ -202,7 +204,7 @@ public class DefaultProductSerializer implements ProductSerializer {
* @param value
*/
protected void addKeyValueToMap(Map<String, List<String>> map, String key, String value) {
-
+
if (!map.containsKey(key)) {
map.put(key, new ArrayList<String>());
}
@@ -213,33 +215,33 @@ public class DefaultProductSerializer implements ProductSerializer {
map.get(key).add(Parameters.NULL);
}
}
-
+
/**
* Utility method to generate a Solr insert document.
- *
+ *
* @param productId
* @param fields
* @return
*/
protected String generateInsertDocuments(String productId, Map<String,List<String>> fields) {
-
+
StringBuilder doc = new StringBuilder();
doc.append("<doc>");
-
+
// product Solr id field
doc.append( encodeIndexField(Parameters.ID, productId) );
-
+
// all other fields
for (Map.Entry<String, List<String>> key : fields.entrySet()) {
for (String value : key.getValue()) {
- doc.append( encodeIndexField(key.getKey(), value) );
+ doc.append( encodeIndexField(key.getKey(), StringEscapeUtils.escapeXml(value)) );
}
}
doc.append("</doc>");
return doc.toString();
}
-
+
/**
* Utility method to generate Solr update documents.
* Note that the requests for setting/adding/deleting fields must be sent as separate documents to Solr
@@ -249,51 +251,51 @@ public class DefaultProductSerializer implements ProductSerializer {
* @return
*/
protected List<String> generateUpdateDocuments(String productId, Map<String,List<String>> fields, boolean replace) {
-
+
// list for different instruction types
List<String> setFields = new ArrayList<String>();
List<String> addFields = new ArrayList<String>();
List<String> delFields = new ArrayList<String>();
-
+
// encode update instructions
for (Map.Entry<String, List<String>> key : fields.entrySet()) {
-
+
List<String> values = key.getValue();
-
+
if (replace) {
-
+
if (values.isEmpty()) {
// use special value to flag removal
delFields.add( this.encodeUpdateField(key.getKey(), Parameters.NULL, true) );
-
+
} else {
for (String value : values) {
- setFields.add( this.encodeUpdateField(key.getKey(), value, true) );
+ setFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), true) );
}
}
-
+
} else {
for (String value : values) {
- addFields.add( this.encodeUpdateField(key.getKey(), value, false) );
+ addFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), false) );
}
}
-
+
}
-
+
List<String> docs = new ArrayList<String>();
if (!delFields.isEmpty()) {
- docs.add(toDoc(productId, delFields));
+ docs.add(toDoc(productId, delFields));
}
if (!setFields.isEmpty()) {
- docs.add(toDoc(productId, setFields));
+ docs.add(toDoc(productId, setFields));
}
if (!addFields.isEmpty()) {
- docs.add(toDoc(productId, addFields));
+ docs.add(toDoc(productId, addFields));
}
return docs;
-
+
}
-
+
/**
* Utility method to merge field update instructions into a single document.
* @param productId
@@ -301,24 +303,24 @@ public class DefaultProductSerializer implements ProductSerializer {
* @return
*/
private String toDoc(String productId, List<String> updates) {
-
+
StringBuilder doc = new StringBuilder();
doc.append("<doc>");
-
+
// reference product record id
doc.append( encodeIndexField(Parameters.ID, productId) );
-
+
// loop over field update instructions
for (String update : updates) {
doc.append(update);
}
-
+
doc.append("</doc>");
-
+
return doc.toString();
-
+
}
-
+
/**
* Method to encode a Solr field indexing instruction.
* If the value is null, the empty string is returned.
@@ -333,11 +335,11 @@ public class DefaultProductSerializer implements ProductSerializer {
return "<field name=\""+key+"\">" + value + "</field>";
}
}
-
+
/**
* Method to encode a field update instruction for the three possible cases:
* add new values to a key (1), replace current values for a key (2), remove all values for a key (3).
- *
+ *
* @param key
* @param value
* @param replace
@@ -346,123 +348,135 @@ public class DefaultProductSerializer implements ProductSerializer {
protected String encodeUpdateField(String key, String value, boolean replace) {
StringBuilder sb = new StringBuilder();
sb.append("<field name=\"").append(key).append("\"");
-
+
if (replace) {
-
+
if (value==null || value.equals(Parameters.NULL)) {
-
+
// (3) remove all values for given key
sb.append(" update=\"set\" null=\"true\" />");
-
+
} else {
-
+
// (2) replace existing values with new values
sb.append(" update=\"set\">").append(value).append("</field>");
}
-
+
} else {
-
+
// (1) add new values to existing values
sb.append(" update=\"add\">").append(value).append("</field>");
-
+
}
return sb.toString();
}
-
-
+
+
/**
* Method that parses a single Solr document snippet
* to extract Product and Metadata attributes.
- *
+ *
* @param doc
* @return
*/
private CompleteProduct deserialize(Element doc) {
-
+
CompleteProduct cp = new CompleteProduct();
Product product = cp.getProduct();
ProductType productType = product.getProductType();
Metadata metadata = cp.getMetadata();
List<Reference> references = product.getProductReferences();
Reference rootReference = product.getRootRef();
-
+
NodeList children = doc.getChildNodes();
for (int j=0; j<children.getLength(); j++) {
-
+
Node child = children.item(j);
Element element = (Element)child;
String name = element.getAttribute("name");
-
+
/**
*<arr name="ScanPointingSource">
* <str>G073.65+0.19</str>
* <str>J2015+3410</str>
- * ..........
+ * ..........
*/
if (child.getNodeName().equals("arr")) {
-
+
NodeList values = element.getChildNodes();
List<String> vals = new ArrayList<String>();
for (int k=0; k<values.getLength(); k++) {
String value = ((Element)values.item(k)).getTextContent();
- vals.add(value);
+ vals.add(StringEscapeUtils.unescapeXml(value));
}
// CAS.reference.... fields
- if (name.startsWith(Parameters.NS)) {
- for (int k=0; k<values.getLength(); k++) {
- // create this reference
- if (references.size()<=k) {
- references.add(new Reference());
- }
- if (name.equals(Parameters.REFERENCE_ORIGINAL)) {
- references.get(k).setOrigReference(vals.get(k));
- } else if (name.equals(Parameters.REFERENCE_DATASTORE)) {
- references.get(k).setDataStoreReference(vals.get(k));
- } else if (name.equals(Parameters.REFERENCE_FILESIZE)) {
- references.get(k).setFileSize(Long.parseLong(vals.get(k)));
- } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) {
- references.get(k).setMimeType(vals.get(k));
- }
+ if (name.startsWith(Parameters.NS)) {
+ for (int k=0; k<values.getLength(); k++) {
+ // create this reference
+ if (references.size()<=k) {
+ references.add(new Reference());
}
- // all other multi-valued fields
+ if (name.equals(Parameters.REFERENCE_ORIGINAL)) {
+ references.get(k).setOrigReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_DATASTORE)) {
+ references.get(k).setDataStoreReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_FILESIZE)) {
+ references.get(k).setFileSize(Long.parseLong(vals.get(k)));
+ } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) {
+ references.get(k).setMimeType(vals.get(k));
+ }
+ }
+ // all other multi-valued fields
} else {
this.deserializeMultiValueField(name, vals, metadata);
}
-
- /**
- * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
- * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
- * <str name="CAS.ProductName">tns_br145x4_20</str>
- * <str name="FileLocation">/usr/local/ska-dc/data/archive</str>
- * ...........
- */
+
+ /**
+ * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductName">tns_br145x4_20</str>
+ * <str name="FileLocation">/usr/local/ska-dc/data/archive</str>
+ * ...........
+ */
} else {
-
- String value = element.getTextContent();
-
+
+ String value = StringEscapeUtils.unescapeXml(element.getTextContent());
+
// core CAS fields
if (name.startsWith(Parameters.NS)) {
if (name.equals(Parameters.PRODUCT_ID)) {
product.setProductId(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_NAME)) {
product.setProductName(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_STRUCTURE)) {
product.setProductStructure(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TRANSFER_STATUS)) {
product.setTransferStatus(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TYPE_NAME)) {
productType.setName(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TYPE_ID)) {
productType.setProductTypeId(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_RECEIVED_TIME)) {
product.setProductRecievedTime(value);
metadata.addMetadata(name, value);
// CAS root reference
} else if (name.startsWith(Parameters.NS+Parameters.ROOT)) {
if (rootReference==null) {
- rootReference = new Reference();
+ rootReference = new Reference();
}
if (name.equals(Parameters.ROOT_REFERENCE_ORIGINAL)) {
rootReference.setOrigReference(value);
@@ -475,48 +489,48 @@ public class DefaultProductSerializer implements ProductSerializer {
}
}
-
- // non core single-valued fields
+
+ // non core single-valued fields
} else {
this.deserializeSingleValueField(name, value, metadata);
} // "CAS".... or not
-
+
} // "arr" or anything else
-
+
} // loop over <doc> children
-
+
return cp;
-
+
}
-
+
private Document parseXml(String xml) throws IOException, SAXException, ParserConfigurationException {
-
+
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder parser = factory.newDocumentBuilder();
- return parser.parse( new InputSource(new StringReader(xml)) );
-
+ return parser.parse( new InputSource(new StringReader(xml)) );
+
}
-
+
/**
* Method that deserializes a single-valued Solr field into a Metadata element.
* This method can be overridden by sub-classes to provide custom behavior.
- *
+ *
* @param name : the Solr field name
* @param value : the Solr field single value
* @param metadata : the metadata container
*/
protected void deserializeSingleValueField(String name, String value, Metadata metadata) {
- // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId)
+ // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId)
if (!name.equals(Parameters.ID)){
metadata.addMetadata(name, value);
}
}
-
+
/**
* Method that deserializes a multi-valued Solr field into a Metadata element.
* This method can be overridden by sub-classes to provide custom behavior.
- *
+ *
* @param name : the Solr field name
* @param values : the Solr field multiple values
* @param metadata : the metadata container
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
index af554c5..a734e91 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
@@ -38,23 +38,23 @@ import org.springframework.util.StringUtils;
/**
* Implementation of the CAS {@link Catalog} interface
* that uses a Solr back-end metadata store.
- *
+ *
* @author Luca Cinquini
*
*/
public class SolrCatalog implements Catalog {
-
+
// Class responsible for serializing/deserializing CAS products into Solr documents
ProductSerializer productSerializer;
-
+
// Class responsible for generating unique identifiers for incoming Products
ProductIdGenerator productIdGenerator;
-
+
// Class responsible for interacting with the Solr server
SolrClient solrClient;
-
+
private static final Logger LOG = Logger.getLogger(SolrCatalog.class.getName());
-
+
public SolrCatalog(String solrUrl, ProductIdGenerator productIdGenerator, ProductSerializer productSerializer) {
this.productIdGenerator = productIdGenerator;
this.productSerializer = productSerializer;
@@ -63,13 +63,15 @@ public class SolrCatalog implements Catalog {
@Override
public void addMetadata(Metadata metadata, Product product) throws CatalogException {
-
+
LOG.info("Adding metadata for product:"+product.getProductName());
-
+ if(metadata.containsKey("_version_")){
+ metadata.removeMetadata("_version_");
+ }
// serialize metadadta to Solr document(s)
- // replace=false i.e. add metadata to existing values
- List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false);
-
+ // replace=false i.e. add metadata to existing values
+ List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false);
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -82,39 +84,42 @@ public class SolrCatalog implements Catalog {
*/
@Override
public void removeMetadata(Metadata metadata, Product product) throws CatalogException {
-
+
// retrieve full existing metadata
Metadata currentMetadata = getMetadata(product);
-
+
// metadata to be updated
Metadata updateMetadata = new Metadata();
-
+
// loop over keys of metadata be updated
for (String key : metadata.getKeys()) {
-
+
// list of values remaining after removal
List<String> values = new ArrayList<String>();
if (currentMetadata.containsKey(key)) {
for (String value : currentMetadata.getAllMetadata(key)) {
if (!metadata.getAllMetadata(key).contains(value)) {
values.add(value);
- }
+ }
}
// add remaining values to updated metadata
if (values.isEmpty()) {
// special value because Metadata will NOT store an empty list
values.add(Parameters.NULL);
- }
+ }
updateMetadata.addMetadata(key, values);
-
+
}
}
-
+
+ if(updateMetadata.containsKey("_version_")){
+ updateMetadata.removeMetadata("_version_");
+ }
// generate Solr update documents
// replace=true to override existing values
List<String> docs = productSerializer.serialize(product.getProductId(), updateMetadata, true);
-
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -127,40 +132,51 @@ public class SolrCatalog implements Catalog {
*/
@Override
public void addProduct(Product product) throws CatalogException {
-
- LOG.info("Adding product:"+product.getProductName());
-
- // generate product identifier if not existing already
- if (!StringUtils.hasText(product.getProductId())) {
- String productId = this.productIdGenerator.generateId(product);
- product.setProductId(productId);
+
+ if(product.getProductId()!=null && this.getCompleteProductById(product.getProductId()) !=null) {
+ throw new CatalogException(
+ "Attempt to add a product that already existed: product: ["
+ + product.getProductName() + "]");
+
+
+
+
+
+ } else {
+ LOG.info("Adding product:" + product.getProductName());
+
+ // generate product identifier if not existing already
+ if (!StringUtils.hasText(product.getProductId())) {
+ String productId = this.productIdGenerator.generateId(product);
+ product.setProductId(productId);
+ }
+
+ // serialize product for ingestion into Solr
+ List<String> docs = productSerializer.serialize(product, true); // create=true
+
+ // send records to Solr
+ solrClient.index(docs, true, productSerializer.getMimeType());
}
-
- // serialize product for ingestion into Solr
- List<String> docs = productSerializer.serialize(product, true); // create=true
-
- // send records to Solr
- solrClient.index(docs, true, productSerializer.getMimeType());
-
+
}
@Override
public void modifyProduct(Product product) throws CatalogException {
-
+
LOG.info("Modifying product:"+product.getProductName());
-
+
// serialize the update product information to Solr document(s)
List<String> docs = productSerializer.serialize(product, false); // create=false
-
+
// send records to Solr
solrClient.index(docs, true, productSerializer.getMimeType());
-
+
}
@Override
public void removeProduct(Product product) throws CatalogException {
-
+
// send message to Solr server
solrClient.delete(product.getProductId(), true);
@@ -168,17 +184,17 @@ public class SolrCatalog implements Catalog {
@Override
public void setProductTransferStatus(Product product) throws CatalogException {
-
+
this.modifyProduct(product);
}
@Override
public void addProductReferences(Product product) throws CatalogException {
-
+
// generate update documents (with replace=true)
- List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true);
-
+ List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true);
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -186,51 +202,51 @@ public class SolrCatalog implements Catalog {
@Override
public Product getProductById(String productId) throws CatalogException {
-
- CompleteProduct cp = getCompleteProductById(productId);
- return cp.getProduct();
-
+
+ CompleteProduct cp = getCompleteProductById(productId);
+ return cp.getProduct();
+
}
@Override
public Product getProductByName(String productName) throws CatalogException {
-
- CompleteProduct cp = getCompleteProductByName(productName);
+
+ CompleteProduct cp = getCompleteProductByName(productName);
if (cp!=null) {
- LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId());
- return cp.getProduct();
+ LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId());
+ return cp.getProduct();
} else {
- LOG.info("Product with name="+productName+" not found");
- return null;
+ LOG.info("Product with name="+productName+" not found");
+ return null;
}
-
+
}
@Override
public List<Reference> getProductReferences(Product product) throws CatalogException {
-
+
CompleteProduct cp = getCompleteProductById(product.getProductId());
return cp.getProduct().getProductReferences();
-
+
}
@Override
public List<Product> getProducts() throws CatalogException {
-
+
// build query parameters
Map<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[] { "*:*" } );
//params.put("rows", new String[] { "20" } ); // control pagination ?
-
+
// query Solr for all matching products
return getProducts(params, 0, -1).getProducts(); // get ALL products
-
+
}
-
+
/**
* Common utility to retrieve a range of products matching the specified {@link Query} and {@link ProductType}.
* This method transforms the given constraints in a map of HTTP (name, value) pairs and delegates to the following method.
- *
+ *
* @param query
* @param type
* @param offset
@@ -239,25 +255,25 @@ public class SolrCatalog implements Catalog {
* @throws CatalogException
*/
private QueryResponse getProducts(Query query, ProductType type, int offset, int limit) throws CatalogException {
-
+
// build HTTP request
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
// product type constraint
params.put("q", new String[]{Parameters.PRODUCT_TYPE_NAME+":"+type.getName()} );
- // convert filemgr query into a Solr query
+ // convert filemgr query into a Solr query
List<String> qc = new ArrayList<String>();
- for (QueryCriteria queryCriteria : query.getCriteria()) {
- LOG.info("Query criteria="+queryCriteria.toString());
- qc.add(queryCriteria.toString());
- }
- params.put("fq", qc.toArray( new String[ qc.size() ] ));
- // sort
- params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
-
- return this.getProducts(params, offset, limit);
-
- }
-
+ for (QueryCriteria queryCriteria : query.getCriteria()) {
+ LOG.info("Query criteria="+queryCriteria.toString());
+ qc.add(queryCriteria.toString());
+ }
+ params.put("fq", qc.toArray( new String[ qc.size() ] ));
+ // sort
+ params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
+
+ return this.getProducts(params, offset, limit);
+
+ }
+
/**
* Common utility to retrieve a range of products matching the specified query parameters
* @param params : HTTP query parameters used for query to Solr
@@ -266,38 +282,38 @@ public class SolrCatalog implements Catalog {
* @return
*/
private QueryResponse getProducts(Map<String, String[]> params, int offset, int limit) throws CatalogException {
-
+
// combined results from pagination
QueryResponse queryResponse = new QueryResponse();
queryResponse.setStart(offset);
int start = offset;
while (queryResponse.getCompleteProducts().size()<limit || limit<0) {
-
+
params.put("start", new String[] { ""+start } );
String response = solrClient.query(params, productSerializer.getMimeType());
QueryResponse qr = productSerializer.deserialize(response);
-
+
for (CompleteProduct cp : qr.getCompleteProducts()) {
if (queryResponse.getCompleteProducts().size()<limit) {
queryResponse.getCompleteProducts().add( cp );
}
}
-
+
queryResponse.setNumFound( qr.getNumFound() );
start = offset+queryResponse.getCompleteProducts().size();
if (limit<0) {
- limit = queryResponse.getNumFound(); // retrieve ALL results
+ limit = queryResponse.getNumFound(); // retrieve ALL results
}
if (start>=queryResponse.getNumFound()) {
- break; // don't query any longer
+ break; // don't query any longer
}
-
+
}
-
+
LOG.info("Total number of products found="+queryResponse.getNumFound());
LOG.info("Total number of products returned="+queryResponse.getCompleteProducts().size());
return queryResponse;
-
+
}
@Override
@@ -308,7 +324,7 @@ public class SolrCatalog implements Catalog {
params.put("q", new String[] { "*:*" } );
// use the product type name as query parameter
params.put("fq", new String[] { Parameters.PRODUCT_TYPE_NAME+":"+type.getName() } );
-
+
// query Solr for all matching products
return getProducts(params, 0, -1).getProducts(); // get ALL products
@@ -316,28 +332,28 @@ public class SolrCatalog implements Catalog {
@Override
public Metadata getMetadata(Product product) throws CatalogException {
-
+
CompleteProduct cp = getCompleteProductById(product.getProductId());
return cp.getMetadata();
-
+
}
@Override
public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException {
-
+
// build HTTP request
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{Parameters.PRODUCT_ID+":"+product.getProductId()} );
// request metadata elements explicitly
params.put("fl", elements.toArray(new String[elements.size()]) );
-
+
// execute request
String doc = solrClient.query(params, productSerializer.getMimeType());
-
+
// parse response
CompleteProduct cp = extractCompleteProduct(doc);
return cp.getMetadata();
-
+
}
/**
@@ -346,50 +362,50 @@ public class SolrCatalog implements Catalog {
*/
@Override
public List<String> query(Query query, ProductType type) throws CatalogException {
-
- // execute request for ALL results
- QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products
+
+ // execute request for ALL results
+ QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products
// extract ids from products
List<String> ids = new ArrayList<String>();
for (CompleteProduct cp : queryResponse.getCompleteProducts()) {
ids.add(cp.getProduct().getProductId());
}
-
+
return ids;
-
+
}
@Override
- public List<Product> getTopNProducts(int n) throws CatalogException {
-
+ public List<Product> getTopNProducts(int n) throws CatalogException {
+
// retrieve most recent n products from Solr
String doc = solrClient.queryProductsByDate(n, productSerializer.getMimeType());
-
+
// parse Solr response into Product objects
return this.getProductsFromDocument(doc);
-
+
}
-
+
@Override
public ProductPage pagedQuery(Query query, ProductType type, int pageNum) throws CatalogException {
-
-
- // execute request for one page of results
- int offset = (pageNum-1)*Parameters.PAGE_SIZE;
- int limit = Parameters.PAGE_SIZE;
- QueryResponse queryResponse = this.getProducts(query, type, offset, limit);
-
- // build product page from query response
- return newProductPage(pageNum, queryResponse);
-
- }
-
+
+
+ // execute request for one page of results
+ int offset = (pageNum-1)*Parameters.PAGE_SIZE;
+ int limit = Parameters.PAGE_SIZE;
+ QueryResponse queryResponse = this.getProducts(query, type, offset, limit);
+
+ // build product page from query response
+ return newProductPage(pageNum, queryResponse);
+
+ }
+
@Override
public ProductPage getFirstPage(ProductType type) {
try {
return this.pagedQuery(new Query(), type, 1);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
@@ -397,82 +413,82 @@ public class SolrCatalog implements Catalog {
@Override
public ProductPage getLastProductPage(ProductType type) {
-
+
try {
-
+
// query for total number of products of this type
int numTotalResults = this.getNumProducts(type);
-
+
// compute last page number
int numOfPages = PaginationUtils.getTotalPage(numTotalResults, Parameters.PAGE_SIZE);
-
+
// request last page
return pagedQuery(new Query(), type, numOfPages);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
@Override
public ProductPage getNextPage(ProductType type, ProductPage currentPage) {
-
+
int nextPageNumber = currentPage.getPageNum()+1;
if (nextPageNumber>currentPage.getTotalPages()) {
- throw new RuntimeException("Invalid next page number: " + nextPageNumber);
+ throw new RuntimeException("Invalid next page number: " + nextPageNumber);
}
try {
return this.pagedQuery(new Query(), type, currentPage.getPageNum()+1);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
@Override
public ProductPage getPrevPage(ProductType type, ProductPage currentPage) {
-
+
int prevPageNumber = currentPage.getPageNum()-1;
if (prevPageNumber<=0) {
- throw new RuntimeException("Invalid previous page number: " + prevPageNumber);
+ throw new RuntimeException("Invalid previous page number: " + prevPageNumber);
}
-
+
try {
return this.pagedQuery(new Query(), type, prevPageNumber);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
-
+
/**
* Common functionality for extracting products from a Solr response document.
* @param doc
* @return
*/
private List<Product> getProductsFromDocument(String doc) throws CatalogException {
-
+
// extract full product information from Solr response
QueryResponse queryResponse = productSerializer.deserialize(doc);
-
+
// return products only
return queryResponse.getProducts();
-
+
}
@Override
public List<Product> getTopNProducts(int n, ProductType type) throws CatalogException {
-
+
// retrieve most recent n products from Solr
String doc = solrClient.queryProductsByDateAndType(n, type, productSerializer.getMimeType());
-
+
// parse Solr response into Product objects
return this.getProductsFromDocument(doc);
-
+
}
@Override
@@ -484,43 +500,43 @@ public class SolrCatalog implements Catalog {
@Override
public int getNumProducts(ProductType type) throws CatalogException {
-
+
// build query parameters
Map<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[] { "CAS.ProductTypeName:"+type.getName() } );
params.put("rows", new String[] { "0" } ); // don't return any results
-
+
// execute query
String response = solrClient.query(params, productSerializer.getMimeType());
-
+
// parse response
QueryResponse queryResponse = productSerializer.deserialize(response);
return queryResponse.getNumFound();
-
+
}
-
+
private CompleteProduct getCompleteProductById(String productId) throws CatalogException {
-
+
// request document with given id
String doc = solrClient.queryProductById(productId, productSerializer.getMimeType());
-
+
// parse document into complete product
return extractCompleteProduct(doc);
-
+
}
-
- private CompleteProduct getCompleteProductByName(String productName) throws CatalogException {
-
+
+ private CompleteProduct getCompleteProductByName(String productName) throws CatalogException {
+
// request document with given id
String doc = solrClient.queryProductByName(productName, productSerializer.getMimeType());
-
+
// parse document into complete product
return extractCompleteProduct(doc);
-
+
}
-
+
private CompleteProduct extractCompleteProduct(String doc) throws CatalogException {
-
+
// deserialize document into Product
LOG.info("Parsing Solr document: "+doc);
QueryResponse queryResponse = productSerializer.deserialize(doc);
@@ -532,7 +548,7 @@ public class SolrCatalog implements Catalog {
} else {
return queryResponse.getCompleteProducts().get(0);
}
-
+
}
/**
@@ -541,7 +557,7 @@ public class SolrCatalog implements Catalog {
* @return
*/
private ProductPage newProductPage(int pageNum, QueryResponse queryResponse) {
-
+
ProductPage page = new ProductPage();
page.setPageNum(pageNum);
page.setPageSize(queryResponse.getProducts().size());
@@ -549,7 +565,7 @@ public class SolrCatalog implements Catalog {
page.setPageProducts(queryResponse.getProducts());
page.setTotalPages(PaginationUtils.getTotalPage(queryResponse.getNumFound(), Parameters.PAGE_SIZE));
return page;
-
+
}
}
[2/5] oodt git commit: update some dependenceies
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 5cba7d7..44ee93a 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -27,6 +27,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
@@ -34,14 +35,12 @@ import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
-import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.springframework.util.StringUtils;
@@ -74,31 +73,31 @@ public class SolrIndexer {
private final static String ACCESS_KEY = "access.key";
private final static String ACCESS_URL = "access.url";
private final static String PRODUCT_NAME = "CAS.ProductName";
+ private final HttpSolrClient server;
private IndexerConfig config = null;
- private final SolrServer server;
private String fmUrl;
private String solrUrl;
private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
private final static SimpleDateFormat solrFormat = new SimpleDateFormat(
- "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+ "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
/**
* Constructor reads in the configuration and initiates the connection to the
* Solr instance.
- *
+ *
* @param solrUrl
* URL for the Solr instance.
* @param fmUrl
* URL for the File Manager instance.
*/
public SolrIndexer(String solrUrl, String fmUrl)
- throws InstantiationException {
+ throws InstantiationException {
InputStream input = null;
String filename;
try {
LOG.info("System property " + SOLR_INDEXER_CONFIG + " set to "
- + System.getProperty(SOLR_INDEXER_CONFIG));
+ + System.getProperty(SOLR_INDEXER_CONFIG));
filename = System.getProperty(SOLR_INDEXER_CONFIG);
if (filename != null) {
LOG.info("Reading config from " + filename);
@@ -110,7 +109,7 @@ public class SolrIndexer {
config = new IndexerConfig(input);
} catch (IOException e) {
LOG
- .severe("Could not read in configuration for indexer from classpath or file");
+ .severe("Could not read in configuration for indexer from classpath or file");
throw new InstantiationException(e.getMessage());
} finally {
if (input != null) {
@@ -133,13 +132,10 @@ public class SolrIndexer {
}
LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl);
+ SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ server = new HttpSolrClient(this.solrUrl, httpClient);
- try {
- server = new CommonsHttpSolrServer(this.solrUrl);
- } catch (MalformedURLException e) {
- LOG.severe("Could not connect to Solr server " + this.solrUrl);
- throw new InstantiationException(e.getMessage());
- }
+// server = new SolrClient(this.solrUrl);
}
@@ -166,7 +162,7 @@ public class SolrIndexer {
/**
* This method transforms the product metadata into a Solr document.
- *
+ *
* @param metadata
* The metadata object for the product to index.
* @return Returns the SolrInputDocument containing product metadata.
@@ -174,6 +170,8 @@ public class SolrIndexer {
private SolrInputDocument getSolrDocument(Metadata metadata) {
SolrInputDocument doc = new SolrInputDocument();
// Only grab metadata which have a mapping in the indexer.properties
+ String official = "";
+
for (Object objKey : config.getMapProperties().keySet()) {
// The key in the metadata object
String key = (String) objKey;
@@ -184,19 +182,21 @@ public class SolrIndexer {
for (String value : values) {
// Add each metadata value into the
if (value != null && !config.getIgnoreValues().contains(value.trim())) {
- LOG.fine("Adding field: " + fieldName + " value: " + value);
+ official += " Key: " + key + " Value: " + value;
doc.addField(fieldName, value);
}
}
}
}
+ // LOG.info("KEYS WERE:" +official);
+
return doc;
}
/**
* This method adds a single product extracted from a metadata file to the
* Solr index.
- *
+ *
* @param file
* The file containing product metadata.
* @param delete
@@ -206,24 +206,24 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexMetFile(File file, boolean delete)
- throws
- SolrServerException {
+ throws
+ SolrServerException {
LOG.info("Attempting to index product from metadata file.");
try {
SerializableMetadata metadata = new SerializableMetadata("UTF-8", false);
metadata.loadMetadataFromXmlStream(new FileInputStream(file));
metadata.addMetadata("id", metadata.getMetadata("CAS."
- + CoreMetKeys.PRODUCT_ID));
+ + CoreMetKeys.PRODUCT_ID));
metadata.addMetadata(config.getProperty(ACCESS_KEY), config
- .getProperty(ACCESS_URL)
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .getProperty(ACCESS_URL)
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
if (delete) {
server
- .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
}
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: "
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
} catch (InstantiationException e) {
LOG.severe("Could not instantiate metadata object: " + e.getMessage());
} catch (FileNotFoundException e) {
@@ -236,7 +236,7 @@ public class SolrIndexer {
/**
* This method indexes all product types retrieved from the File Manager to
* the Solr index.
- *
+ *
* @param delete
* Flag indicating whether each product type retrieved from the File
* Manager should be deleted from the index.
@@ -246,7 +246,8 @@ public class SolrIndexer {
public void indexProductTypes(boolean delete) {
LOG.info("Indexing product types...");
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -255,9 +256,9 @@ public class SolrIndexer {
metadata.addMetadata("id", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeId", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeDescription", type
- .getDescription());
+ .getDescription());
metadata.addMetadata("CAS.ProductTypeRepositoryPath", type
- .getProductRepositoryPath());
+ .getProductRepositoryPath());
metadata.addMetadata("CAS.ProductTypeVersioner", type.getVersioner());
metadata.addMetadata("CAS.ProductTypeName", type.getName());
metadata.addMetadata("ProductType", "ProductType");
@@ -267,7 +268,7 @@ public class SolrIndexer {
server.deleteById(type.getProductTypeId());
} catch (Exception e) {
LOG.severe("Could not delete product type " + type.getName()
- + " from index: " + e.getMessage());
+ + " from index: " + e.getMessage());
}
}
try {
@@ -276,7 +277,7 @@ public class SolrIndexer {
LOG.info("Indexed product type: " + type.getName());
} catch (Exception e) {
LOG.severe("Could not index " + type.getName() + ": "
- + e.getMessage());
+ + e.getMessage());
}
} else {
LOG.info("Ignoring product type: " + type.getName());
@@ -288,15 +289,15 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
LOG.info("Finished indexing product types.");
}
-
+
/**
- * Suppresses exception that occurred with older file managers.
+ * Suppresses exception that occurred with older file managers.
*/
- private ProductPage safeFirstPage(FileManagerClient fmClient, ProductType type) {
+ private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) {
ProductPage page = null;
try {
page = fmClient.getFirstPage(type);
@@ -310,7 +311,7 @@ public class SolrIndexer {
* This method indexes all products retrieved from the File Manager to the
* Solr index. Metadata from the product's associated ProductType is also
* included.
- *
+ *
* @param delete
* Flag indicating whether each product retrieved from the File
* Manager should be deleted from the index.
@@ -320,28 +321,30 @@ public class SolrIndexer {
public void indexAll(boolean delete) {
LOG.info("Indexing products...");
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
if (!config.getIgnoreTypes().contains(type.getName().trim())) {
+ ProductPage page = safeFirstPage(fmClient, type);
LOG.info("Paging through products for product type: "
- + type.getName());
- ProductPage page = safeFirstPage(fmClient, type);
+ + type.getName()+" Total pages are "+page.getTotalPages());
while (page != null) {
- for (Product product : page.getPageProducts()) {
+ for (Product product : page.getPageProducts()) {
try {
- this.indexProduct(product.getProductId(), fmClient
- .getMetadata(product), type.getTypeMetadata());
+ String p = product.getProductId();
+ Metadata m = fmClient.getMetadata(product);
+ this.indexProduct(p, m, type.getTypeMetadata());
} catch (Exception e) {
LOG.severe("Could not index " + product.getProductId() + ": "
- + e.getMessage());
+ + e.getLocalizedMessage());
}
}
- if (page.isLastPage()) {
- break;
- }
- page = fmClient.getNextPage(type, page);
+ if (page.getPageNum() >= page.getTotalPages() || page.isLastPage()) {
+ break;
+ }
+ page = fmClient.getNextPage(type, page);
}
}
}
@@ -352,10 +355,10 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve products from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
}
@@ -363,38 +366,50 @@ public class SolrIndexer {
* This method adds a single product retrieved from the File Manager by its
* product identifier to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productId
* The identifier of the product (CAS.ProductId).
* @throws SolrServerException
* When an error occurs communicating with the Solr server instance.
*/
public void indexProduct(String productId)
- throws SolrServerException {
+ throws SolrServerException {
LOG.info("Attempting to index product: " + productId);
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
Product product = fmClient.getProductById(productId);
Metadata productMetadata = fmClient.getMetadata(product);
+ List<String> keys = productMetadata.getAllKeys();
+ String logger = "";
+ for(String k :keys){
+ logger+=" key: "+keys;
+ List<String> values = productMetadata.getAllValues(k);
+
+ for(String v : values){
+ logger+=" value: "+v + ", ";
+ }
+ }
+ LOG.info("Metadata: "+ logger);
indexProduct(product.getProductId(), productMetadata, product
- .getProductType().getTypeMetadata());
+ .getProductType().getTypeMetadata());
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
-
+
/**
* This method adds a single product retrieved from the File Manager by its
* product name to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productName
* The identifier of the product (CAS.ProductId).
* @param delete
@@ -404,7 +419,7 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexProductByName(String productName, boolean delete) throws SolrServerException {
-
+
LOG.info("Attempting to index product: " + productName);
try {
@@ -423,27 +438,28 @@ public class SolrIndexer {
}
}
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
Product product = fmClient.getProductByName(productName);
Metadata productMetadata = fmClient.getMetadata(product);
// NOTE: delete (by id) is now false
indexProduct(product.getProductId(), productMetadata, product.getProductType().getTypeMetadata());
-
+
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
private void indexProduct(String productId, Metadata productMetadata,
- Metadata typeMetadata) throws SolrServerException,
- java.text.ParseException {
+ Metadata typeMetadata) throws SolrServerException,
+ java.text.ParseException {
Metadata metadata = new Metadata();
metadata.addMetadata("id", productId);
// Add in product type metadata
@@ -469,7 +485,7 @@ public class SolrIndexer {
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: " + productId);
} catch (IOException e) {
- LOG.severe("Could not index product: " + productId);
+ LOG.severe("Could not index product: " + productId+ "Exception:"+e.getLocalizedMessage());
}
} else {
LOG.info("Could not find metadata for product: " + productId);
@@ -478,7 +494,7 @@ public class SolrIndexer {
/**
* This method deletes a single product identified by a productID from the Solr index
- *
+ *
* @param productId
* @throws IOException
* @throws SolrServerException
@@ -490,7 +506,7 @@ public class SolrIndexer {
/**
* This method deletes a product(s) from the Solr index identified by a given name
- *
+ *
* @param productName
* @throws IOException
* @throws SolrServerException
@@ -514,16 +530,16 @@ public class SolrIndexer {
private void deleteProductFromIndex(String productId) throws IOException, SolrServerException {
server.deleteById(productId);
}
-
+
/**
* Quick helper method to do substitution on the keys specified in the config
- *
+ *
* @param metadata
* to substitute on
* @throws java.text.ParseException
*/
private void performSubstitution(Metadata metadata)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Do metadata replacement
for (String key : config.getReplacementKeys()) {
List<String> values = metadata.getAllMetadata(key);
@@ -543,10 +559,12 @@ public class SolrIndexer {
List<String> values = metadata.getAllMetadata(keyString);
if (values != null) {
List<String> newValues = new ArrayList<String>();
- SimpleDateFormat format = new SimpleDateFormat(config
- .getFormatProperties().getProperty(keyString).trim());
+
+ /* SimpleDateFormat format = new SimpleDateFormat(config
+ .getFormatProperties().getProperty(keyString).trim());*/
for (String value : values) {
- newValues.add(formatDate(format, value));
+ String d = value.trim();
+ newValues.add(d);
}
metadata.removeMetadata(keyString);
metadata.addMetadata(keyString, newValues);
@@ -556,17 +574,17 @@ public class SolrIndexer {
}
private String formatDate(SimpleDateFormat format, String value)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Ignore formating if its an ignore value
if (config.getIgnoreValues().contains(value.trim())) {
- return value;
+ return value;
}
return solrFormat.format(format.parse(value));
}
/**
* This method builds the command-line options.
- *
+ *
* @return Returns the supported Options.
*/
@SuppressWarnings("static-access")
@@ -575,31 +593,31 @@ public class SolrIndexer {
options.addOption(new Option("h", "help", false, "Print this message"));
options.addOption(new Option("o", "optimize", false,
- "Optimize the Solr index"));
+ "Optimize the Solr index"));
options.addOption(new Option("d", "delete", false,
- "Delete item before indexing"));
+ "Delete item before indexing"));
options.addOption(OptionBuilder.withArgName("Solr URL").hasArg()
- .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
- .create("su"));
+ .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
+ .create("su"));
options.addOption(OptionBuilder.withArgName("Filemgr URL").hasArg()
- .withDescription("URL to the File Manager").withLongOpt("fmUrl")
- .create("fmu"));
+ .withDescription("URL to the File Manager").withLongOpt("fmUrl")
+ .create("fmu"));
OptionGroup group = new OptionGroup();
Option all = new Option("a", "all", false,
- "Index all products from the File Manager");
+ "Index all products from the File Manager");
Option product = OptionBuilder.withArgName("productId").hasArg()
- .withDescription("Index the product from the File Manager")
- .withLongOpt("product").create("p");
+ .withDescription("Index the product from the File Manager")
+ .withLongOpt("product").create("p");
Option met = OptionBuilder.withArgName("file").hasArg().withDescription(
- "Index the product from a metadata file").withLongOpt("metFile")
- .create("mf");
+ "Index the product from a metadata file").withLongOpt("metFile")
+ .create("mf");
Option read = new Option("r", "read", false,
- "Index all products based on a list of product identifiers passed in");
+ "Index all products based on a list of product identifiers passed in");
Option types = new Option("t", "types", false,
- "Index all product types from the File Manager");
+ "Index all product types from the File Manager");
Option deleteAll = new Option("da", "deleteAll", false,
- "Delete all products/types from the Solr index");
+ "Delete all products/types from the Solr index");
group.addOption(all);
group.addOption(product);
@@ -614,7 +632,7 @@ public class SolrIndexer {
/**
* The main method. Execution without argument displays help message.
- *
+ *
* @param args
* Command-line arguments.
*/
@@ -633,8 +651,8 @@ public class SolrIndexer {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("java " + SolrIndexer.class.getName(), options);
} else if (line.hasOption("all") || line.hasOption("product")
- || line.hasOption("metFile") || line.hasOption("read")
- || line.hasOption("types") || line.hasOption("deleteAll")) {
+ || line.hasOption("metFile") || line.hasOption("read")
+ || line.hasOption("types") || line.hasOption("deleteAll")) {
SolrIndexer indexer;
String solrUrl = null;
String fmUrl = null;
@@ -652,7 +670,7 @@ public class SolrIndexer {
indexer.indexProduct(line.getOptionValue("product"));
} else if (line.hasOption("metFile")) {
indexer.indexMetFile(new File(line.getOptionValue("metFile")), line
- .hasOption("delete"));
+ .hasOption("delete"));
} else if (line.hasOption("read")) {
for (String productId : readProductIdsFromStdin()) {
indexer.indexProduct(productId);
@@ -671,14 +689,14 @@ public class SolrIndexer {
} catch (Exception e) {
LOG.severe("An error occurred indexing: " + e.getMessage());
LOG
- .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
+ .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
}
}
}
/**
* This method reads product identifiers from the standard input.
- *
+ *
* @return Returns a List of product identifiers.
*/
private static List<String> readProductIdsFromStdin() {
@@ -694,12 +712,12 @@ public class SolrIndexer {
}
} catch (IOException e) {
LOG.severe("Error reading product id: line: [" + line + "]: Message: "
- + e.getMessage());
+ + e.getMessage());
} finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
return productIds;
}
@@ -731,32 +749,32 @@ public class SolrIndexer {
String key = (String) objKey;
if (key.startsWith(PREFIX_CONFIG)) {
properties.put(key.substring(PREFIX_CONFIG.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_MET)) {
mapProperties.put(key.substring(PREFIX_MET.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_FORMAT)) {
formatProperties.put(key.substring(PREFIX_FORMAT.length()), props
- .getProperty(key));
+ .getProperty(key));
}
}
if (properties.getProperty(IGNORE_TYPES) != null) {
String[] values = properties.getProperty(IGNORE_TYPES).trim()
- .split(",");
- Collections.addAll(ignoreTypes, values);
+ .split(",");
+ Collections.addAll(ignoreTypes, values);
}
if (properties.getProperty(IGNORE_VALUES) != null) {
String[] values = properties.getProperty(IGNORE_VALUES).trim().split(
- ",");
- Collections.addAll(ignoreValues, values);
+ ",");
+ Collections.addAll(ignoreValues, values);
}
if (properties.getProperty(REPLACEMENT_KEYS) != null) {
String[] values = properties.getProperty(REPLACEMENT_KEYS).trim()
- .split(",");
- Collections.addAll(replacementKeys, values);
+ .split(",");
+ Collections.addAll(replacementKeys, values);
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/profile/pom.xml
----------------------------------------------------------------------
diff --git a/profile/pom.xml b/profile/pom.xml
index 3d67496..68f8fa9 100644
--- a/profile/pom.xml
+++ b/profile/pom.xml
@@ -62,9 +62,8 @@
<version>1.3</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0-alpha1</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/protocol/http/pom.xml
----------------------------------------------------------------------
diff --git a/protocol/http/pom.xml b/protocol/http/pom.xml
index 4aaf834..19b5685 100644
--- a/protocol/http/pom.xml
+++ b/protocol/http/pom.xml
@@ -41,8 +41,8 @@
-->
<dependencies>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/resource/pom.xml
----------------------------------------------------------------------
diff --git a/resource/pom.xml b/resource/pom.xml
index 6ccb724..3e1bc8e 100644
--- a/resource/pom.xml
+++ b/resource/pom.xml
@@ -73,8 +73,8 @@ the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/sso/pom.xml
----------------------------------------------------------------------
diff --git a/sso/pom.xml b/sso/pom.xml
index cdb59a2..82979ec 100644
--- a/sso/pom.xml
+++ b/sso/pom.xml
@@ -53,9 +53,8 @@ the License.
<version>1.3</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
----------------------------------------------------------------------
diff --git a/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java b/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
index 5d77083..d0d399f 100755
--- a/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
+++ b/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
@@ -18,30 +18,37 @@
package org.apache.oodt.security.sso.opensso;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.PostMethod;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- *
+ *
* A client class to the services provided by the <a
* href="https://opensso.dev.java.net/">OpenSSO</a> project. The descriptions of
* these services are <a
* href="http://developers.sun.com/identity/reference/techart/id-svcs.html"
* >here</a>.
- *
+ *
* @author mattmann
* @version $Revision$
- *
+ *
*/
public class SSOProxy implements SSOMetKeys {
@@ -54,53 +61,63 @@ public class SSOProxy implements SSOMetKeys {
private static final String IDENT_ATTR_ENDPOINT_KEY = "IDENT_ATTR_ENDPOINT";
private static final String LOG_ENDPOINT;
private static final String LOG_ENDPOINT_KEY = "LOG_ENDPOINT";
-
+
static {
- if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
- AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
- } else {
- AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
- }
- if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
- IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
- } else {
- IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
- }
- if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
- IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
- } else {
- IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
- }
- if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
- LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
- } else {
- LOG_ENDPOINT = LOGOUT_ENDPOINT;
- }
-
- LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
- LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
- LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
- LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
+ if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
+ AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
+ } else {
+ AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
+ }
+ if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
+ IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
+ } else {
+ IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
+ }
+ if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
+ IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
+ } else {
+ IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
+ }
+ if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
+ LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
+ } else {
+ LOG_ENDPOINT = LOGOUT_ENDPOINT;
+ }
+
+ LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
+ LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
+ LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
+ LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
}
public String authenticate(String username, String password) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(AUTH_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(AUTH_ENDPOINT);
+ //PostMethod post = new PostMethod(AUTH_ENDPOINT);
String response;
String ssoToken = null;
- NameValuePair[] data = { new NameValuePair("username", username),
- new NameValuePair("password", password),
- new NameValuePair("uri", "realm/lmmp") };
+ NameValuePair[] data = { new BasicNameValuePair("username", username),
+ new BasicNameValuePair("password", password),
+ new BasicNameValuePair("uri", "realm/lmmp") };
- post.setRequestBody(data);
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new HttpException(response1.getStatusLine().toString());
}
- response = post.getResponseBodyAsString().trim();
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ response = handler.handleResponse(response1);
ssoToken = response.substring(9);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
@@ -112,56 +129,85 @@ public class SSOProxy implements SSOMetKeys {
}
public IdentityDetails readIdentity(String username, String token)
- throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_READ_ENDPOINT);
+ throws IOException, SingleSignOnException {
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
LOG.log(Level.INFO, "Obtaining identity: username: [" + username
- + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
- + "]");
- NameValuePair[] data = { new NameValuePair("name", username),
- new NameValuePair("admin", token) };
+ + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
+ + "]");
+ NameValuePair[] data = { new BasicNameValuePair("name", username),
+ new BasicNameValuePair("admin", token) };
- post.setRequestBody(data);
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new SingleSignOnException(response1.getStatusLine().toString());
}
- return parseIdentityDetails(post.getResponseBodyAsString().trim());
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+
+ return parseIdentityDetails(handler.handleResponse(response1).trim());
}
public UserDetails getUserAttributes(String token) throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_ATTR_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
LOG.log(Level.INFO, "Obtaining user attributes: token: [" + token
- + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
+ + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
+ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
+
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
- post.setRequestBody(data);
+ post.setEntity(entity);
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new SingleSignOnException(response1.getStatusLine().toString());
}
- return parseUserDetails(post.getResponseBodyAsString().trim());
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+
+ return parseUserDetails(handler.handleResponse(response1).trim());
}
public void logout(String token) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(LOG_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(LOG_ENDPOINT);
LOG.log(Level.INFO, "Logging out: token: [" + token + "]: REST url: ["
- + LOG_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
- post.setRequestBody(data);
+ + LOG_ENDPOINT + "]");
+ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
+
+
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new HttpException(response1.getStatusLine().toString());
}
} catch (HttpException e) {
// TODO Auto-generated catch block
@@ -176,7 +222,7 @@ public class SSOProxy implements SSOMetKeys {
private IdentityDetails parseIdentityDetails(String serviceResponse) {
ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
+ .getBytes());
BufferedReader br = new BufferedReader(new InputStreamReader(is));
IdentityDetails details = new IdentityDetails();
String line = null, lastAttrKeyRead = null;
@@ -214,7 +260,7 @@ public class SSOProxy implements SSOMetKeys {
} catch (IOException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
+ + "]: Message: " + e.getMessage());
} finally {
try {
is.close();
@@ -233,7 +279,7 @@ public class SSOProxy implements SSOMetKeys {
private UserDetails parseUserDetails(String serviceResponse) {
ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
+ .getBytes());
BufferedReader br = new BufferedReader(new InputStreamReader(is));
UserDetails details = new UserDetails();
String line = null, lastAttrKeyRead = null;
@@ -264,7 +310,7 @@ public class SSOProxy implements SSOMetKeys {
} catch (IOException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
+ + "]: Message: " + e.getMessage());
} finally {
try {
is.close();
@@ -283,8 +329,8 @@ public class SSOProxy implements SSOMetKeys {
public static void main(String[] args) throws IOException, SingleSignOnException {
String usage = "SSOProxy <cmd> [args]\n\n" + "Where cmd is one of:\n"
- + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
- + "attributes <token>\nlogout <token>\n";
+ + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
+ + "attributes <token>\nlogout <token>\n";
if (args.length < 2 || args.length > 3) {
System.err.println(usage);
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/workflow/pom.xml
----------------------------------------------------------------------
diff --git a/workflow/pom.xml b/workflow/pom.xml
index 72fb549..1fe5189 100644
--- a/workflow/pom.xml
+++ b/workflow/pom.xml
@@ -76,8 +76,8 @@ the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
[5/5] oodt git commit: fix lucene querying
Posted by ma...@apache.org.
fix lucene querying
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ceadfcb2
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ceadfcb2
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ceadfcb2
Branch: refs/heads/development
Commit: ceadfcb2b015956d915d6e9c58bd1b8ed7da4655
Parents: ab366e2
Author: Tom Barber <to...@analytical-labs.com>
Authored: Wed Jul 26 13:16:31 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Wed Jul 26 13:16:31 2017 +0100
----------------------------------------------------------------------
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 29 +++++++++++++++-----
.../oodt/cas/filemgr/tools/QueryTool.java | 1 +
2 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ceadfcb2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
index bfe2384..53efeb9 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
@@ -19,17 +19,23 @@ package org.apache.oodt.cas.filemgr.tools;
//Lucene imports
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.custom.CustomAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
//JDK imports
import java.io.Reader;
import java.util.Set;
+import org.apache.lucene.util.AttributeFactory;
/**
@@ -44,6 +50,7 @@ import java.util.Set;
*/
public class CASAnalyzer extends Analyzer {
private Set stopSet;
+ AttributeFactory factory = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
/**
* An array containing some common English words that are usually not useful
@@ -60,17 +67,19 @@ public class CASAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
TokenStream result = new WhitespaceTokenizer(/*reader*/);
- result = new StandardFilter(result);
+ /*result = new StandardFilter(result);
result = new StopFilter(result, STOP_WORDS);
-
- //TODO FIX
try {
- throw new Exception("needs fixing");
- } catch (Exception e) {
+ result.reset();
+ } catch (IOException e) {
e.printStackTrace();
}
- return null; //new TokenStreamComponents();
+ StandardTokenizer tokenizer = new StandardTokenizer(factory);
+
+ return new TokenStreamComponents(tokenizer, result);*/
+ return new TokenStreamComponents(new WhitespaceTokenizer());
+
}
public void tokenStreams(String fname, Reader reader){
@@ -78,7 +87,13 @@ public class CASAnalyzer extends Analyzer {
}
/** Builds an analyzer with the given stop words. */
public CASAnalyzer(CharArraySet stopWords) {
- stopSet = StopFilter.makeStopSet(stopWords.toArray(new String[stopWords.size()]));
+ Iterator iter = stopWords.iterator();
+ List<String> sw = new ArrayList<>();
+ while(iter.hasNext()) {
+ char[] stopWord = (char[]) iter.next();
+ sw.add(new String(stopWord));
+ }
+ stopSet = StopFilter.makeStopSet(sw);
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ceadfcb2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index 4a9641f..7b37f90 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -77,6 +77,7 @@ public final class QueryTool {
QueryParser parser;
// note that "__FREE__" is a control work for free text searching
parser = new QueryParser(freeTextBlock, new CASAnalyzer());
+
Query luceneQ = null;
try {
luceneQ = (Query) parser.parse(query);
[4/5] oodt git commit: fix merge
Posted by ma...@apache.org.
fix merge
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ab366e2a
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ab366e2a
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ab366e2a
Branch: refs/heads/development
Commit: ab366e2a648df7c7ac37aaa1c49d1d98a1aeb20c
Parents: 78ce45e 07d9030
Author: Tom Barber <to...@analytical-labs.com>
Authored: Wed Jul 26 13:16:15 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Wed Jul 26 13:16:15 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 21 +-
app/fmbrowser/pom.xml | 1 +
.../cas/filemgr/browser/model/QueryBuilder.java | 30 +-
app/weditor/.gitignore | 1 +
catalog/.gitignore | 1 +
catalog/pom.xml | 15 +
commons/.gitignore | 1 +
.../apache/oodt/commons/exec/EnvUtilities.java | 16 +-
.../oodt/commons/exec/TestEnvUtilities.java | 9 +
core/pom.xml | 50 ++-
crawler/.gitignore | 1 +
crawler/pom.xml | 14 +
.../cas/crawl/MetExtractorProductCrawler.java | 37 +-
.../oodt/cas/crawl/StdProductCrawler.java | 16 +-
.../cas/crawl/action/SolrIndexingAction.java | 18 +-
curator/webapp/pom.xml | 12 -
docker/Dockerfile | 6 +-
filemgr/pom.xml | 11 +-
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 430 +++++++++++--------
.../filemgr/catalog/LuceneCatalogFactory.java | 18 +-
.../cli/action/LuceneQueryCliAction.java | 33 +-
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 37 +-
.../oodt/cas/filemgr/tools/CatalogSearch.java | 26 +-
.../filemgr/tools/OptimizeLuceneCatalog.java | 24 +-
.../oodt/cas/filemgr/tools/QueryTool.java | 23 +-
.../cas/filemgr/tools/RangeQueryTester.java | 50 ++-
.../cas/filemgr/catalog/TestLuceneCatalog.java | 4 +-
metadata/.gitignore | 1 +
opendapps/pom.xml | 6 +-
.../apache/oodt/opendapps/DatasetExtractor.java | 5 +
pcs/input/.gitignore | 1 +
product/.gitignore | 1 +
protocol/api/.gitignore | 1 +
protocol/ftp/.gitignore | 1 +
protocol/http/.gitignore | 1 +
protocol/imaps/.gitignore | 1 +
protocol/sftp/.gitignore | 1 +
workflow/pom.xml | 4 +
.../LuceneWorkflowInstanceRepository.java | 320 ++++++++------
...LuceneWorkflowInstanceRepositoryFactory.java | 25 +-
.../workflow/tools/TestInstanceRepoCleaner.java | 2 +-
workflow/src/test/resources/testinstrepo/_4.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_4.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_4.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/_43.cfs | Bin 14148 -> 0 bytes
workflow/src/test/resources/testinstrepo/_5.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_5.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_5.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/deletable | Bin 4 -> 0 bytes
.../resources/testinstrepo/pending_segments_b | Bin 0 -> 198 bytes
.../src/test/resources/testinstrepo/segments | Bin 28 -> 0 bytes
.../src/test/resources/testinstrepo/segments_a | Bin 0 -> 136 bytes
.../src/test/resources/testinstrepo/write.lock | 0
xmlps/.gitignore | 1 +
xmlquery/.gitignore | 1 +
55 files changed, 825 insertions(+), 452 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/catalog/pom.xml
----------------------------------------------------------------------
diff --cc catalog/pom.xml
index 07805a3,fa13c3a..cc73c07
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@@ -60,9 -60,24 +60,24 @@@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/crawler/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/filemgr/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/workflow/pom.xml
----------------------------------------------------------------------
[3/5] oodt git commit: update some dependenceies
Posted by ma...@apache.org.
update some dependenceies
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/78ce45e2
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/78ce45e2
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/78ce45e2
Branch: refs/heads/development
Commit: 78ce45e2efad87835af9d29972e88cc32561c5d7
Parents: 29fc616
Author: Tom Barber <to...@analytical-labs.com>
Authored: Tue Dec 6 11:34:33 2016 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Tue Dec 6 11:34:33 2016 +0000
----------------------------------------------------------------------
catalog/pom.xml | 4 +-
core/pom.xml | 22 +-
crawler/pom.xml | 4 +-
filemgr/pom.xml | 4 +-
.../cas/filemgr/catalog/solr/SolrClient.java | 318 ++++++++-------
.../filemgr/system/CommonsXmlRpcTransport.java | 183 +++++++++
.../filemgr/system/XmlRpcFileManagerClient.java | 398 ++++++++++---------
.../oodt/cas/filemgr/tools/SolrIndexer.java | 236 ++++++-----
profile/pom.xml | 5 +-
protocol/http/pom.xml | 4 +-
resource/pom.xml | 4 +-
sso/pom.xml | 5 +-
.../oodt/security/sso/opensso/SSOProxy.java | 214 ++++++----
workflow/pom.xml | 4 +-
14 files changed, 842 insertions(+), 563 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index 77ffa77..07805a3 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -61,8 +61,8 @@
</exclusions>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 3420597..f1dea9e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -111,9 +111,15 @@ the License.
<version>1.4</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.1</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ <version>4.5.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -267,11 +273,6 @@ the License.
<version>1.1</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.2</version>
- </dependency>
- <dependency>
<groupId>org.apache.jena</groupId>
<artifactId>apache-jena-libs</artifactId>
<version>${jena.version}</version>
@@ -489,11 +490,6 @@ the License.
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j</artifactId>
- <version>1.7.12</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.12</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/crawler/pom.xml
----------------------------------------------------------------------
diff --git a/crawler/pom.xml b/crawler/pom.xml
index 5428b53..46378c8 100644
--- a/crawler/pom.xml
+++ b/crawler/pom.xml
@@ -56,8 +56,8 @@ the License.
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index cc9a6e5..745778f 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -91,8 +91,8 @@
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
index 1b17b62..475ff8b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
@@ -16,20 +16,35 @@
*/
package org.apache.oodt.cas.filemgr.catalog.solr;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.solr.client.solrj.util.ClientUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
import java.util.Map;
@@ -39,30 +54,30 @@ import java.util.logging.Logger;
/**
* Class containing client-side functionality for interacting with a Solr server.
* This class uses an {@link HttpClient} for all HTTP communication.
- *
+ *
* @author Luca Cinquini
*
*/
public class SolrClient {
-
+
// base URL of Solr server
private String solrUrl;
-
+
private final Logger LOG = Logger.getLogger(this.getClass().getName());
-
+
/**
* Constructor initializes the Solr URL
* @param url
*/
public SolrClient(final String url) {
-
+
solrUrl = url;
-
+
}
-
+
/**
* Method to send one or more documents to be indexed to Solr.
- *
+ *
* @param docs
* @param commit
* @param mimeType : the mime-type format of the documents
@@ -70,38 +85,38 @@ public class SolrClient {
* @throws CatalogException
*/
public String index(List<String> docs, boolean commit, String mimeType) throws CatalogException {
-
+
try {
-
+
final String url = this.buildUpdateUrl();
-
+
// build message
StringBuilder message = new StringBuilder("<add>");
for (String doc : docs) {
message.append(doc);
}
- message.append("</add>");
-
- // send POST request
- LOG.info("Posting message:"+message+" to URL:"+url);
- String response = doPost(url, message.toString(), mimeType);
- LOG.info(response);
-
- // commit changes ?
- if (commit) {
- this.commit();
- }
-
- LOG.info(response);
- return response;
-
+ message.append("</add>");
+
+ // send POST request
+ LOG.info("Posting message:"+message+" to URL:"+url);
+ String response = doPost(url, message.toString(), mimeType);
+ LOG.info(response);
+
+ // commit changes ?
+ if (commit) {
+ this.commit();
+ }
+
+ LOG.info(response);
+ return response;
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to send a message containing a 'delete' instruction to Solr.
* @param id
@@ -110,41 +125,41 @@ public class SolrClient {
* @throws CatalogException
*/
public String delete(String id, boolean commit) throws CatalogException {
-
+
try {
-
+
// build POST request
String url = this.buildUpdateUrl();
if (commit) {
- url += "?commit=true";
+ url += "?commit=true";
}
String message = "<delete><query>id:"+id+"</query></delete>";
-
- // send POST request
- LOG.info("Posting message:"+message+" to URL:"+url);
- return doPost(url, message, Parameters.MIME_TYPE_XML);
-
+ // send POST request
+ LOG.info("Posting message:"+message+" to URL:"+url);
+
+ return doPost(url, message, Parameters.MIME_TYPE_XML);
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to query the Solr index for a product with the specified id.
* @param id
* @return
*/
public String queryProductById(String id, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{Parameters.PRODUCT_ID+":"+id} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query the Solr index for a product with the specified name.
* @param name
@@ -152,13 +167,13 @@ public class SolrClient {
* @return
*/
public String queryProductByName(String name, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
- params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+name} );
+ params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+ ClientUtils.escapeQueryChars(name)} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query Solr for the most recent 'n' products.
* @param n
@@ -166,15 +181,15 @@ public class SolrClient {
* @throws CatalogException
*/
public String queryProductsByDate(int n, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{ "*:*"} );
params.put("rows", new String[]{ ""+n} );
params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query Solr for the most recent 'n' products of a specified type.
* @param n
@@ -182,51 +197,51 @@ public class SolrClient {
* @throws CatalogException
*/
public String queryProductsByDateAndType(int n, ProductType type, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{ Parameters.PRODUCT_TYPE_NAME+type.getName() } );
params.put("rows", new String[]{ ""+n} );
params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to commit the current changes to the Solr index.
* @throws Exception
*/
public void commit() throws IOException, CatalogException {
-
+
String message = "<commit waitSearcher=\"true\"/>";
String url = this.buildUpdateUrl();
doPost(url, message, Parameters.MIME_TYPE_XML);
-
+
}
-
+
/**
* Method to send a generic query to the Solr server.
- *
+ *
* @param parameters
* @param mimeType : the desired mime type for the results (XML, JSON, ...)
* @return
*/
public String query(Map<String, String[]> parameters, String mimeType) throws CatalogException {
-
+
try {
-
+
// build HTTP request
String url = this.buildSelectUrl();
-
+
// execute request
- return this.doGet(url, parameters, mimeType);
-
+ return this.doGet(url, parameters, mimeType);
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to execute a GET request to the given URL with the given parameters.
* @param url
@@ -234,28 +249,32 @@ public class SolrClient {
* @return
*/
private String doGet(String url, Map<String, String[]> parameters, String mimeType)
- throws IOException, CatalogException {
-
+ throws IOException, CatalogException {
+
// build HTTP/GET request
- GetMethod method = new GetMethod(url);
- List<NameValuePair> nvps = new ArrayList<NameValuePair>();
- for (Map.Entry<String, String[]> key : parameters.entrySet()) {
- for (String value : key.getValue()) {
- nvps.add(new NameValuePair(key.getKey(), value));
- }
- }
- // request results in JSON format
- if (mimeType.equals(Parameters.MIME_TYPE_JSON)) {
- nvps.add(new NameValuePair("wt", "json"));
+
+
+ List<BasicNameValuePair> nvps = new ArrayList<BasicNameValuePair>();
+ for (Map.Entry<String, String[]> key : parameters.entrySet()) {
+ for (String value : key.getValue()) {
+ nvps.add(new BasicNameValuePair(key.getKey(), value));
+ }
+ }
+ // request results in JSON format
+ if (mimeType.equals(Parameters.MIME_TYPE_JSON)) {
+ nvps.add(new BasicNameValuePair("wt", "json"));
+ }
+
+ String paramString = URLEncodedUtils.format(nvps, "utf-8");
+
+ HttpRequestBase method = new HttpGet(url+"?"+paramString);
+ LOG.info("GET url: "+url+" query string: "+method.getURI());
+
+ // send HTTP/GET request, return response
+ return doHttp(method);
+
}
- method.setQueryString( nvps.toArray( new NameValuePair[nvps.size()] ) );
- LOG.info("GET url: "+url+" query string: "+method.getQueryString());
-
- // send HTTP/GET request, return response
- return doHttp(method);
-
- }
-
+
/**
* Method to execute a POST request to the given URL.
* @param url
@@ -263,91 +282,96 @@ public class SolrClient {
* @return
*/
private String doPost(String url, String document, String mimeType) throws IOException, CatalogException {
-
+
// build HTTP/POST request
- PostMethod method = new PostMethod(url);
- StringRequestEntity requestEntity = new StringRequestEntity(document, mimeType, "UTF-8");
- method.setRequestEntity(requestEntity);
-
- // send HTTP/POST request, return response
- return doHttp(method);
-
+ HttpPost method = new HttpPost(url);
+ HttpEntity requestEntity = null;
+ try {
+ requestEntity = new StringEntity(document, mimeType, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ method.setEntity(requestEntity);
+ // send HTTP/POST request, return response
+ return doHttp(method);
+
}
-
+
/**
* Common functionality for HTTP GET and POST requests.
* @param method
* @return
* @throws Exception
*/
- private String doHttp(HttpMethod method) throws IOException, CatalogException {
-
- StringBuilder response = new StringBuilder();
+ private String doHttp(HttpRequestBase method) throws IOException, CatalogException {
+
+ String response = null;
BufferedReader br = null;
try {
-
- // send request
- HttpClient httpClient = new HttpClient();
- // OODT-719 Prevent httpclient from spawning closewait tcp connections
- method.setRequestHeader("Connection", "close");
-
- int statusCode = httpClient.executeMethod(method);
-
- // read response
- if (statusCode != HttpStatus.SC_OK) {
-
- // still consume the response
- method.getResponseBodyAsString();
- throw new CatalogException("HTTP method failed: " + method.getStatusLine());
-
- } else {
-
- // read the response body.
- br = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
- String readLine;
- while(((readLine = br.readLine()) != null)) {
- response.append(readLine);
- }
-
- }
-
- } finally {
- // must release the connection even if an exception occurred
- method.releaseConnection();
- if (br!=null) {
- try {
- br.close();
- } catch (Exception ignored) {
- }
+
+ // send request
+ HttpClient httpClient = new DefaultHttpClient();
+ // OODT-719 Prevent httpclient from spawning closewait tcp connections
+ method.setHeader("Connection", "close");
+
+ HttpResponse statusCode = httpClient.execute(method);
+
+ // read response
+ if (statusCode.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+
+ // still consume the response
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ handler.handleResponse(statusCode);
+ throw new CatalogException("HTTP method failed: " + statusCode.getStatusLine().toString());
+
+ } else {
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ String resp = handler.handleResponse(statusCode);
+
+ response=resp;
+
+ }
+
+ } finally {
+ // must release the connection even if an exception occurred
+ method.releaseConnection();
+ if (br!=null) {
+ try {
+ br.close();
+ } catch (Exception ignored) {
+ }
+ }
}
- }
-
- return response.toString();
-
+
+ return response;
+
}
-
+
/**
* Builds the URL used to update the Solr index.
- *
+ *
* Example: http://localhost:8983/solr/update?
* @return
*/
private String buildUpdateUrl() {
-
+
return solrUrl + (solrUrl.endsWith("/") ? "" : "/") + "update";
-
+
}
-
+
/**
* Builds the URL used to query the Solr index.
- *
+ *
* Example: http://localhost:8983/solr/select?
* @return
*/
private String buildSelectUrl() {
-
+
return solrUrl + (solrUrl.endsWith("/") ? "" : "/") + "select";
-
+
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
new file mode 100644
index 0000000..baf9e73
--- /dev/null
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
@@ -0,0 +1,183 @@
+//
+// Source code recreated from a .class file by IntelliJ IDEA
+// (powered by Fernflower decompiler)
+//
+
+package org.apache.oodt.cas.filemgr.system;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.*;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+
+import org.apache.http.impl.auth.BasicSchemeFactory;
+import org.apache.http.impl.auth.DigestSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.apache.xmlrpc.XmlRpcClientException;
+import org.apache.xmlrpc.XmlRpcTransport;
+
+public class CommonsXmlRpcTransport implements XmlRpcTransport {
+ private URL url;
+ private HttpClient client;
+ private Header userAgentHeader;
+ private boolean http11;
+ private boolean gzip;
+ private boolean rgzip;
+ private Credentials creds;
+ protected HttpPost method;
+ private int timeout = 10;
+ private int connecttimeout = 10;
+ private String password;
+ private String user;
+ private String auth;
+
+ public CommonsXmlRpcTransport(URL url, HttpClient client) {
+ this.userAgentHeader = new BasicHeader("User-Agent", "Apache XML-RPC 2.0");
+ this.http11 = false;
+ this.gzip = false;
+ this.rgzip = false;
+ this.url = url;
+ if(client == null) {
+ RequestConfig config = RequestConfig.custom()
+ .setSocketTimeout(timeout * 1000)
+ .setConnectTimeout(connecttimeout * 1000)
+ .build();
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ if(!user.equals(null)) {
+ credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user));
+ }
+ else if(!user.equals(null)&& !password.equals(null)){
+ credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));
+ }
+ else if(!auth.equals(null)){
+
+ }
+
+ System.out.println("building empty registry");
+ Registry<AuthSchemeProvider> r = RegistryBuilder.<AuthSchemeProvider>create().build();
+ HttpClient newClient = HttpClients.custom().setDefaultAuthSchemeRegistry(r).setDefaultCredentialsProvider(credsProvider).setDefaultRequestConfig(config).build();
+ this.client = newClient;
+ } else {
+ this.client = client;
+ }
+
+ }
+
+ public CommonsXmlRpcTransport(URL url) {
+ this(url, (HttpClient)null);
+ }
+
+
+
+ public InputStream sendXmlRpc(byte[] request) throws IOException, XmlRpcClientException {
+ this.method = new HttpPost(this.url.toString());
+ //this.method.setHttp11(this.http11);
+ this.method.setHeader(new BasicHeader("Content-Type", "text/xml"));
+ if(this.rgzip) {
+ this.method.setHeader(new BasicHeader("Content-Encoding", "gzip"));
+ }
+
+ if(this.gzip) {
+ this.method.setHeader(new BasicHeader("Accept-Encoding", "gzip"));
+ }
+
+ this.method.setHeader(this.userAgentHeader);
+ if(this.rgzip) {
+ ByteArrayOutputStream hostURI = new ByteArrayOutputStream();
+ GZIPOutputStream hostConfig = new GZIPOutputStream(hostURI);
+ hostConfig.write(request);
+ hostConfig.finish();
+ hostConfig.close();
+ byte[] lgzipo = hostURI.toByteArray();
+
+ HttpEntity entity = new ByteArrayEntity(lgzipo);
+
+
+ this.method.setEntity(entity);
+
+ //this.method.setRequestContentLength(-1);
+ } else {
+ HttpEntity entity = new ByteArrayEntity(request);
+ this.method.setEntity(entity);
+// this.method.setRequestBody(new ByteArrayInputStream(request));
+ }
+
+ URI hostURI1 = null;
+ try {
+ hostURI1 = new URI(this.url.toString());
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ HttpHost hostConfig1 = new HttpHost(hostURI1.toString());
+ HttpResponse response = this.client.execute(this.method);
+ boolean lgzipo1 = false;
+ Header[] lHeader = response.getHeaders("Content-Encoding");
+ if(lHeader != null && lHeader.length>0) {
+ String lValue = lHeader[0].getValue();
+ if(lValue != null) {
+ lgzipo1 = lValue.indexOf("gzip") >= 0;
+ }
+ }
+
+ return (InputStream)(lgzipo1?new GZIPInputStream(response.getEntity().getContent()):response.getEntity().getContent());
+ }
+
+ public void setHttp11(boolean http11) {
+ this.http11 = http11;
+ }
+
+ public void setGzip(boolean gzip) {
+ this.gzip = gzip;
+ }
+
+ public void setRGzip(boolean gzip) {
+ this.rgzip = gzip;
+ }
+
+ public void setUserAgent(String userAgent) {
+ this.userAgentHeader =new BasicHeader("User-Agent", userAgent);
+ }
+
+ public void setTimeout(int timeout) {
+ this.timeout = timeout;
+ }
+
+ public void setConnectionTimeout(int ctimeout) {
+ this.connecttimeout = ctimeout;
+ }
+
+ public void setBasicAuthentication(String user, String password) {
+ this.user = user;
+ this.password = password;
+
+ }
+
+ public void setBasicAuthentication(String auth) {
+ this.auth = auth;
+ }
+
+ public void endClientRequest() throws XmlRpcClientException {
+ this.method.releaseConnection();
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
index db64716..9c3a5e4 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
@@ -17,10 +17,14 @@
package org.apache.oodt.cas.filemgr.system;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
import org.apache.oodt.cas.cli.CmdLineUtility;
import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
import org.apache.oodt.cas.filemgr.exceptions.FileManagerException;
@@ -43,7 +47,6 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.filemgr.util.XmlRpcStructFactory;
import org.apache.oodt.cas.filemgr.versioning.Versioner;
import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.xmlrpc.CommonsXmlRpcTransport;
import org.apache.xmlrpc.XmlRpcClient;
import org.apache.xmlrpc.XmlRpcClientException;
import org.apache.xmlrpc.XmlRpcException;
@@ -76,7 +79,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
/* our log stream */
private static Logger LOG = Logger.getLogger(XmlRpcFileManagerClient.class
- .getName());
+ .getName());
/* file manager url */
private URL fileManagerUrl = null;
@@ -95,21 +98,21 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
* @param testConnection Whether or not to check if server at given url is alive.
*/
public XmlRpcFileManagerClient(final URL url, boolean testConnection)
- throws ConnectionException {
+ throws ConnectionException {
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
String configFile = System
- .getProperty("org.apache.oodt.cas.filemgr.properties");
+ .getProperty("org.apache.oodt.cas.filemgr.properties");
LOG.log(Level.INFO,
- "Loading File Manager Configuration Properties from: ["
- + configFile + "]");
+ "Loading File Manager Configuration Properties from: ["
+ + configFile + "]");
try {
System.getProperties().load(
- new FileInputStream(new File(configFile)));
+ new FileInputStream(new File(configFile)));
} catch (Exception e) {
LOG.log(Level.INFO,
- "Error loading configuration properties from: ["
- + configFile + "]");
+ "Error loading configuration properties from: ["
+ + configFile + "]");
}
}
@@ -117,41 +120,52 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
XmlRpcTransportFactory transportFactory = new XmlRpcTransportFactory() {
public XmlRpcTransport createTransport()
- throws XmlRpcClientException {
- HttpClient client = new HttpClient();
- client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
- new HttpMethodRetryHandler() {
-
- public boolean retryMethod(HttpMethod method,
- IOException e, int count) {
- if (count < Integer
+ throws XmlRpcClientException {
+
+ HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
+ public boolean retryRequest(
+ IOException exception,
+ int count,
+ HttpContext context){
+ if (count < Integer
.getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries",
- 3)) {
- try {
- Thread
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries",
+ 3)) {
+ try {
+ Thread
.sleep(Integer
- .getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds",
- 0) * 1000);
- return true;
- } catch (Exception ignored) {
- }
- }
- return false;
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds",
+ 0) * 1000);
+ return true;
+ } catch (Exception ignored) {
}
+ }
+ return false;
+ }
+ };
+ RequestConfig config = RequestConfig.custom()
+ .setSocketTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
+ 20) * 60 * 1000)
+ .setConnectTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes",
+ 60) * 60 * 1000)
+ .build();
+ Registry<AuthSchemeProvider> r = RegistryBuilder.<AuthSchemeProvider>create().build();
+ HttpClient client = HttpClients.custom().setRetryHandler(myRetryHandler).setDefaultAuthSchemeRegistry(r).setDefaultRequestConfig(config).build();
- });
- CommonsXmlRpcTransport transport = new CommonsXmlRpcTransport(
- url, client);
+ CommonsXmlRpcTransport transport = new CommonsXmlRpcTransport(url, client);
transport
- .setConnectionTimeout(Integer
- .getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
- 20) * 60 * 1000);
+ .setConnectionTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
+ 20) * 60 * 1000);
transport
- .setTimeout(Integer
- .getInteger(
+ .setTimeout(Integer
+ .getInteger(
"org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes",
60) * 60 * 1000);
@@ -168,7 +182,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (testConnection && !isAlive()) {
throw new ConnectionException("Exception connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
}
}
@@ -179,14 +193,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
Vector<Object> argList = new Vector<Object>();
try {
success = (Boolean) client.execute("filemgr.refreshConfigAndPolicy",
- argList);
+ argList);
} catch (XmlRpcException e) {
LOG.log(Level.WARNING, "XmlRpcException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
success = false;
} catch (IOException e) {
LOG.log(Level.WARNING, "IOException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
success = false;
}
@@ -201,12 +215,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
connected = (Boolean) client.execute("filemgr.isAlive", argList);
} catch (XmlRpcException e) {
LOG.log(Level.WARNING,
- "XmlRpcException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ "XmlRpcException when connecting to filemgr: ["
+ + this.fileManagerUrl + "]");
connected = false;
} catch (IOException e) {
LOG.log(Level.WARNING, "IOException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
connected = false;
}
@@ -214,17 +228,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean transferringProduct(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
boolean success;
try {
success = (Boolean) client.execute("filemgr.transferringProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -235,9 +249,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean removeProductTransferStatus(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -245,7 +259,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute(
- "filemgr.removeProductTransferStatus", argList);
+ "filemgr.removeProductTransferStatus", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -256,9 +270,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean isTransferComplete(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -266,7 +280,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.isTransferComplete",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -277,9 +291,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean moveProduct(Product product, String newPath)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
argList.add(newPath);
@@ -299,7 +313,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public boolean modifyProduct(Product product) throws CatalogException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -308,7 +322,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.modifyProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -321,7 +335,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public boolean removeProduct(Product product) throws CatalogException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -330,7 +344,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.removeProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -343,7 +357,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public FileTransferStatus getCurrentFileTransfer()
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> statusHash;
@@ -351,7 +365,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
statusHash = (Map<String, Object>) client.execute(
- "filemgr.getCurrentFileTransfer", argList);
+ "filemgr.getCurrentFileTransfer", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -360,7 +374,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (statusHash != null) {
status = XmlRpcStructFactory
- .getFileTransferStatusFromXmlRpc(statusHash);
+ .getFileTransferStatusFromXmlRpc(statusHash);
}
return status;
@@ -368,7 +382,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<FileTransferStatus> getCurrentFileTransfers()
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> statusVector;
@@ -376,7 +390,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
statusVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getCurrentFileTransfers", argList);
+ "filemgr.getCurrentFileTransfers", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -385,23 +399,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (statusVector != null) {
statuses = XmlRpcStructFactory
- .getFileTransferStatusesFromXmlRpc(statusVector);
+ .getFileTransferStatusesFromXmlRpc(statusVector);
}
return statuses;
}
public double getProductPctTransferred(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
Double pct;
try {
pct = (Double) client.execute("filemgr.getProductPctTransferred",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -416,16 +430,16 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public double getRefPctTransferred(Reference reference)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> refHash = XmlRpcStructFactory
- .getXmlRpcReference(reference);
+ .getXmlRpcReference(reference);
argList.add(refHash);
Double pct;
try {
pct = (Double) client.execute("filemgr.getRefPctTransferred",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -441,12 +455,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage pagedQuery(Query query, ProductType type, int pageNum)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> queryHash = XmlRpcStructFactory
- .getXmlRpcQuery(query);
+ .getXmlRpcQuery(query);
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(queryHash);
argList.add(typeHash);
argList.add(pageNum);
@@ -455,7 +469,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.pagedQuery", argList);
+ "filemgr.pagedQuery", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -475,7 +489,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getFirstPage", argList);
+ "filemgr.getFirstPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -500,7 +514,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getLastPage", argList);
+ "filemgr.getLastPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -516,7 +530,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage getNextPage(ProductType type, ProductPage currPage)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
argList.add(XmlRpcStructFactory.getXmlRpcProductPage(currPage));
@@ -526,7 +540,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getNextPage", argList);
+ "filemgr.getNextPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -542,7 +556,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage getPrevPage(ProductType type, ProductPage currPage)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
argList.add(XmlRpcStructFactory.getXmlRpcProductPage(currPage));
@@ -552,7 +566,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getPrevPage", argList);
+ "filemgr.getPrevPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -567,16 +581,16 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public String addProductType(ProductType type)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
String productTypeId;
Vector<Object> argList = new Vector<Object>();
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(typeHash);
try {
productTypeId = (String) client.execute("filemgr.addProductType",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -595,7 +609,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
hasProduct = (Boolean) client.execute("filemgr.hasProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -614,7 +628,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
numProducts = (Integer) client.execute("filemgr.getNumProducts",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -633,7 +647,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
topNProducts = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getTopNProducts", argList);
+ "filemgr.getTopNProducts", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -641,23 +655,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
return XmlRpcStructFactory
- .getProductListFromXmlRpc(topNProducts);
+ .getProductListFromXmlRpc(topNProducts);
}
@SuppressWarnings("unchecked")
public List<Product> getTopNProducts(int n, ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(n);
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
Vector<Map<String, Object>> topNProducts;
try {
topNProducts = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getTopNProducts", argList);
+ "filemgr.getTopNProducts", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -665,14 +679,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
return XmlRpcStructFactory
- .getProductListFromXmlRpc(topNProducts);
+ .getProductListFromXmlRpc(topNProducts);
}
public void setProductTransferStatus(Product product)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
@@ -688,7 +702,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public void addProductReferences(Product product) throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
@@ -701,7 +715,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public void addMetadata(Product product, Metadata metadata)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProduct(product));
@@ -717,7 +731,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean updateMetadata(Product product, Metadata met)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProduct(product));
argList.add(met.getHashTable());
@@ -744,7 +758,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productId = (String) client.execute("filemgr.catalogProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -758,14 +772,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public Metadata getMetadata(Product product) throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
Map<String, Object> metadata;
try {
metadata = (Map<String, Object>) client.execute(
- "filemgr.getMetadata", argList);
+ "filemgr.getMetadata", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -780,10 +794,10 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings({ "unchecked", "rawtypes" })
public Metadata getReducedMetadata(Product product, List<?> elements)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
argList.add(new Vector(elements));
@@ -791,7 +805,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
metadata = (Map<String, Object>) client.execute(
- "filemgr.getReducedMetadata", argList);
+ "filemgr.getReducedMetadata", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -822,7 +836,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public byte[] retrieveFile(String filePath, int offset, int numBytes)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
argList.add(filePath);
argList.add(offset);
@@ -856,17 +870,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Product> getProductsByProductType(ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
Vector<Map<String, Object>> productVector;
try {
productVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getProductsByProductType", argList);
+ "filemgr.getProductsByProductType", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -882,10 +896,10 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Element> getElementsByProductType(ProductType type)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
@@ -893,7 +907,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getElementsByProductType", argList);
+ "filemgr.getElementsByProductType", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -909,7 +923,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementById(String elementId)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementId);
@@ -917,7 +931,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementById", argList);
+ "filemgr.getElementById", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -933,7 +947,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementByName(String elementName)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementName);
@@ -941,7 +955,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementByName", argList);
+ "filemgr.getElementByName", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -957,7 +971,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementByName(String elementName, ProductType type)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementName);
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
@@ -966,7 +980,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementByName", argList);
+ "filemgr.getElementByName", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -981,17 +995,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public List<QueryResult> complexQuery(ComplexQuery complexQuery)
- throws CatalogException {
+ throws CatalogException {
try {
Map<String, Object> complexQueryHash = XmlRpcStructFactory
- .getXmlRpcComplexQuery(complexQuery);
+ .getXmlRpcComplexQuery(complexQuery);
Vector<Object> argList = new Vector<Object>();
argList.add(complexQueryHash);
@SuppressWarnings("unchecked")
Vector<Map<String, Object>> queryResultHashVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.complexQuery", argList);
+ .execute("filemgr.complexQuery", argList);
return XmlRpcStructFactory
- .getQueryResultsFromXmlRpc(queryResultHashVector);
+ .getQueryResultsFromXmlRpc(queryResultHashVector);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e);
@@ -1000,12 +1014,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Product> query(Query query, ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> queryHash = XmlRpcStructFactory
- .getXmlRpcQuery(query);
+ .getXmlRpcQuery(query);
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(queryHash);
argList.add(typeHash);
@@ -1013,7 +1027,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.query", argList);
+ "filemgr.query", argList);
} catch (XmlRpcException e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e);
@@ -1032,14 +1046,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductType getProductTypeByName(String productTypeName)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Hashtable<String, Object> productTypeHash;
Vector<Object> argList = new Vector<Object>();
argList.add(productTypeName);
try {
productTypeHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getProductTypeByName", argList);
+ "filemgr.getProductTypeByName", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e.getLocalizedMessage());
} catch (IOException e) {
@@ -1050,20 +1064,20 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeFromXmlRpc(productTypeHash);
+ .getProductTypeFromXmlRpc(productTypeHash);
}
}
@SuppressWarnings("unchecked")
public ProductType getProductTypeById(String productTypeId)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Hashtable<String, Object> productTypeHash;
Vector<Object> argList = new Vector<Object>();
argList.add(productTypeId);
try {
productTypeHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getProductTypeById", argList);
+ "filemgr.getProductTypeById", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -1074,20 +1088,20 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeFromXmlRpc(productTypeHash);
+ .getProductTypeFromXmlRpc(productTypeHash);
}
}
@SuppressWarnings("unchecked")
public List<ProductType> getProductTypes()
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> productTypeVector;
try {
productTypeVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.getProductTypes", argList);
+ .execute("filemgr.getProductTypes", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -1098,23 +1112,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeListFromXmlRpc(productTypeVector);
+ .getProductTypeListFromXmlRpc(productTypeVector);
}
}
@SuppressWarnings("unchecked")
public List<Reference> getProductReferences(Product product)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> productReferenceVector;
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
productReferenceVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.getProductReferences", argList);
+ .execute("filemgr.getProductReferences", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1125,7 +1139,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getReferencesFromXmlRpc(productReferenceVector);
+ .getReferencesFromXmlRpc(productReferenceVector);
}
}
@@ -1138,7 +1152,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productHash = (Map<String, Object>) client.execute(
- "filemgr.getProductById", argList);
+ "filemgr.getProductById", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1161,7 +1175,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productHash = (Map<String, Object>) client.execute(
- "filemgr.getProductByName", argList);
+ "filemgr.getProductByName", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1177,38 +1191,38 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public String ingestProduct(Product product, Metadata metadata,
boolean clientTransfer)
- throws VersioningException, XmlRpcException, FileManagerException {
+ throws VersioningException, XmlRpcException, FileManagerException {
try {
// ingest product
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
argList.add(metadata.getHashTable());
argList.add(clientTransfer);
String productId = (String) client.execute("filemgr.ingestProduct",
- argList);
+ argList);
if (clientTransfer) {
LOG.log(Level.FINEST,
- "File Manager Client: clientTransfer enabled: "
- + "transfering product ["
- + product.getProductName() + "]");
+ "File Manager Client: clientTransfer enabled: "
+ + "transfering product ["
+ + product.getProductName() + "]");
// we need to transfer the product ourselves
// make sure we have the product ID
if (productId == null) {
throw new Exception("Request to ingest product: "
- + product.getProductName()
- + " but no product ID returned from File "
- + "Manager ingest");
+ + product.getProductName()
+ + " but no product ID returned from File "
+ + "Manager ingest");
}
if (dataTransfer == null) {
throw new Exception("Request to ingest product: ["
- + product.getProductName()
- + "] using client transfer, but no "
- + "dataTransferer specified!");
+ + product.getProductName()
+ + "] using client transfer, but no "
+ + "dataTransferer specified!");
}
product.setProductId(productId);
@@ -1216,8 +1230,8 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (!Boolean.getBoolean("org.apache.oodt.cas.filemgr.serverside.versioning")) {
// version the product
Versioner versioner = GenericFileManagerObjectFactory
- .getVersionerFromClassName(product.getProductType()
- .getVersioner());
+ .getVersionerFromClassName(product.getProductType()
+ .getVersioner());
if (versioner != null) {
versioner.createDataStoreReferences(product, metadata);
}
@@ -1227,13 +1241,13 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
addProductReferences(product);
} catch (CatalogException e) {
LOG
- .log(
- Level.SEVERE,
- "ingestProduct: RepositoryManagerException "
- + "when adding Product References for Product : "
- + product.getProductName()
- + " to RepositoryManager: Message: "
- + e);
+ .log(
+ Level.SEVERE,
+ "ingestProduct: RepositoryManagerException "
+ + "when adding Product References for Product : "
+ + product.getProductName()
+ + " to RepositoryManager: Message: "
+ + e);
throw e;
}
} else {
@@ -1251,19 +1265,19 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
setProductTransferStatus(product);
} catch (CatalogException e) {
LOG
- .log(
- Level.SEVERE,
- "ingestProduct: RepositoryManagerException "
- + "when updating product transfer status for Product: "
- + product.getProductName()
- + " Message: " + e);
+ .log(
+ Level.SEVERE,
+ "ingestProduct: RepositoryManagerException "
+ + "when updating product transfer status for Product: "
+ + product.getProductName()
+ + " Message: " + e);
throw e;
}
} catch (Exception e) {
LOG.log(Level.SEVERE,
- "ingestProduct: DataTransferException when transfering Product: "
- + product.getProductName() + ": Message: "
- + e);
+ "ingestProduct: DataTransferException when transfering Product: "
+ + product.getProductName() + ": Message: "
+ + e);
throw new DataTransferException(e);
}
@@ -1274,81 +1288,81 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
} catch (VersioningException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.SEVERE,
- "ingestProduct: VersioningException when versioning Product: "
- + product.getProductName() + " with Versioner "
- + product.getProductType().getVersioner()
- + ": Message: " + e);
+ "ingestProduct: VersioningException when versioning Product: "
+ + product.getProductName() + " with Versioner "
+ + product.getProductType().getVersioner()
+ + ": Message: " + e);
throw new VersioningException(e);
} catch (XmlRpcException e2) {
LOG.log(Level.SEVERE, "Failed to ingest product [ name:" + product.getProductName() + "] :" + e2.getMessage()
- + " -- rolling back ingest");
+ + " -- rolling back ingest");
try {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
client.execute("filemgr.removeProduct", argList);
} catch (Exception e1) {
LOG.log(Level.SEVERE, "Failed to rollback ingest of product ["
- + product + "] : " + e2.getMessage());
+ + product + "] : " + e2.getMessage());
}
throw e2;
} catch (Exception e) {
LOG.log(Level.SEVERE, "Failed to ingest product [ id: " + product.getProductId() +
- "/ name:" + product.getProductName() + "] :" + e + " -- rolling back ingest");
+ "/ name:" + product.getProductName() + "] :" + e + " -- rolling back ingest");
try {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
client.execute("filemgr.removeProduct", argList);
} catch (Exception e1) {
LOG.log(Level.SEVERE, "Failed to rollback ingest of product ["
- + product + "] : " + e);
+ + product + "] : " + e);
}
throw new FileManagerException("Failed to ingest product [" + product + "] : "
- + e);
+ + e);
}
}
@SuppressWarnings("unchecked")
public Metadata getCatalogValues(Metadata metadata, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(metadata.getHashTable());
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
Metadata m = new Metadata();
m.addMetadata((Map<String, Object>) this.client.execute(
- "filemgr.getCatalogValues", args));
+ "filemgr.getCatalogValues", args));
return m;
}
@SuppressWarnings("unchecked")
public Metadata getOrigValues(Metadata metadata, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(metadata.getHashTable());
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
Metadata m = new Metadata();
m.addMetadata((Map<String, Object>) this.client.execute(
- "filemgr.getOrigValues", args));
+ "filemgr.getOrigValues", args));
return m;
}
@SuppressWarnings("unchecked")
public Query getCatalogQuery(Query query, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(XmlRpcStructFactory.getXmlRpcQuery(query));
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
return XmlRpcStructFactory
- .getQueryFromXmlRpc((Hashtable<String, Object>) this.client
- .execute("filemgr.getCatalogQuery", args));
+ .getQueryFromXmlRpc((Hashtable<String, Object>) this.client
+ .execute("filemgr.getCatalogQuery", args));
}
public static void main(String[] args) {