You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/10/12 17:57:25 UTC
[01/50] [abbrv] oodt git commit: OODT-940 update solr assets
Repository: oodt
Updated Branches:
refs/heads/master ebaae1f33 -> 8a3d51944
OODT-940 update solr assets
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/29fc6162
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/29fc6162
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/29fc6162
Branch: refs/heads/master
Commit: 29fc61620fb11360e4beae6216238e555f2afc35
Parents: e55f2cd
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Nov 10 15:59:24 2016 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Thu Nov 10 15:59:24 2016 +0000
----------------------------------------------------------------------
core/pom.xml | 4 +-
.../catalog/solr/DefaultProductSerializer.java | 310 +++++++++---------
.../cas/filemgr/catalog/solr/SolrCatalog.java | 324 ++++++++++---------
3 files changed, 334 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 6fafae6..3420597 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -382,12 +382,12 @@ the License.
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>1.3.0</version>
+ <version>6.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>1.3.0</version>
+ <version>6.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
index 72180e3..5b42a8b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/DefaultProductSerializer.java
@@ -16,12 +16,14 @@
*/
package org.apache.oodt.cas.filemgr.catalog.solr;
+import org.apache.commons.lang.StringEscapeUtils;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.solr.client.solrj.util.ClientUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
@@ -44,21 +46,21 @@ import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
/**
- * Default implementation of {@link ProductSerializer}
- * that transforms a CAS product into a single Solr document based on the following rules:
+ * Default implementation of {@link ProductSerializer}
+ * that transforms a CAS product into a single Solr document based on the following rules:
* o) the core product attributes are used to generate Solr fields starting with "CAS...."
* o) the product references are converted to Solr fields starting with "CAS.Reference..." or "CAS.RootReference..."
- * o) all other metadata fields are converted into Solr fields with the same name and number of values.
+ * o) all other metadata fields are converted into Solr fields with the same name and number of values.
* Note that the field multiplicity must be consistent with its definition in the Solr schema.xml.
- *
+ *
* This class generates all Solr documents in XML format.
- *
+ *
* @author Luca Cinquini
*
*/
public class DefaultProductSerializer implements ProductSerializer {
- private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName());
+ private static Logger LOG = Logger.getLogger(DefaultProductSerializer.class.getName());
/**
* {@inheritDoc}
*/
@@ -72,10 +74,10 @@ public class DefaultProductSerializer implements ProductSerializer {
*/
@Override
public List<String> serialize(Product product, boolean create) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
List<String> docs = new ArrayList<String>();
-
+
// add core product attributes to map
this.addKeyValueToMap(fields, Parameters.PRODUCT_ID, product.getProductId());
this.addKeyValueToMap(fields, Parameters.PRODUCT_NAME, product.getProductName());
@@ -84,117 +86,117 @@ public class DefaultProductSerializer implements ProductSerializer {
ProductType productType = product.getProductType();
if (productType!=null) {
this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_NAME, productType.getName());
- this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId());
+ this.addKeyValueToMap(fields, Parameters.PRODUCT_TYPE_ID, productType.getProductTypeId());
}
if (create) {
// only insert date/time when product is first created
Date productDateTime = new Date(); // current datetime
this.addKeyValueToMap(fields, Parameters.PRODUCT_RECEIVED_TIME, Parameters.SOLR_DATE_TIME_FORMATTER.format(productDateTime));
}
-
+
// create new product: use Solr id == CAS id
- if (create) {
+ if (create) {
docs.add( this.generateInsertDocuments(product.getProductId(), fields) );
-
- // update existing product
- } else {
+
+ // update existing product
+ } else {
docs.addAll( this.generateUpdateDocuments(product.getProductId(), fields, true) ); // replace=true
-
+
}
-
+
return docs;
-
+
}
-
+
/**
* {@inheritDoc}
*/
public List<String> serialize(String productId, Reference rootReference, List<Reference> references, boolean replace) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
-
+
// product root reference
if (rootReference!=null) {
-
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, rootReference.getOrigReference());
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, rootReference.getDataStoreReference());
+
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(rootReference.getOrigReference()));
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(rootReference.getDataStoreReference()));
addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_FILESIZE, ""+rootReference.getFileSize());
- addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, rootReference.getMimeType().toString());
-
+ addKeyValueToMap(fields, Parameters.ROOT_REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(rootReference.getMimeType().toString()));
+
}
-
+
// all other product references
- // note that Solr will preserve the indexing order.
+ // note that Solr will preserve the indexing order.
for (Reference reference : references) {
-
- addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, reference.getOrigReference());
- addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, reference.getDataStoreReference());
+
+ addKeyValueToMap(fields, Parameters.REFERENCE_ORIGINAL, StringEscapeUtils.escapeXml(reference.getOrigReference()));
+ addKeyValueToMap(fields, Parameters.REFERENCE_DATASTORE, StringEscapeUtils.escapeXml(reference.getDataStoreReference()));
addKeyValueToMap(fields, Parameters.REFERENCE_FILESIZE, ""+reference.getFileSize());
- addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, reference.getMimeType().toString());
-
+ addKeyValueToMap(fields, Parameters.REFERENCE_MIMETYPE, StringEscapeUtils.escapeXml(reference.getMimeType().toString()));
+
}
-
+
return generateUpdateDocuments(productId, fields, replace);
-
+
}
-
+
/**
* {@inheritDoc}
*/
public QueryResponse deserialize(String xml) throws CatalogException {
-
+
try {
-
+
QueryResponse queryResponse = new QueryResponse();
-
+
// parse XML into DOM
Document document = parseXml(xml);
-
+
// extract information from DOM to Product
Element response = document.getDocumentElement();
Node result = response.getElementsByTagName("result").item(0);
queryResponse.setNumFound( Integer.parseInt( ((Element)result).getAttribute("numFound") ) );
- queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) );
+ queryResponse.setStart( Integer.parseInt( ((Element)result).getAttribute("start") ) );
NodeList docs = result.getChildNodes();
for (int i=0; i< docs.getLength(); i++) {
Node node = docs.item(i);
if (node.getNodeName().equals("doc")) {
- Element doc = (Element)node;
+ Element doc = (Element)node;
CompleteProduct cp = this.deserialize(doc);
queryResponse.getCompleteProducts().add(cp);
}
- }
+ }
return queryResponse;
-
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage(), e);
}
-
+
}
-
+
/**
* {@inheritDoc}
*/
public List<String> serialize(String productId, Metadata metadata, boolean replace) {
-
+
Map<String, List<String>> fields = new ConcurrentHashMap<String, List<String>>();
-
+
for (String key : metadata.getKeys()) {
if (! (key.startsWith(Parameters.NS) // skip metadata keys starting with reserved namespace
- || Parameters.PRODUCT_TYPE_NAME.contains(key)
- // skip 'ProductType' as already stored as 'CAS.ProductTypeName'
- || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure'
+ //|| Parameters.PRODUCT_TYPE_NAME.contains(key)
+ // skip 'ProductType' as already stored as 'CAS.ProductTypeName'
+ || Parameters.PRODUCT_STRUCTURE.contains(key))) { // skip 'ProductType' as already stored as 'CAS.ProductStructure'
for (String value : metadata.getAllMetadata(key)) {
- this.addKeyValueToMap(fields, key, value);
+ this.addKeyValueToMap(fields, key, StringEscapeUtils.escapeXml(value));
}
}
}
-
+
return this.generateUpdateDocuments(productId, fields, replace);
-
+
}
-
+
/**
* Method to add a (key, value) to a multi-valued map with appropriate checks.
* @param map
@@ -202,7 +204,7 @@ public class DefaultProductSerializer implements ProductSerializer {
* @param value
*/
protected void addKeyValueToMap(Map<String, List<String>> map, String key, String value) {
-
+
if (!map.containsKey(key)) {
map.put(key, new ArrayList<String>());
}
@@ -213,33 +215,33 @@ public class DefaultProductSerializer implements ProductSerializer {
map.get(key).add(Parameters.NULL);
}
}
-
+
/**
* Utility method to generate a Solr insert document.
- *
+ *
* @param productId
* @param fields
* @return
*/
protected String generateInsertDocuments(String productId, Map<String,List<String>> fields) {
-
+
StringBuilder doc = new StringBuilder();
doc.append("<doc>");
-
+
// product Solr id field
doc.append( encodeIndexField(Parameters.ID, productId) );
-
+
// all other fields
for (Map.Entry<String, List<String>> key : fields.entrySet()) {
for (String value : key.getValue()) {
- doc.append( encodeIndexField(key.getKey(), value) );
+ doc.append( encodeIndexField(key.getKey(), StringEscapeUtils.escapeXml(value)) );
}
}
doc.append("</doc>");
return doc.toString();
}
-
+
/**
* Utility method to generate Solr update documents.
* Note that the requests for setting/adding/deleting fields must be sent as separate documents to Solr
@@ -249,51 +251,51 @@ public class DefaultProductSerializer implements ProductSerializer {
* @return
*/
protected List<String> generateUpdateDocuments(String productId, Map<String,List<String>> fields, boolean replace) {
-
+
// list for different instruction types
List<String> setFields = new ArrayList<String>();
List<String> addFields = new ArrayList<String>();
List<String> delFields = new ArrayList<String>();
-
+
// encode update instructions
for (Map.Entry<String, List<String>> key : fields.entrySet()) {
-
+
List<String> values = key.getValue();
-
+
if (replace) {
-
+
if (values.isEmpty()) {
// use special value to flag removal
delFields.add( this.encodeUpdateField(key.getKey(), Parameters.NULL, true) );
-
+
} else {
for (String value : values) {
- setFields.add( this.encodeUpdateField(key.getKey(), value, true) );
+ setFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), true) );
}
}
-
+
} else {
for (String value : values) {
- addFields.add( this.encodeUpdateField(key.getKey(), value, false) );
+ addFields.add( this.encodeUpdateField(key.getKey(), StringEscapeUtils.escapeXml(value), false) );
}
}
-
+
}
-
+
List<String> docs = new ArrayList<String>();
if (!delFields.isEmpty()) {
- docs.add(toDoc(productId, delFields));
+ docs.add(toDoc(productId, delFields));
}
if (!setFields.isEmpty()) {
- docs.add(toDoc(productId, setFields));
+ docs.add(toDoc(productId, setFields));
}
if (!addFields.isEmpty()) {
- docs.add(toDoc(productId, addFields));
+ docs.add(toDoc(productId, addFields));
}
return docs;
-
+
}
-
+
/**
* Utility method to merge field update instructions into a single document.
* @param productId
@@ -301,24 +303,24 @@ public class DefaultProductSerializer implements ProductSerializer {
* @return
*/
private String toDoc(String productId, List<String> updates) {
-
+
StringBuilder doc = new StringBuilder();
doc.append("<doc>");
-
+
// reference product record id
doc.append( encodeIndexField(Parameters.ID, productId) );
-
+
// loop over field update instructions
for (String update : updates) {
doc.append(update);
}
-
+
doc.append("</doc>");
-
+
return doc.toString();
-
+
}
-
+
/**
* Method to encode a Solr field indexing instruction.
* If the value is null, the empty string is returned.
@@ -333,11 +335,11 @@ public class DefaultProductSerializer implements ProductSerializer {
return "<field name=\""+key+"\">" + value + "</field>";
}
}
-
+
/**
* Method to encode a field update instruction for the three possible cases:
* add new values to a key (1), replace current values for a key (2), remove all values for a key (3).
- *
+ *
* @param key
* @param value
* @param replace
@@ -346,123 +348,135 @@ public class DefaultProductSerializer implements ProductSerializer {
protected String encodeUpdateField(String key, String value, boolean replace) {
StringBuilder sb = new StringBuilder();
sb.append("<field name=\"").append(key).append("\"");
-
+
if (replace) {
-
+
if (value==null || value.equals(Parameters.NULL)) {
-
+
// (3) remove all values for given key
sb.append(" update=\"set\" null=\"true\" />");
-
+
} else {
-
+
// (2) replace existing values with new values
sb.append(" update=\"set\">").append(value).append("</field>");
}
-
+
} else {
-
+
// (1) add new values to existing values
sb.append(" update=\"add\">").append(value).append("</field>");
-
+
}
return sb.toString();
}
-
-
+
+
/**
* Method that parses a single Solr document snippet
* to extract Product and Metadata attributes.
- *
+ *
* @param doc
* @return
*/
private CompleteProduct deserialize(Element doc) {
-
+
CompleteProduct cp = new CompleteProduct();
Product product = cp.getProduct();
ProductType productType = product.getProductType();
Metadata metadata = cp.getMetadata();
List<Reference> references = product.getProductReferences();
Reference rootReference = product.getRootRef();
-
+
NodeList children = doc.getChildNodes();
for (int j=0; j<children.getLength(); j++) {
-
+
Node child = children.item(j);
Element element = (Element)child;
String name = element.getAttribute("name");
-
+
/**
*<arr name="ScanPointingSource">
* <str>G073.65+0.19</str>
* <str>J2015+3410</str>
- * ..........
+ * ..........
*/
if (child.getNodeName().equals("arr")) {
-
+
NodeList values = element.getChildNodes();
List<String> vals = new ArrayList<String>();
for (int k=0; k<values.getLength(); k++) {
String value = ((Element)values.item(k)).getTextContent();
- vals.add(value);
+ vals.add(StringEscapeUtils.unescapeXml(value));
}
// CAS.reference.... fields
- if (name.startsWith(Parameters.NS)) {
- for (int k=0; k<values.getLength(); k++) {
- // create this reference
- if (references.size()<=k) {
- references.add(new Reference());
- }
- if (name.equals(Parameters.REFERENCE_ORIGINAL)) {
- references.get(k).setOrigReference(vals.get(k));
- } else if (name.equals(Parameters.REFERENCE_DATASTORE)) {
- references.get(k).setDataStoreReference(vals.get(k));
- } else if (name.equals(Parameters.REFERENCE_FILESIZE)) {
- references.get(k).setFileSize(Long.parseLong(vals.get(k)));
- } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) {
- references.get(k).setMimeType(vals.get(k));
- }
+ if (name.startsWith(Parameters.NS)) {
+ for (int k=0; k<values.getLength(); k++) {
+ // create this reference
+ if (references.size()<=k) {
+ references.add(new Reference());
}
- // all other multi-valued fields
+ if (name.equals(Parameters.REFERENCE_ORIGINAL)) {
+ references.get(k).setOrigReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_DATASTORE)) {
+ references.get(k).setDataStoreReference(vals.get(k));
+ } else if (name.equals(Parameters.REFERENCE_FILESIZE)) {
+ references.get(k).setFileSize(Long.parseLong(vals.get(k)));
+ } else if (name.equals(Parameters.REFERENCE_MIMETYPE)) {
+ references.get(k).setMimeType(vals.get(k));
+ }
+ }
+ // all other multi-valued fields
} else {
this.deserializeMultiValueField(name, vals, metadata);
}
-
- /**
- * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
- * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
- * <str name="CAS.ProductName">tns_br145x4_20</str>
- * <str name="FileLocation">/usr/local/ska-dc/data/archive</str>
- * ...........
- */
+
+ /**
+ * <str name="id">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductId">6684d79d-a011-4bc0-b3b3-4f11817091c8</str>
+ * <str name="CAS.ProductName">tns_br145x4_20</str>
+ * <str name="FileLocation">/usr/local/ska-dc/data/archive</str>
+ * ...........
+ */
} else {
-
- String value = element.getTextContent();
-
+
+ String value = StringEscapeUtils.unescapeXml(element.getTextContent());
+
// core CAS fields
if (name.startsWith(Parameters.NS)) {
if (name.equals(Parameters.PRODUCT_ID)) {
product.setProductId(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_NAME)) {
product.setProductName(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_STRUCTURE)) {
product.setProductStructure(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TRANSFER_STATUS)) {
product.setTransferStatus(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TYPE_NAME)) {
productType.setName(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_TYPE_ID)) {
productType.setProductTypeId(value);
+ metadata.addMetadata(name, value);
+
} else if (name.equals(Parameters.PRODUCT_RECEIVED_TIME)) {
product.setProductRecievedTime(value);
metadata.addMetadata(name, value);
// CAS root reference
} else if (name.startsWith(Parameters.NS+Parameters.ROOT)) {
if (rootReference==null) {
- rootReference = new Reference();
+ rootReference = new Reference();
}
if (name.equals(Parameters.ROOT_REFERENCE_ORIGINAL)) {
rootReference.setOrigReference(value);
@@ -475,48 +489,48 @@ public class DefaultProductSerializer implements ProductSerializer {
}
}
-
- // non core single-valued fields
+
+ // non core single-valued fields
} else {
this.deserializeSingleValueField(name, value, metadata);
} // "CAS".... or not
-
+
} // "arr" or anything else
-
+
} // loop over <doc> children
-
+
return cp;
-
+
}
-
+
private Document parseXml(String xml) throws IOException, SAXException, ParserConfigurationException {
-
+
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder parser = factory.newDocumentBuilder();
- return parser.parse( new InputSource(new StringReader(xml)) );
-
+ return parser.parse( new InputSource(new StringReader(xml)) );
+
}
-
+
/**
* Method that deserializes a single-valued Solr field into a Metadata element.
* This method can be overridden by sub-classes to provide custom behavior.
- *
+ *
* @param name : the Solr field name
* @param value : the Solr field single value
* @param metadata : the metadata container
*/
protected void deserializeSingleValueField(String name, String value, Metadata metadata) {
- // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId)
+ // ignore Solr internal identifier (as it is duplicate information of CAS.ProductId)
if (!name.equals(Parameters.ID)){
metadata.addMetadata(name, value);
}
}
-
+
/**
* Method that deserializes a multi-valued Solr field into a Metadata element.
* This method can be overridden by sub-classes to provide custom behavior.
- *
+ *
* @param name : the Solr field name
* @param values : the Solr field multiple values
* @param metadata : the metadata container
http://git-wip-us.apache.org/repos/asf/oodt/blob/29fc6162/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
index af554c5..a734e91 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrCatalog.java
@@ -38,23 +38,23 @@ import org.springframework.util.StringUtils;
/**
* Implementation of the CAS {@link Catalog} interface
* that uses a Solr back-end metadata store.
- *
+ *
* @author Luca Cinquini
*
*/
public class SolrCatalog implements Catalog {
-
+
// Class responsible for serializing/deserializing CAS products into Solr documents
ProductSerializer productSerializer;
-
+
// Class responsible for generating unique identifiers for incoming Products
ProductIdGenerator productIdGenerator;
-
+
// Class responsible for interacting with the Solr server
SolrClient solrClient;
-
+
private static final Logger LOG = Logger.getLogger(SolrCatalog.class.getName());
-
+
public SolrCatalog(String solrUrl, ProductIdGenerator productIdGenerator, ProductSerializer productSerializer) {
this.productIdGenerator = productIdGenerator;
this.productSerializer = productSerializer;
@@ -63,13 +63,15 @@ public class SolrCatalog implements Catalog {
@Override
public void addMetadata(Metadata metadata, Product product) throws CatalogException {
-
+
LOG.info("Adding metadata for product:"+product.getProductName());
-
+ if(metadata.containsKey("_version_")){
+ metadata.removeMetadata("_version_");
+ }
// serialize metadadta to Solr document(s)
- // replace=false i.e. add metadata to existing values
- List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false);
-
+ // replace=false i.e. add metadata to existing values
+ List<String> docs = productSerializer.serialize(product.getProductId(), metadata, false);
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -82,39 +84,42 @@ public class SolrCatalog implements Catalog {
*/
@Override
public void removeMetadata(Metadata metadata, Product product) throws CatalogException {
-
+
// retrieve full existing metadata
Metadata currentMetadata = getMetadata(product);
-
+
// metadata to be updated
Metadata updateMetadata = new Metadata();
-
+
// loop over keys of metadata be updated
for (String key : metadata.getKeys()) {
-
+
// list of values remaining after removal
List<String> values = new ArrayList<String>();
if (currentMetadata.containsKey(key)) {
for (String value : currentMetadata.getAllMetadata(key)) {
if (!metadata.getAllMetadata(key).contains(value)) {
values.add(value);
- }
+ }
}
// add remaining values to updated metadata
if (values.isEmpty()) {
// special value because Metadata will NOT store an empty list
values.add(Parameters.NULL);
- }
+ }
updateMetadata.addMetadata(key, values);
-
+
}
}
-
+
+ if(updateMetadata.containsKey("_version_")){
+ updateMetadata.removeMetadata("_version_");
+ }
// generate Solr update documents
// replace=true to override existing values
List<String> docs = productSerializer.serialize(product.getProductId(), updateMetadata, true);
-
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -127,40 +132,51 @@ public class SolrCatalog implements Catalog {
*/
@Override
public void addProduct(Product product) throws CatalogException {
-
- LOG.info("Adding product:"+product.getProductName());
-
- // generate product identifier if not existing already
- if (!StringUtils.hasText(product.getProductId())) {
- String productId = this.productIdGenerator.generateId(product);
- product.setProductId(productId);
+
+ if(product.getProductId()!=null && this.getCompleteProductById(product.getProductId()) !=null) {
+ throw new CatalogException(
+ "Attempt to add a product that already existed: product: ["
+ + product.getProductName() + "]");
+
+
+
+
+
+ } else {
+ LOG.info("Adding product:" + product.getProductName());
+
+ // generate product identifier if not existing already
+ if (!StringUtils.hasText(product.getProductId())) {
+ String productId = this.productIdGenerator.generateId(product);
+ product.setProductId(productId);
+ }
+
+ // serialize product for ingestion into Solr
+ List<String> docs = productSerializer.serialize(product, true); // create=true
+
+ // send records to Solr
+ solrClient.index(docs, true, productSerializer.getMimeType());
}
-
- // serialize product for ingestion into Solr
- List<String> docs = productSerializer.serialize(product, true); // create=true
-
- // send records to Solr
- solrClient.index(docs, true, productSerializer.getMimeType());
-
+
}
@Override
public void modifyProduct(Product product) throws CatalogException {
-
+
LOG.info("Modifying product:"+product.getProductName());
-
+
// serialize the update product information to Solr document(s)
List<String> docs = productSerializer.serialize(product, false); // create=false
-
+
// send records to Solr
solrClient.index(docs, true, productSerializer.getMimeType());
-
+
}
@Override
public void removeProduct(Product product) throws CatalogException {
-
+
// send message to Solr server
solrClient.delete(product.getProductId(), true);
@@ -168,17 +184,17 @@ public class SolrCatalog implements Catalog {
@Override
public void setProductTransferStatus(Product product) throws CatalogException {
-
+
this.modifyProduct(product);
}
@Override
public void addProductReferences(Product product) throws CatalogException {
-
+
// generate update documents (with replace=true)
- List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true);
-
+ List<String> docs = productSerializer.serialize(product.getProductId(), product.getRootRef(), product.getProductReferences(), true);
+
// send documents to Solr server
solrClient.index(docs, true, productSerializer.getMimeType());
@@ -186,51 +202,51 @@ public class SolrCatalog implements Catalog {
@Override
public Product getProductById(String productId) throws CatalogException {
-
- CompleteProduct cp = getCompleteProductById(productId);
- return cp.getProduct();
-
+
+ CompleteProduct cp = getCompleteProductById(productId);
+ return cp.getProduct();
+
}
@Override
public Product getProductByName(String productName) throws CatalogException {
-
- CompleteProduct cp = getCompleteProductByName(productName);
+
+ CompleteProduct cp = getCompleteProductByName(productName);
if (cp!=null) {
- LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId());
- return cp.getProduct();
+ LOG.info("Found product name="+productName+" id="+cp.getProduct().getProductId());
+ return cp.getProduct();
} else {
- LOG.info("Product with name="+productName+" not found");
- return null;
+ LOG.info("Product with name="+productName+" not found");
+ return null;
}
-
+
}
@Override
public List<Reference> getProductReferences(Product product) throws CatalogException {
-
+
CompleteProduct cp = getCompleteProductById(product.getProductId());
return cp.getProduct().getProductReferences();
-
+
}
@Override
public List<Product> getProducts() throws CatalogException {
-
+
// build query parameters
Map<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[] { "*:*" } );
//params.put("rows", new String[] { "20" } ); // control pagination ?
-
+
// query Solr for all matching products
return getProducts(params, 0, -1).getProducts(); // get ALL products
-
+
}
-
+
/**
* Common utility to retrieve a range of products matching the specified {@link Query} and {@link ProductType}.
* This method transforms the given constraints in a map of HTTP (name, value) pairs and delegates to the following method.
- *
+ *
* @param query
* @param type
* @param offset
@@ -239,25 +255,25 @@ public class SolrCatalog implements Catalog {
* @throws CatalogException
*/
private QueryResponse getProducts(Query query, ProductType type, int offset, int limit) throws CatalogException {
-
+
// build HTTP request
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
// product type constraint
params.put("q", new String[]{Parameters.PRODUCT_TYPE_NAME+":"+type.getName()} );
- // convert filemgr query into a Solr query
+ // convert filemgr query into a Solr query
List<String> qc = new ArrayList<String>();
- for (QueryCriteria queryCriteria : query.getCriteria()) {
- LOG.info("Query criteria="+queryCriteria.toString());
- qc.add(queryCriteria.toString());
- }
- params.put("fq", qc.toArray( new String[ qc.size() ] ));
- // sort
- params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
-
- return this.getProducts(params, offset, limit);
-
- }
-
+ for (QueryCriteria queryCriteria : query.getCriteria()) {
+ LOG.info("Query criteria="+queryCriteria.toString());
+ qc.add(queryCriteria.toString());
+ }
+ params.put("fq", qc.toArray( new String[ qc.size() ] ));
+ // sort
+ params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
+
+ return this.getProducts(params, offset, limit);
+
+ }
+
/**
* Common utility to retrieve a range of products matching the specified query parameters
* @param params : HTTP query parameters used for query to Solr
@@ -266,38 +282,38 @@ public class SolrCatalog implements Catalog {
* @return
*/
private QueryResponse getProducts(Map<String, String[]> params, int offset, int limit) throws CatalogException {
-
+
// combined results from pagination
QueryResponse queryResponse = new QueryResponse();
queryResponse.setStart(offset);
int start = offset;
while (queryResponse.getCompleteProducts().size()<limit || limit<0) {
-
+
params.put("start", new String[] { ""+start } );
String response = solrClient.query(params, productSerializer.getMimeType());
QueryResponse qr = productSerializer.deserialize(response);
-
+
for (CompleteProduct cp : qr.getCompleteProducts()) {
if (queryResponse.getCompleteProducts().size()<limit) {
queryResponse.getCompleteProducts().add( cp );
}
}
-
+
queryResponse.setNumFound( qr.getNumFound() );
start = offset+queryResponse.getCompleteProducts().size();
if (limit<0) {
- limit = queryResponse.getNumFound(); // retrieve ALL results
+ limit = queryResponse.getNumFound(); // retrieve ALL results
}
if (start>=queryResponse.getNumFound()) {
- break; // don't query any longer
+ break; // don't query any longer
}
-
+
}
-
+
LOG.info("Total number of products found="+queryResponse.getNumFound());
LOG.info("Total number of products returned="+queryResponse.getCompleteProducts().size());
return queryResponse;
-
+
}
@Override
@@ -308,7 +324,7 @@ public class SolrCatalog implements Catalog {
params.put("q", new String[] { "*:*" } );
// use the product type name as query parameter
params.put("fq", new String[] { Parameters.PRODUCT_TYPE_NAME+":"+type.getName() } );
-
+
// query Solr for all matching products
return getProducts(params, 0, -1).getProducts(); // get ALL products
@@ -316,28 +332,28 @@ public class SolrCatalog implements Catalog {
@Override
public Metadata getMetadata(Product product) throws CatalogException {
-
+
CompleteProduct cp = getCompleteProductById(product.getProductId());
return cp.getMetadata();
-
+
}
@Override
public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException {
-
+
// build HTTP request
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{Parameters.PRODUCT_ID+":"+product.getProductId()} );
// request metadata elements explicitly
params.put("fl", elements.toArray(new String[elements.size()]) );
-
+
// execute request
String doc = solrClient.query(params, productSerializer.getMimeType());
-
+
// parse response
CompleteProduct cp = extractCompleteProduct(doc);
return cp.getMetadata();
-
+
}
/**
@@ -346,50 +362,50 @@ public class SolrCatalog implements Catalog {
*/
@Override
public List<String> query(Query query, ProductType type) throws CatalogException {
-
- // execute request for ALL results
- QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products
+
+ // execute request for ALL results
+ QueryResponse queryResponse = this.getProducts(query, type, 0, -1); // get ALL products
// extract ids from products
List<String> ids = new ArrayList<String>();
for (CompleteProduct cp : queryResponse.getCompleteProducts()) {
ids.add(cp.getProduct().getProductId());
}
-
+
return ids;
-
+
}
@Override
- public List<Product> getTopNProducts(int n) throws CatalogException {
-
+ public List<Product> getTopNProducts(int n) throws CatalogException {
+
// retrieve most recent n products from Solr
String doc = solrClient.queryProductsByDate(n, productSerializer.getMimeType());
-
+
// parse Solr response into Product objects
return this.getProductsFromDocument(doc);
-
+
}
-
+
@Override
public ProductPage pagedQuery(Query query, ProductType type, int pageNum) throws CatalogException {
-
-
- // execute request for one page of results
- int offset = (pageNum-1)*Parameters.PAGE_SIZE;
- int limit = Parameters.PAGE_SIZE;
- QueryResponse queryResponse = this.getProducts(query, type, offset, limit);
-
- // build product page from query response
- return newProductPage(pageNum, queryResponse);
-
- }
-
+
+
+ // execute request for one page of results
+ int offset = (pageNum-1)*Parameters.PAGE_SIZE;
+ int limit = Parameters.PAGE_SIZE;
+ QueryResponse queryResponse = this.getProducts(query, type, offset, limit);
+
+ // build product page from query response
+ return newProductPage(pageNum, queryResponse);
+
+ }
+
@Override
public ProductPage getFirstPage(ProductType type) {
try {
return this.pagedQuery(new Query(), type, 1);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
@@ -397,82 +413,82 @@ public class SolrCatalog implements Catalog {
@Override
public ProductPage getLastProductPage(ProductType type) {
-
+
try {
-
+
// query for total number of products of this type
int numTotalResults = this.getNumProducts(type);
-
+
// compute last page number
int numOfPages = PaginationUtils.getTotalPage(numTotalResults, Parameters.PAGE_SIZE);
-
+
// request last page
return pagedQuery(new Query(), type, numOfPages);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
@Override
public ProductPage getNextPage(ProductType type, ProductPage currentPage) {
-
+
int nextPageNumber = currentPage.getPageNum()+1;
if (nextPageNumber>currentPage.getTotalPages()) {
- throw new RuntimeException("Invalid next page number: " + nextPageNumber);
+ throw new RuntimeException("Invalid next page number: " + nextPageNumber);
}
try {
return this.pagedQuery(new Query(), type, currentPage.getPageNum()+1);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
@Override
public ProductPage getPrevPage(ProductType type, ProductPage currentPage) {
-
+
int prevPageNumber = currentPage.getPageNum()-1;
if (prevPageNumber<=0) {
- throw new RuntimeException("Invalid previous page number: " + prevPageNumber);
+ throw new RuntimeException("Invalid previous page number: " + prevPageNumber);
}
-
+
try {
return this.pagedQuery(new Query(), type, prevPageNumber);
-
+
} catch(CatalogException e) {
throw new RuntimeException(e.getMessage(), e);
}
-
+
}
-
+
/**
* Common functionality for extracting products from a Solr response document.
* @param doc
* @return
*/
private List<Product> getProductsFromDocument(String doc) throws CatalogException {
-
+
// extract full product information from Solr response
QueryResponse queryResponse = productSerializer.deserialize(doc);
-
+
// return products only
return queryResponse.getProducts();
-
+
}
@Override
public List<Product> getTopNProducts(int n, ProductType type) throws CatalogException {
-
+
// retrieve most recent n products from Solr
String doc = solrClient.queryProductsByDateAndType(n, type, productSerializer.getMimeType());
-
+
// parse Solr response into Product objects
return this.getProductsFromDocument(doc);
-
+
}
@Override
@@ -484,43 +500,43 @@ public class SolrCatalog implements Catalog {
@Override
public int getNumProducts(ProductType type) throws CatalogException {
-
+
// build query parameters
Map<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[] { "CAS.ProductTypeName:"+type.getName() } );
params.put("rows", new String[] { "0" } ); // don't return any results
-
+
// execute query
String response = solrClient.query(params, productSerializer.getMimeType());
-
+
// parse response
QueryResponse queryResponse = productSerializer.deserialize(response);
return queryResponse.getNumFound();
-
+
}
-
+
private CompleteProduct getCompleteProductById(String productId) throws CatalogException {
-
+
// request document with given id
String doc = solrClient.queryProductById(productId, productSerializer.getMimeType());
-
+
// parse document into complete product
return extractCompleteProduct(doc);
-
+
}
-
- private CompleteProduct getCompleteProductByName(String productName) throws CatalogException {
-
+
+ private CompleteProduct getCompleteProductByName(String productName) throws CatalogException {
+
// request document with given id
String doc = solrClient.queryProductByName(productName, productSerializer.getMimeType());
-
+
// parse document into complete product
return extractCompleteProduct(doc);
-
+
}
-
+
private CompleteProduct extractCompleteProduct(String doc) throws CatalogException {
-
+
// deserialize document into Product
LOG.info("Parsing Solr document: "+doc);
QueryResponse queryResponse = productSerializer.deserialize(doc);
@@ -532,7 +548,7 @@ public class SolrCatalog implements Catalog {
} else {
return queryResponse.getCompleteProducts().get(0);
}
-
+
}
/**
@@ -541,7 +557,7 @@ public class SolrCatalog implements Catalog {
* @return
*/
private ProductPage newProductPage(int pageNum, QueryResponse queryResponse) {
-
+
ProductPage page = new ProductPage();
page.setPageNum(pageNum);
page.setPageSize(queryResponse.getProducts().size());
@@ -549,7 +565,7 @@ public class SolrCatalog implements Catalog {
page.setPageProducts(queryResponse.getProducts());
page.setTotalPages(PaginationUtils.getTotalPage(queryResponse.getNumFound(), Parameters.PAGE_SIZE));
return page;
-
+
}
}
[12/50] [abbrv] oodt git commit: ignore
Posted by ma...@apache.org.
ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/d7a228e4
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/d7a228e4
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/d7a228e4
Branch: refs/heads/master
Commit: d7a228e4c403f290664f7b43b98b8353ece30e9a
Parents: 4827e9c
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:32:18 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:11:33 2017 -0800
----------------------------------------------------------------------
protocol/imaps/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/d7a228e4/protocol/imaps/.gitignore
----------------------------------------------------------------------
diff --git a/protocol/imaps/.gitignore b/protocol/imaps/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/protocol/imaps/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[37/50] [abbrv] oodt git commit: Remove grid.
Posted by ma...@apache.org.
Remove grid.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/758eece9
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/758eece9
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/758eece9
Branch: refs/heads/master
Commit: 758eece9ea648ed784101aa8612286a8d1a17241
Parents: 423444c
Author: Chris Mattmann <ma...@apache.org>
Authored: Sat Sep 2 08:37:04 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sat Sep 2 08:37:04 2017 -0700
----------------------------------------------------------------------
grid/web-grid.iml.orig | 134 ---------------------------
grid/web-grid.iml.rej | 216 --------------------------------------------
2 files changed, 350 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/758eece9/grid/web-grid.iml.orig
----------------------------------------------------------------------
diff --git a/grid/web-grid.iml.orig b/grid/web-grid.iml.orig
deleted file mode 100755
index 780b5aa..0000000
--- a/grid/web-grid.iml.orig
+++ /dev/null
@@ -1,134 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
- <component name="FacetManager">
- <facet type="web" name="Web">
- <configuration>
- <descriptors>
- <deploymentDescriptor name="web.xml" url="file://$MODULE_DIR$/src/main/webapp/WEB-INF/web.xml" />
- </descriptors>
- <webroots>
- <root url="file://$MODULE_DIR$/src/main/webapp" relative="/" />
- </webroots>
- </configuration>
- </facet>
- </component>
- <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
- <output url="file://$MODULE_DIR$/target/classes" />
- <output-test url="file://$MODULE_DIR$/target/test-classes" />
- <content url="file://$MODULE_DIR$">
- <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
- <excludeFolder url="file://$MODULE_DIR$/target" />
- </content>
- <orderEntry type="library" name="Clover IDEA Plugin" level="application" />
- <orderEntry type="inheritedJdk" />
- <orderEntry type="sourceFolder" forTests="false" />
- <orderEntry type="library" name="Maven: org.springframework:spring-expression:3.0.5.RELEASE" level="project" />
- <orderEntry type="library" name="Maven: joda-time:joda-time:2.7" level="project" />
- <orderEntry type="module" module-name="oodt-commons" />
- <orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.1" level="project" />
- <orderEntry type="library" name="Maven: commons-collections:commons-collections:2.1" level="project" />
- <orderEntry type="library" name="Maven: commons-pool:commons-pool:1.2" level="project" />
- <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.3" level="project" />
- <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-core:2.5.4" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-hibernate3:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
- <orderEntry type="library" name="Maven: org.hibernate:hibernate:3.2.5.ga" level="project" />
- <orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:1.2.3" level="project" />
- <orderEntry type="library" name="Maven: asm:asm-attrs:1.5.3" level="project" />
- <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
- <orderEntry type="library" name="Maven: antlr:antlr:2.7.6" level="project" />
- <orderEntry type="library" name="Maven: cglib:cglib:2.1_3" level="project" />
- <orderEntry type="library" name="Maven: asm:asm:1.5.3" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-beans:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-context:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-dao:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:2.0.8" level="project" />
- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
- <orderEntry type="module" module-name="oodt-profile" />
- <orderEntry type="module" module-name="cas-filemgr" />
- <orderEntry type="library" name="Maven: org.apache.tika:tika-parsers:1.7" level="project" />
- <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-tika:0.6" level="project" />
- <orderEntry type="library" name="Maven: edu.ucar:netcdf:4.2.20" level="project" />
- <orderEntry type="library" name="Maven: edu.ucar:unidataCommon:4.2.20" level="project" />
- <orderEntry type="library" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
- <orderEntry type="library" name="Maven: net.sourceforge.jmatio:jmatio:1.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-core:0.7.2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-dom:0.7.2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.8.1" level="project" />
- <orderEntry type="library" name="Maven: org.tukaani:xz:1.5" level="project" />
- <orderEntry type="library" name="Maven: org.apache.pdfbox:pdfbox:1.8.8" level="project" />
- <orderEntry type="library" name="Maven: org.apache.pdfbox:fontbox:1.8.8" level="project" />
- <orderEntry type="library" name="Maven: org.apache.pdfbox:jempbox:1.8.8" level="project" />
- <orderEntry type="library" name="Maven: org.bouncycastle:bcmail-jdk15:1.45" level="project" />
- <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15:1.45" level="project" />
- <orderEntry type="library" name="Maven: org.apache.poi:poi:3.11" level="project" />
- <orderEntry type="library" name="Maven: org.apache.poi:poi-scratchpad:3.11" level="project" />
- <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.11" level="project" />
- <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.11" level="project" />
- <orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
- <orderEntry type="library" name="Maven: org.ccil.cowan.tagsoup:tagsoup:1.2.1" level="project" />
- <orderEntry type="library" name="Maven: org.ow2.asm:asm-debug-all:4.1" level="project" />
- <orderEntry type="library" name="Maven: com.googlecode.mp4parser:isoparser:1.0.2" level="project" />
- <orderEntry type="library" name="Maven: org.aspectj:aspectjrt:1.8.0" level="project" />
- <orderEntry type="library" name="Maven: com.drewnoakes:metadata-extractor:2.6.2" level="project" />
- <orderEntry type="library" name="Maven: com.adobe.xmp:xmpcore:5.1.2" level="project" />
- <orderEntry type="library" name="Maven: de.l3s.boilerpipe:boilerpipe:1.1.0" level="project" />
- <orderEntry type="library" name="Maven: rome:rome:1.0" level="project" />
- <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
- <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-core:0.6" level="project" />
- <orderEntry type="library" name="Maven: com.googlecode.juniversalchardet:juniversalchardet:1.0.3" level="project" />
- <orderEntry type="library" name="Maven: com.uwyn:jhighlight:1.0" level="project" />
- <orderEntry type="library" name="Maven: com.pff:java-libpst:0.8.1" level="project" />
- <orderEntry type="module" module-name="cas-metadata" />
- <orderEntry type="module" module-name="pcs-input" />
- <orderEntry type="module" module-name="cas-cli" />
- <orderEntry type="library" name="Maven: com.google.guava:guava:10.0.1" level="project" />
- <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
- <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-common:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: commons-fileupload:commons-fileupload:1.2" level="project" />
- <orderEntry type="library" name="Maven: stax:stax:1.2.0" level="project" />
- <orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
- <orderEntry type="library" name="Maven: net.java.dev.stax-utils:stax-utils:20040917" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-core:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-analyzers:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-highlighter:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-queries:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-snowball:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-spellchecker:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.solr:solr-commons-csv:1.3.0" level="project" />
- <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-stax-api_1.0_spec:1.0.1" level="project" />
- <orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:2.0.0" level="project" />
- <orderEntry type="library" name="Maven: org.safehaus.jug:jug:asl:2.0.0" level="project" />
- <orderEntry type="library" name="Maven: hsqldb:hsqldb:1.8.0.7" level="project" />
- <orderEntry type="library" scope="RUNTIME" name="Maven: org.python:jython:2.2-beta1" level="project" />
- <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk:1.7.4" level="project" />
- <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2" level="project" />
- <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2" level="project" />
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.1.1" level="project" />
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.1.1" level="project" />
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.1.1" level="project" />
- <orderEntry type="library" name="Maven: joda-time:joda-time:2.8.2" level="project" />
- <orderEntry type="library" name="Maven: com.ibm.icu:icu4j:3.4.4" level="project" />
- <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.0-alpha1" level="project" />
- <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
- <orderEntry type="module" module-name="oodt-product" />
- <orderEntry type="library" name="Maven: commons-io:commons-io:1.4" level="project" />
- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.7" level="project" />
- <orderEntry type="module" module-name="oodt-xmlquery" />
- <orderEntry type="library" name="Maven: com.hp.hpl.jena:jena:2.6.3" level="project" />
- <orderEntry type="library" name="Maven: com.hp.hpl.jena:iri:0.8" level="project" />
- <orderEntry type="library" name="Maven: com.hp.hpl.jena:iri:sources:0.8" level="project" />
- <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.5.8" level="project" />
- <orderEntry type="library" scope="RUNTIME" name="Maven: org.slf4j:slf4j-log4j12:1.5.8" level="project" />
- <orderEntry type="library" name="Maven: log4j:log4j:1.2.13" level="project" />
- <orderEntry type="library" scope="PROVIDED" name="Maven: javax.servlet:servlet-api:2.4" level="project" />
- <orderEntry type="library" scope="TEST" name="Maven: junit:junit:3.8.2" level="project" />
- <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
- <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
- <orderEntry type="library" name="Maven: xalan:xalan:2.7.1" level="project" />
- <orderEntry type="library" name="Maven: xalan:serializer:2.7.1" level="project" />
- </component>
-</module>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/758eece9/grid/web-grid.iml.rej
----------------------------------------------------------------------
diff --git a/grid/web-grid.iml.rej b/grid/web-grid.iml.rej
deleted file mode 100755
index dcb6b28..0000000
--- a/grid/web-grid.iml.rej
+++ /dev/null
@@ -1,216 +0,0 @@
---- grid/web-grid.iml
-+++ grid/web-grid.iml
-@@ -1,5 +1,20 @@
- <?xml version="1.0" encoding="UTF-8"?>
- <module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
-+ <component name="FacetManager">
-+ <facet type="web" name="Web">
-+ <configuration>
-+ <descriptors>
-+ <deploymentDescriptor name="web.xml" url="file://$MODULE_DIR$/src/main/webapp/WEB-INF/web.xml" />
-+ </descriptors>
-+ <webroots>
-+ <root url="file://$MODULE_DIR$/src/main/webapp" relative="/" />
-+ </webroots>
-+ <sourceRoots>
-+ <root url="file://$MODULE_DIR$/src/main/java" />
-+ </sourceRoots>
-+ </configuration>
-+ </facet>
-+ </component>
- <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
- <output url="file://$MODULE_DIR$/target/classes" />
- <output-test url="file://$MODULE_DIR$/target/test-classes" />
-@@ -35,10 +52,42 @@
- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
- <orderEntry type="module" module-name="oodt-profile" />
- <orderEntry type="module" module-name="cas-filemgr" />
-+ <orderEntry type="library" name="Maven: org.apache.tika:tika-parsers:1.7" level="project" />
-+ <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-tika:0.6" level="project" />
-+ <orderEntry type="library" name="Maven: edu.ucar:netcdf:4.2.20" level="project" />
-+ <orderEntry type="library" name="Maven: edu.ucar:unidataCommon:4.2.20" level="project" />
-+ <orderEntry type="library" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
-+ <orderEntry type="library" name="Maven: net.sourceforge.jmatio:jmatio:1.0" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-core:0.7.2" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-dom:0.7.2" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.8.1" level="project" />
-+ <orderEntry type="library" name="Maven: org.tukaani:xz:1.5" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.pdfbox:pdfbox:1.8.8" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.pdfbox:fontbox:1.8.8" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.pdfbox:jempbox:1.8.8" level="project" />
-+ <orderEntry type="library" name="Maven: org.bouncycastle:bcmail-jdk15:1.45" level="project" />
-+ <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15:1.45" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.poi:poi:3.11" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.poi:poi-scratchpad:3.11" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.11" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.11" level="project" />
-+ <orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
-+ <orderEntry type="library" name="Maven: org.ccil.cowan.tagsoup:tagsoup:1.2.1" level="project" />
-+ <orderEntry type="library" name="Maven: org.ow2.asm:asm-debug-all:4.1" level="project" />
-+ <orderEntry type="library" name="Maven: com.googlecode.mp4parser:isoparser:1.0.2" level="project" />
-+ <orderEntry type="library" name="Maven: org.aspectj:aspectjrt:1.8.0" level="project" />
-+ <orderEntry type="library" name="Maven: com.drewnoakes:metadata-extractor:2.6.2" level="project" />
-+ <orderEntry type="library" name="Maven: com.adobe.xmp:xmpcore:5.1.2" level="project" />
-+ <orderEntry type="library" name="Maven: de.l3s.boilerpipe:boilerpipe:1.1.0" level="project" />
-+ <orderEntry type="library" name="Maven: rome:rome:1.0" level="project" />
-+ <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
-+ <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-core:0.6" level="project" />
-+ <orderEntry type="library" name="Maven: com.googlecode.juniversalchardet:juniversalchardet:1.0.3" level="project" />
-+ <orderEntry type="library" name="Maven: com.uwyn:jhighlight:1.0" level="project" />
-+ <orderEntry type="library" name="Maven: com.pff:java-libpst:0.8.1" level="project" />
- <orderEntry type="module" module-name="cas-metadata" />
- <orderEntry type="module" module-name="pcs-input" />
- <orderEntry type="module" module-name="cas-cli" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-expression:3.0.5.RELEASE" level="project" />
- <orderEntry type="library" name="Maven: com.google.guava:guava:10.0.1" level="project" />
- <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
- <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
-@@ -66,7 +115,7 @@
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.1.1" level="project" />
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.1.1" level="project" />
- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.1.1" level="project" />
-- <orderEntry type="library" name="Maven: joda-time:joda-time:2.7" level="project" />
-+ <orderEntry type="library" name="Maven: joda-time:joda-time:2.8.1" level="project" />
- <orderEntry type="library" name="Maven: com.ibm.icu:icu4j:3.4.4" level="project" />
- <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.0-alpha1" level="project" />
- <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
---- grid/web-grid.iml
-+++ grid/web-grid.iml
-@@ -1,137 +0,0 @@
--<?xml version="1.0" encoding="UTF-8"?>
--<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
-- <component name="FacetManager">
-- <facet type="web" name="Web">
-- <configuration>
-- <descriptors>
-- <deploymentDescriptor name="web.xml" url="file://$MODULE_DIR$/src/main/webapp/WEB-INF/web.xml" />
-- </descriptors>
-- <webroots>
-- <root url="file://$MODULE_DIR$/src/main/webapp" relative="/" />
-- </webroots>
-- <sourceRoots>
-- <root url="file://$MODULE_DIR$/src/main/java" />
-- </sourceRoots>
-- </configuration>
-- </facet>
-- </component>
-- <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
-- <output url="file://$MODULE_DIR$/target/classes" />
-- <output-test url="file://$MODULE_DIR$/target/test-classes" />
-- <content url="file://$MODULE_DIR$">
-- <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
-- <sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
-- <excludeFolder url="file://$MODULE_DIR$/target" />
-- </content>
-- <orderEntry type="inheritedJdk" />
-- <orderEntry type="sourceFolder" forTests="false" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-expression:3.0.5.RELEASE" level="project" />
-- <orderEntry type="library" name="Maven: joda-time:joda-time:2.7" level="project" />
-- <orderEntry type="module" module-name="oodt-commons" />
-- <orderEntry type="library" name="Maven: commons-dbcp:commons-dbcp:1.2.1" level="project" />
-- <orderEntry type="library" name="Maven: commons-collections:commons-collections:2.1" level="project" />
-- <orderEntry type="library" name="Maven: commons-pool:commons-pool:1.2" level="project" />
-- <orderEntry type="library" name="Maven: commons-lang:commons-lang:2.3" level="project" />
-- <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.0.3" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-core:2.5.4" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-hibernate3:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: aopalliance:aopalliance:1.0" level="project" />
-- <orderEntry type="library" name="Maven: org.hibernate:hibernate:3.2.5.ga" level="project" />
-- <orderEntry type="library" name="Maven: net.sf.ehcache:ehcache:1.2.3" level="project" />
-- <orderEntry type="library" name="Maven: asm:asm-attrs:1.5.3" level="project" />
-- <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
-- <orderEntry type="library" name="Maven: antlr:antlr:2.7.6" level="project" />
-- <orderEntry type="library" name="Maven: cglib:cglib:2.1_3" level="project" />
-- <orderEntry type="library" name="Maven: asm:asm:1.5.3" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-beans:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-context:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-dao:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.springframework:spring-jdbc:2.0.8" level="project" />
-- <orderEntry type="library" name="Maven: xmlrpc:xmlrpc:2.0.1" level="project" />
-- <orderEntry type="module" module-name="oodt-profile" />
-- <orderEntry type="module" module-name="cas-filemgr" />
-- <orderEntry type="library" name="Maven: org.apache.tika:tika-parsers:1.7" level="project" />
-- <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-tika:0.6" level="project" />
-- <orderEntry type="library" name="Maven: edu.ucar:netcdf:4.2.20" level="project" />
-- <orderEntry type="library" name="Maven: edu.ucar:unidataCommon:4.2.20" level="project" />
-- <orderEntry type="library" name="Maven: net.jcip:jcip-annotations:1.0" level="project" />
-- <orderEntry type="library" name="Maven: net.sourceforge.jmatio:jmatio:1.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-core:0.7.2" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-dom:0.7.2" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.8.1" level="project" />
-- <orderEntry type="library" name="Maven: org.tukaani:xz:1.5" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.pdfbox:pdfbox:1.8.8" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.pdfbox:fontbox:1.8.8" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.pdfbox:jempbox:1.8.8" level="project" />
-- <orderEntry type="library" name="Maven: org.bouncycastle:bcmail-jdk15:1.45" level="project" />
-- <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15:1.45" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.poi:poi:3.11" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.poi:poi-scratchpad:3.11" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.11" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.11" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
-- <orderEntry type="library" name="Maven: org.ccil.cowan.tagsoup:tagsoup:1.2.1" level="project" />
-- <orderEntry type="library" name="Maven: org.ow2.asm:asm-debug-all:4.1" level="project" />
-- <orderEntry type="library" name="Maven: com.googlecode.mp4parser:isoparser:1.0.2" level="project" />
-- <orderEntry type="library" name="Maven: org.aspectj:aspectjrt:1.8.0" level="project" />
-- <orderEntry type="library" name="Maven: com.drewnoakes:metadata-extractor:2.6.2" level="project" />
-- <orderEntry type="library" name="Maven: com.adobe.xmp:xmpcore:5.1.2" level="project" />
-- <orderEntry type="library" name="Maven: de.l3s.boilerpipe:boilerpipe:1.1.0" level="project" />
-- <orderEntry type="library" name="Maven: rome:rome:1.0" level="project" />
-- <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
-- <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-core:0.6" level="project" />
-- <orderEntry type="library" name="Maven: com.googlecode.juniversalchardet:juniversalchardet:1.0.3" level="project" />
-- <orderEntry type="library" name="Maven: com.uwyn:jhighlight:1.0" level="project" />
-- <orderEntry type="library" name="Maven: com.pff:java-libpst:0.8.1" level="project" />
-- <orderEntry type="module" module-name="cas-metadata" />
-- <orderEntry type="module" module-name="pcs-input" />
-- <orderEntry type="module" module-name="cas-cli" />
-- <orderEntry type="library" name="Maven: com.google.guava:guava:10.0.1" level="project" />
-- <orderEntry type="library" name="Maven: com.google.code.findbugs:jsr305:1.3.9" level="project" />
-- <orderEntry type="library" name="Maven: commons-cli:commons-cli:1.2" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-solrj:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-common:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: commons-fileupload:commons-fileupload:1.2" level="project" />
-- <orderEntry type="library" name="Maven: stax:stax:1.2.0" level="project" />
-- <orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
-- <orderEntry type="library" name="Maven: net.java.dev.stax-utils:stax-utils:20040917" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-core:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-analyzers:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-highlighter:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-queries:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-snowball:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-lucene-spellchecker:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.solr:solr-commons-csv:1.3.0" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-stax-api_1.0_spec:1.0.1" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:2.0.0" level="project" />
-- <orderEntry type="library" name="Maven: org.safehaus.jug:jug:asl:2.0.0" level="project" />
-- <orderEntry type="library" name="Maven: hsqldb:hsqldb:1.8.0.7" level="project" />
-- <orderEntry type="library" scope="RUNTIME" name="Maven: org.python:jython:2.2-beta1" level="project" />
-- <orderEntry type="library" name="Maven: com.amazonaws:aws-java-sdk:1.7.4" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.2" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.2" level="project" />
-- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.1.1" level="project" />
-- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.1.1" level="project" />
-- <orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.1.1" level="project" />
-- <orderEntry type="library" name="Maven: joda-time:joda-time:2.8.1" level="project" />
-- <orderEntry type="library" name="Maven: com.ibm.icu:icu4j:3.4.4" level="project" />
-- <orderEntry type="library" name="Maven: commons-httpclient:commons-httpclient:3.0-alpha1" level="project" />
-- <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.3" level="project" />
-- <orderEntry type="module" module-name="oodt-product" />
-- <orderEntry type="library" name="Maven: commons-io:commons-io:1.4" level="project" />
-- <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.7" level="project" />
-- <orderEntry type="module" module-name="oodt-xmlquery" />
-- <orderEntry type="library" name="Maven: com.hp.hpl.jena:jena:2.6.3" level="project" />
-- <orderEntry type="library" name="Maven: com.hp.hpl.jena:iri:0.8" level="project" />
-- <orderEntry type="library" name="Maven: com.hp.hpl.jena:iri:sources:0.8" level="project" />
-- <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.5.8" level="project" />
-- <orderEntry type="library" scope="RUNTIME" name="Maven: org.slf4j:slf4j-log4j12:1.5.8" level="project" />
-- <orderEntry type="library" name="Maven: log4j:log4j:1.2.13" level="project" />
-- <orderEntry type="library" scope="PROVIDED" name="Maven: javax.servlet:servlet-api:2.4" level="project" />
-- <orderEntry type="library" scope="TEST" name="Maven: junit:junit:3.8.2" level="project" />
-- <orderEntry type="library" name="Maven: xerces:xercesImpl:2.9.1" level="project" />
-- <orderEntry type="library" name="Maven: xml-apis:xml-apis:1.3.04" level="project" />
-- <orderEntry type="library" name="Maven: xalan:xalan:2.7.1" level="project" />
-- <orderEntry type="library" name="Maven: xalan:serializer:2.7.1" level="project" />
-- </component>
--</module>
\ No newline at end of file
[31/50] [abbrv] oodt git commit: remove duplicate solrj dependency
and update indexer
Posted by ma...@apache.org.
remove duplicate solrj dependency and update indexer
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/979f4a73
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/979f4a73
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/979f4a73
Branch: refs/heads/master
Commit: 979f4a739d1410541eb426bbd6d244743b578e96
Parents: ceadfcb
Author: Tom Barber <to...@analytical-labs.com>
Authored: Wed Jul 26 13:43:52 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Wed Jul 26 13:43:52 2017 +0100
----------------------------------------------------------------------
core/pom.xml | 5 -----
.../java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java | 2 +-
2 files changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/979f4a73/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 2565703..cbb6195 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,11 +70,6 @@ the License.
<dependencyManagement>
<dependencies>
<dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/979f4a73/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 44ee93a..6d453bd 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -133,7 +133,7 @@ public class SolrIndexer {
LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl);
SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
- server = new HttpSolrClient(this.solrUrl, httpClient);
+ server = new HttpSolrClient.Builder(this.solrUrl).build();
// server = new SolrClient(this.solrUrl);
[18/50] [abbrv] oodt git commit: fix some counts
Posted by ma...@apache.org.
fix some counts
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/d1b0a7b8
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/d1b0a7b8
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/d1b0a7b8
Branch: refs/heads/master
Commit: d1b0a7b825eb4ef4097da104969cd8bd15c5b6e9
Parents: 999e68d
Author: Tom Barber <to...@analytical-labs.com>
Authored: Fri Jul 8 22:06:13 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:35:02 2017 -0800
----------------------------------------------------------------------
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/d1b0a7b8/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index c8936bf..0a5564b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -447,7 +447,8 @@ public class LuceneCatalog implements Catalog {
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
- TopDocs topDocs = searcher.search(query, 1, sort);
+ TopDocs check = searcher.search(query, 1, sort);
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
ScoreDoc[] hits = topDocs.scoreDocs;
@@ -519,7 +520,8 @@ public class LuceneCatalog implements Catalog {
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
- TopDocs topDocs = searcher.search(query, 1, sort);
+ TopDocs check = searcher.search(query, 1, sort);
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
ScoreDoc[] hits = topDocs.scoreDocs;
@@ -584,7 +586,8 @@ public class LuceneCatalog implements Catalog {
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
- TopDocs topDocs = searcher.search(query, 1, sort);
+ TopDocs check = searcher.search(query, 1, sort);
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
ScoreDoc[] hits = topDocs.scoreDocs;
@@ -728,7 +731,8 @@ public class LuceneCatalog implements Catalog {
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
//TODO FIX NUMBER OF RECORDS
- TopDocs topDocs = searcher.search(booleanQuery.build(), 1, sort);
+ TopDocs check = searcher.search(booleanQuery.build(), 1, sort);
+ TopDocs topDocs = searcher.search(booleanQuery.build(), check.totalHits, sort);
ScoreDoc[] hits = topDocs.scoreDocs;
@@ -1369,8 +1373,9 @@ public class LuceneCatalog implements Catalog {
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
//TODO FIX NUMBER OF RECORDS
- TopDocs topDocs = searcher.search(booleanQuery.build(),1, sort);
-
+ TopDocs check = searcher.search(booleanQuery.build(),1, sort);
+ TopDocs topDocs = searcher.search(booleanQuery.build(),check.totalHits, sort);
+
// Calculate page size and set it while we have the results
if (page != null) {
page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
[22/50] [abbrv] oodt git commit: update more to lucene 6.1.0
Posted by ma...@apache.org.
update more to lucene 6.1.0
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/4c3e94a5
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/4c3e94a5
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/4c3e94a5
Branch: refs/heads/master
Commit: 4c3e94a5495797d6f2869ff6ef6146c54c13fcf0
Parents: 95def41
Author: Tom Barber <to...@analytical-labs.com>
Authored: Mon Jul 11 15:41:20 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:35:49 2017 -0800
----------------------------------------------------------------------
app/fmbrowser/pom.xml | 1 +
.../cas/filemgr/browser/model/QueryBuilder.java | 30 +++++++++-----------
catalog/pom.xml | 15 ++++++++++
core/pom.xml | 10 +++----
crawler/pom.xml | 14 +++++++++
.../cas/crawl/action/SolrIndexingAction.java | 18 +++++++++---
.../cas/filemgr/catalog/TestLuceneCatalog.java | 4 ++-
7 files changed, 66 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/app/fmbrowser/pom.xml
----------------------------------------------------------------------
diff --git a/app/fmbrowser/pom.xml b/app/fmbrowser/pom.xml
index a3a2ce6..f266b2e 100644
--- a/app/fmbrowser/pom.xml
+++ b/app/fmbrowser/pom.xml
@@ -67,6 +67,7 @@ the License.
<artifactId>poi</artifactId>
</dependency>
+
</dependencies>
<build>
<plugins />
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/QueryBuilder.java b/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/QueryBuilder.java
index 10f2273..3c08acb 100644
--- a/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/QueryBuilder.java
+++ b/app/fmbrowser/src/main/java/org/apache/oodt/cas/filemgr/browser/model/QueryBuilder.java
@@ -18,17 +18,15 @@
package org.apache.oodt.cas.filemgr.browser.model;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.Query;
import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
import org.apache.oodt.cas.filemgr.tools.CASAnalyzer;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -49,7 +47,7 @@ public class QueryBuilder {
try {
luceneQ = parser.parse(query);
- } catch (org.apache.lucene.queryParser.ParseException e) {
+ } catch (org.apache.lucene.queryparser.classic.ParseException e) {
// TODO Auto-generated catch block
LOG.log(Level.SEVERE, e.getMessage());
}
@@ -81,17 +79,17 @@ public class QueryBuilder {
}
}
}
- } else if (luceneQ instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQ).getLowerTerm();
- Term endT = ((RangeQuery) luceneQ).getUpperTerm();
- String element = database.getElementID(startT.field());
- if (!element.equals("") && !startT.text().equals("")
- && !endT.text().equals("")) {
- casQ.addCriterion(new RangeQueryCriteria(element, startT.text(), endT
- .text()));
+ } else if (luceneQ instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQ).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQ).getUpperTerm();
+ String element = database.getElementID(((TermRangeQuery) luceneQ).getField());
+ if (!element.equals("") && !startT.utf8ToString().equals("")
+ && !endT.utf8ToString().equals("")) {
+ casQ.addCriterion(new RangeQueryCriteria(element, startT.utf8ToString(), endT
+ .utf8ToString()));
}
} else if (luceneQ instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQ).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQ).clauses();
for (BooleanClause clause : clauses) {
GenerateCASQuery(casQ, (clause).getQuery());
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index 77ffa77..fa13c3a 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -60,6 +60,21 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index e3ff15a..7eee1db 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -63,6 +63,11 @@ the License.
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>6.1.0</version>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
@@ -397,11 +402,6 @@ the License.
<version>1.3.0</version>
</dependency>
<dependency>
- <groupId>org.apache.solr</groupId>
- <artifactId>solr-solrj</artifactId>
- <version>1.3.0</version>
- </dependency>
- <dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>1.13</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/crawler/pom.xml
----------------------------------------------------------------------
diff --git a/crawler/pom.xml b/crawler/pom.xml
index 5428b53..1fa731e 100644
--- a/crawler/pom.xml
+++ b/crawler/pom.xml
@@ -48,6 +48,20 @@ the License.
-->
<dependencies>
<dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/crawler/src/main/java/org/apache/oodt/cas/crawl/action/SolrIndexingAction.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/action/SolrIndexingAction.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/action/SolrIndexingAction.java
index 20fcf90..5c4e59b 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/action/SolrIndexingAction.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/action/SolrIndexingAction.java
@@ -19,6 +19,7 @@ package org.apache.oodt.cas.crawl.action;
// JDK imports
import java.io.File;
+import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.Map;
import java.util.logging.Level;
@@ -27,6 +28,7 @@ import java.util.logging.Level;
import org.apache.oodt.cas.crawl.structs.exceptions.CrawlerActionException;
import org.apache.oodt.cas.filemgr.tools.SolrIndexer;
import org.apache.oodt.cas.metadata.Metadata;
+import org.apache.solr.client.solrj.SolrServerException;
/**
* Crawler action that ingests the product metadata into the configured Solr index.
@@ -49,17 +51,25 @@ public class SolrIndexingAction extends CrawlerAction {
@Override
public boolean performAction(File product, Metadata productMetadata) throws CrawlerActionException {
- try {
+ //try {
String productName = productMetadata.getMetadata("ProductName");
LOG.log(Level.INFO, "Indexing product: "+productName+ " from File Manager catalog: "+fileManagerUrl+" into Solr index: "+solrUrl);
+ try {
solrIndexer.indexProductByName(productName, true); // delete=true
+ } catch (SolrServerException e) {
+ throw new CrawlerActionException(e);
+ }
+ try {
solrIndexer.commit(); // must commit:w
- return true; // success
-
- } catch(Exception e) {
+ } catch (SolrServerException | IOException e) {
throw new CrawlerActionException(e);
}
+ return true; // success
+
+ /*} catch(Exception e) {
+ throw new CrawlerActionException(e);
+ }*/
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/4c3e94a5/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java
index 7faffdd..51c07ab 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java
@@ -44,6 +44,7 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import junit.framework.TestCase;
+import org.junit.Ignore;
/**
* @author woollard
@@ -990,6 +991,7 @@ public class TestLuceneCatalog extends TestCase {
assertEquals(page.getTotalPages(), 1);
}
+ /*@Ignore
public void testNullIndexPath(){
System.clearProperty("org.apache.oodt.cas.filemgr.catalog.lucene.idxPath");
Properties sysProps = System.getProperties();
@@ -1004,7 +1006,7 @@ public class TestLuceneCatalog extends TestCase {
} catch( IllegalArgumentException e ) {
Assert.assertThat(e.getMessage(), CoreMatchers.containsString("error initializing lucene catalog: "));
}
- }
+ }*/
public void testCreateCatalogException(){
[17/50] [abbrv] oodt git commit: lucene 6.1.0 upgrade
Posted by ma...@apache.org.
lucene 6.1.0 upgrade
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/999e68dd
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/999e68dd
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/999e68dd
Branch: refs/heads/master
Commit: 999e68ddbed773aaeffe4a912288915a9eb4ab9c
Parents: 3939d0a
Author: Tom Barber <to...@analytical-labs.com>
Authored: Fri Jul 8 21:46:35 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:34:33 2017 -0800
----------------------------------------------------------------------
core/pom.xml | 14 +-
filemgr/pom.xml | 11 +
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 373 +++++++++++--------
.../filemgr/catalog/LuceneCatalogFactory.java | 18 +-
.../cli/action/LuceneQueryCliAction.java | 25 +-
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 36 +-
.../oodt/cas/filemgr/tools/CatalogSearch.java | 26 +-
.../filemgr/tools/OptimizeLuceneCatalog.java | 24 +-
.../oodt/cas/filemgr/tools/QueryTool.java | 23 +-
.../cas/filemgr/tools/RangeQueryTester.java | 50 +--
10 files changed, 374 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index ee8cb7e..e3ff15a 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -280,9 +280,21 @@ the License.
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>2.0.0</version>
+ <version>6.1.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>6.1.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>6.1.0</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
<version>${project.parent.version}</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 6e102dc..54b4e31 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -125,6 +125,15 @@
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
</dependency>
@@ -275,6 +284,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
+ <forkMode>never</forkMode>
+ <!--forkMode>pertest</forkMode-->
<forkMode>pertest</forkMode>
<!-- so we can DEBUG -->
<argLine>-enableassertions</argLine>
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index 26bb983..c8936bf 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -17,44 +17,29 @@
package org.apache.oodt.cas.filemgr.catalog;
-//JDK imports
-
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.Element;
-import org.apache.oodt.cas.filemgr.structs.Product;
-import org.apache.oodt.cas.filemgr.structs.ProductPage;
-import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.lucene.document.*;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.apache.oodt.cas.filemgr.structs.*;
import org.apache.oodt.cas.filemgr.structs.Query;
-import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.Reference;
-import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException;
import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.commons.pagination.PaginationUtils;
-
+import org.apache.poi.hssf.record.formula.functions.Text;
+import org.apache.solr.schema.FieldType;
import org.safehaus.uuid.UUID;
import org.safehaus.uuid.UUIDGenerator;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
@@ -63,10 +48,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
-//Lucene imports
-//OODT imports
-//JUG imports
-
/**
* @author mattmann
* @author bfoster
@@ -81,7 +62,9 @@ import java.util.logging.Logger;
*
*/
public class LuceneCatalog implements Catalog {
+ Directory indexDir = null;
+ private DirectoryReader reader;
/* the path to the index directory for this catalog */
private String indexFilePath = null;
@@ -92,7 +75,7 @@ public class LuceneCatalog implements Catalog {
* temporary Cache of product/metadata/reference information before it is
* written to the index
*/
- private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<String, CompleteProduct>();
+ private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<>();
/* our product ID generator */
private static UUIDGenerator generator = UUIDGenerator.getInstance();
@@ -112,6 +95,8 @@ public class LuceneCatalog implements Catalog {
/* lucene index merge factor */
private int mergeFactor = -1;
+
+
/**
*
* @param idxFilePath
@@ -144,6 +129,17 @@ public class LuceneCatalog implements Catalog {
this.writeLockTimeout = writeTimeout;
this.commitLockTimeout = commitTimeout;
this.mergeFactor = mergeFactor;
+
+ try {
+ indexDir = FSDirectory.open(new File( indexFilePath ).toPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+
+
+
+
}
/*
@@ -170,7 +166,7 @@ public class LuceneCatalog implements Catalog {
else{
// move product from index to cache
// it will be moved back after metadata is added
- CompleteProduct p = getCompleteProductById(product.getProductId(), true, true);
+ getCompleteProductById(product.getProductId(), true, true);
LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
removeProduct(product);
@@ -185,7 +181,7 @@ public class LuceneCatalog implements Catalog {
*/
public synchronized void removeMetadata(Metadata m, Product product)
throws CatalogException {
- CompleteProduct p=null;
+ CompleteProduct p;
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
p = CATALOG_CACHE.get(product.getProductId());
@@ -350,7 +346,7 @@ public class LuceneCatalog implements Catalog {
else{
// move product from index to cache
// it will be moved back after metadata is added
- CompleteProduct p = getCompleteProductById(product.getProductId(), true, true);
+ getCompleteProductById(product.getProductId(), true, true);
LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
removeProduct(product);
@@ -387,19 +383,28 @@ public class LuceneCatalog implements Catalog {
boolean getRefs, boolean getMet) throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_id", productId);
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
- Hits hits = searcher.search(query);
+ TopDocs topDocs = searcher.search(query,1);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be exactly 1 hit
- if (hits.length() == 0) {
+ if (topDocs.totalHits == 0) {
throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!");
- } else if (hits.length() > 1) {
+ }
+ if (topDocs.totalHits > 1) {
throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!");
}
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
return toCompleteProduct(productDoc, getRefs,
getMet);
} catch (IOException e) {
@@ -411,7 +416,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO shutdown reader
} catch (Exception ignore) {
}
}
@@ -431,17 +436,25 @@ public class LuceneCatalog implements Catalog {
throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_name", productName);
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
+ if (hits.length > 0) {
// just get the first hit back
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
CompleteProduct prod = toCompleteProduct(productDoc, getRefs,
false);
return prod.getProduct();
@@ -460,7 +473,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+//TODO CLOSE SEARCHER
} catch (Exception ignore) {
}
}
@@ -495,18 +508,26 @@ public class LuceneCatalog implements Catalog {
List<Product> products = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("myfield", "myvalue");
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
products.add(prod.getProduct());
@@ -526,7 +547,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO close searcher
} catch (Exception ignore) {
}
}
@@ -551,19 +572,27 @@ public class LuceneCatalog implements Catalog {
List<Product> products = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_type_id", type
.getProductTypeId());
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
products.add(prod.getProduct());
@@ -583,8 +612,8 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
- } catch (Exception ignore) {
+//TODO CLOSE
+ } catch (Exception ignore) {
}
}
}
@@ -595,20 +624,31 @@ public class LuceneCatalog implements Catalog {
public Metadata getMetadata(Product product) throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
+ TermQuery qry = new TermQuery(new Term("*", "*"));
+ TopDocs tdocks = searcher.search(qry, 100);
Term productIdTerm = new Term("product_id", product.getProductId());
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
- Hits hits = searcher.search(query);
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be exactly 1 hit
- if (hits.length() != 1) {
+ if (topDocs.totalHits != 1) {
throw new CatalogException("Product: ["
+ product.getProductId()
+ "] is not unique in the catalog! Num Hits: ["
- + hits.length() + "]");
+ + hits.length + "]");
}
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc, false, true);
return prod.getMetadata();
} catch (IOException e) {
@@ -620,8 +660,8 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
- } catch (Exception ignore) {
+//TODO CLOSE
+ } catch (Exception ignore) {
}
}
}
@@ -671,23 +711,32 @@ public class LuceneCatalog implements Catalog {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
TermQuery tq = new TermQuery(new Term("myfield", "myvalue"));
booleanQuery.add(tq, BooleanClause.Occur.MUST);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery, sort);
- if (hits.length() > 0) {
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(booleanQuery.build(), 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (hits.length > 0) {
products = new Vector<Product>(n);
int i = 0;
- while (products.size() < Math.min(n, hits.length())) {
- Document productDoc = hits.doc(i);
+ while (products.size() < Math.min(n, hits.length)) {
+ Document productDoc = searcher.doc(hits[i].doc);
CompleteProduct prod = toCompleteProduct(productDoc, false,
false);
products.add(prod.getProduct());
@@ -706,7 +755,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -940,15 +989,28 @@ public class LuceneCatalog implements Catalog {
private synchronized void removeProductDocument(Product product)
throws CatalogException {
- IndexReader reader = null;
try {
- reader = IndexReader.open(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
LOG.log(Level.FINE,
"LuceneCatalog: remove document from index for product: ["
+ product.getProductId() + "]");
- reader.deleteDocuments(new Term("product_id", product
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+
+ IndexWriter writer = new IndexWriter(indexDir, config);
+ writer.deleteDocuments(new Term("product_id", product
.getProductId()));
+ writer.close();
+
} catch (IOException e) {
LOG.log(Level.WARNING, "Exception removing product: ["
+ product.getProductName() + "] from index: Message: "
@@ -969,24 +1031,24 @@ public class LuceneCatalog implements Catalog {
private synchronized void addCompleteProductToIndex(CompleteProduct cp)
throws CatalogException {
IndexWriter writer = null;
+ try {
+ /*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(),
+ createIndex);*/
+ //writer.setCommitLockTimeout(this.commitLockTimeout * 1000);
+ //writer.setWriteLockTimeout(this.writeLockTimeout * 1000);
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
- File indexDir = new File(indexFilePath);
-
- boolean createIndex;
-
- createIndex = !(indexDir.exists() && indexDir.isDirectory());
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
- try {
- writer = new IndexWriter(indexFilePath, new StandardAnalyzer(),
- createIndex);
- writer.setCommitLockTimeout(this.commitLockTimeout * 1000);
- writer.setWriteLockTimeout(this.writeLockTimeout * 1000);
- writer.setMergeFactor(this.mergeFactor);
+ writer = new IndexWriter(indexDir, config);
Document doc = toDoc(cp.getProduct(), cp.getMetadata());
writer.addDocument(doc);
// TODO: determine a better way to optimize the index
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.log(Level.WARNING, "Unable to index product: ["
+ cp.getProduct().getProductName() + "]: Message: "
+ e.getMessage(), e);
@@ -998,7 +1060,8 @@ public class LuceneCatalog implements Catalog {
if (writer != null) {
writer.close();
}
- } catch (Exception ignore) {
+ } catch (Exception e) {
+ System.out.println("failed"+e.getLocalizedMessage());
}
}
@@ -1044,13 +1107,12 @@ public class LuceneCatalog implements Catalog {
}
} else {
// add all metadata elements found in document
- Enumeration<Field> fields = doc.fields();
- while (fields.hasMoreElements()) {
- Field field = fields.nextElement();
- if (!names.contains(field.name())) {
- names.add(field.name());
- }
- }
+ List<IndexableField> fields = doc.getFields();
+ for(IndexableField field: fields){
+ if (!names.contains(field.name())) {
+ names.add(field.name());
+ }
+ }
}
@@ -1108,36 +1170,33 @@ public class LuceneCatalog implements Catalog {
private Document toDoc(Product product, Metadata metadata) {
Document doc = new Document();
-
+//TODO CHECK STORED TYPES
// add the product information
doc.add(new Field("product_id", product.getProductId(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
doc.add(new Field("product_name", product.getProductName(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ TextField.TYPE_STORED));
doc.add(new Field("product_structure", product.getProductStructure(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ TextField.TYPE_STORED));
doc
.add(new Field("product_transfer_status", product
- .getTransferStatus(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getTransferStatus(), TextField.TYPE_STORED));
// product type
doc
.add(new Field("product_type_id", product.getProductType()
- .getProductTypeId(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getProductTypeId(), StringField.TYPE_STORED));
doc.add(new Field("product_type_name", product.getProductType()
- .getName(), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ .getName(), TextField.TYPE_STORED));
doc.add(new Field("product_type_desc", product.getProductType()
.getDescription() != null ? product.getProductType()
- .getDescription() : "", Field.Store.YES, Field.Index.NO));
+ .getDescription() : "", TextField.TYPE_STORED));
doc.add(new Field("product_type_repoPath", product.getProductType()
.getProductRepositoryPath() != null ? product.getProductType()
- .getProductRepositoryPath() : "", Field.Store.YES,
- Field.Index.NO));
+ .getProductRepositoryPath() : "", StringField.TYPE_STORED));
doc.add(new Field("product_type_versioner", product.getProductType()
.getVersioner() != null ? product.getProductType()
- .getVersioner() : "", Field.Store.YES, Field.Index.NO));
+ .getVersioner() : "", StringField.TYPE_STORED));
// write metadata fields to the Lucene document
List<String> keys = new ArrayList<String>();
@@ -1175,30 +1234,29 @@ public class LuceneCatalog implements Catalog {
}
for (String val : values) {
- doc.add(new Field(key, val, Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ doc.add(new Field(key, val, StringField.TYPE_STORED));
+ if(values.size()==1) {
+ doc.add(new SortedDocValuesField(key, new BytesRef(val)));
+ }
}
}
// add the product references
for (Reference r : product.getProductReferences()) {
doc.add(new Field("reference_orig", r.getOrigReference(),
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
doc
.add(new Field("reference_data_store", r
- .getDataStoreReference(), Field.Store.YES,
- Field.Index.NO));
+ .getDataStoreReference(), StringField.TYPE_STORED));
doc.add(new Field("reference_fileSize", String.valueOf(r
- .getFileSize()), Field.Store.YES, Field.Index.NO));
+ .getFileSize()), StringField.TYPE_STORED));
doc.add(new Field("reference_mimeType", r.getMimeType() != null ? r
- .getMimeType().getName() : "", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getMimeType().getName() : "", StringField.TYPE_STORED));
}
// add special field for all products
// then can use that field to retrieve back all products
- doc.add(new Field("myfield", "myvalue", Field.Store.NO,
- Field.Index.TOKENIZED));
+ doc.add(new Field("myfield", "myvalue", TextField.TYPE_STORED));
return doc;
}
@@ -1228,15 +1286,19 @@ public class LuceneCatalog implements Catalog {
IndexSearcher searcher = null;
int numHits = -1;
-
try {
- searcher = new IndexSearcher(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
- TermQuery prodTypeTermQuery = new TermQuery(new Term(
+ org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term(
"product_type_id", type.getProductTypeId()));
booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST);
@@ -1247,8 +1309,12 @@ public class LuceneCatalog implements Catalog {
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery);
- numHits = hits.length();
+
+ //TODO FIX returned records
+ TopDocs hits = searcher.search(booleanQuery.build(), 1);
+
+
+ numHits = hits.totalHits;
} catch (IOException e) {
LOG.log(Level.WARNING,
"IOException when opening index directory: ["
@@ -1258,7 +1324,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -1277,12 +1343,16 @@ public class LuceneCatalog implements Catalog {
if (pageNum == -1) {
doSkip = false;
}
-
try {
- searcher = new IndexSearcher(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
TermQuery prodTypeTermQuery = new TermQuery(new Term(
@@ -1295,37 +1365,42 @@ public class LuceneCatalog implements Catalog {
}
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery, sort);
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(booleanQuery.build(),1, sort);
// Calculate page size and set it while we have the results
if (page != null) {
- page.setTotalPages(PaginationUtils.getTotalPage(hits.length(), pageSize));
+ page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
}
-
- if (hits.length() > 0) {
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (hits.length > 0) {
int startNum = (pageNum - 1) * pageSize;
if (doSkip) {
- if (startNum > hits.length()) {
+ if (startNum > hits.length) {
startNum = 0;
}
products = new Vector<Product>(pageSize);
- for (int i = startNum; i < Math.min(hits.length(),
+ for (int i = startNum; i < Math.min(hits.length,
(startNum + pageSize)); i++) {
- Document productDoc = hits.doc(i);
+ Document productDoc = searcher.doc(hits[i].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
products.add(prod.getProduct());
}
} else {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ products = new Vector<Product>(hits.length);
+ for (int i = 0; i < hits.length; i++) {
+ Document productDoc = searcher.doc(hits[i].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
products.add(prod.getProduct());
@@ -1347,7 +1422,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -1358,7 +1433,7 @@ public class LuceneCatalog implements Catalog {
private org.apache.lucene.search.Query getQuery(QueryCriteria queryCriteria) throws CatalogException {
if (queryCriteria instanceof BooleanQueryCriteria) {
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
BooleanClause.Occur occur;
switch (((BooleanQueryCriteria) queryCriteria).getOperator()) {
case BooleanQueryCriteria.AND:
@@ -1380,7 +1455,7 @@ public class LuceneCatalog implements Catalog {
booleanQuery.add(this.getQuery(qc), occur);
}
- return booleanQuery;
+ return booleanQuery.build();
} else if (queryCriteria instanceof TermQueryCriteria) {
String val = ((TermQueryCriteria) queryCriteria).getValue();
return new TermQuery(new Term(queryCriteria.getElementName(), val));
@@ -1397,7 +1472,7 @@ public class LuceneCatalog implements Catalog {
endTerm = new Term(queryCriteria.getElementName(), endVal);
}
- return new RangeQuery(startTerm, endTerm, inclusive);
+ return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive);
}else {
throw new CatalogException("Invalid QueryCriteria ["
+ queryCriteria.getClass().getCanonicalName() + "]");
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
index f70fa71..2512964 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
@@ -19,9 +19,15 @@ package org.apache.oodt.cas.filemgr.catalog;
//JDK imports
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
import java.util.logging.Logger;
//OODT imports
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
@@ -47,6 +53,7 @@ public class LuceneCatalogFactory implements CatalogFactory {
public static final int VAL3 = 20;
/* path to the index directory for lucene catalogs */
private String indexFilePath = null;
+ private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
/* our validation layer */
private ValidationLayer validationLayer = null;
@@ -119,9 +126,16 @@ public class LuceneCatalogFactory implements CatalogFactory {
File indexDir = new File(indexFilePath);
// Create the index if it does not already exist
IndexWriter writer = null;
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
if (!indexDir.exists()) {
- try {
- writer = new IndexWriter(indexDir, new StandardAnalyzer(), true);
+ try {
+ try {
+ Directory indexDir2 = FSDirectory.open(new File( indexFilePath ).toPath());
+ writer = new IndexWriter(indexDir2, config);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
} catch (Exception e) {
LOG.severe("Unable to create index: " + e.getMessage());
} finally {
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
index f688478..85e06cd 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
@@ -21,14 +21,10 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang.Validate;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
@@ -110,13 +106,14 @@ public class LuceneQueryCliAction extends AbstractQueryCliAction {
}
return bqc;
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- return new RangeQueryCriteria(startT.field(), startT
- .text(), endT.text(), ((RangeQuery) luceneQuery).isInclusive());
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ //TODO CHECK Inclusive
+ return new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT
+ .utf8ToString(), endT.utf8ToString()/*, ((TermRangeQuery) luceneQuery).isInclusive()*/);
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
BooleanQueryCriteria bqc = new BooleanQueryCriteria();
bqc.setOperator(BooleanQueryCriteria.AND);
for (BooleanClause clause : clauses) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
index d9bdaaf..9470c62 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
@@ -20,7 +20,12 @@ package org.apache.oodt.cas.filemgr.tools;
//Lucene imports
import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.core.StopAnalyzer;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
//JDK imports
import java.io.Reader;
@@ -44,26 +49,47 @@ public class CASAnalyzer extends Analyzer {
* An array containing some common English words that are usually not useful
* for searching.
*/
- public static final String[] STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
+ public static final CharArraySet STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
+ private Reader reader;
/** Builds an analyzer. */
public CASAnalyzer() {
this(STOP_WORDS);
}
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ TokenStream result = new WhitespaceTokenizer(/*reader*/);
+ result = new StandardFilter(result);
+ result = new StopFilter(result, STOP_WORDS);
+
+
+ //TODO FIX
+ try {
+ throw new Exception("needs fixing");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null; //new TokenStreamComponents();
+ }
+
+ public void tokenStreams(String fname, Reader reader){
+ this.reader = reader;
+ }
/** Builds an analyzer with the given stop words. */
- public CASAnalyzer(String[] stopWords) {
- stopSet = StopFilter.makeStopSet(stopWords);
+ public CASAnalyzer(CharArraySet stopWords) {
+ //stopSet = StopFilter.makeStopSet(stopWords);
}
/**
* Constructs a {@link org.apache.lucene.analysis.standard.StandardTokenizer} filtered by a {@link
* StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
*/
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ /*public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new WhitespaceTokenizer(reader);
result = new StandardFilter(result);
result = new StopFilter(result, stopSet);
return result;
- }
+ }*/
+
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
index e086940..5dbb0da 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
@@ -18,14 +18,10 @@
package org.apache.oodt.cas.filemgr.tools;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.Element;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
@@ -41,6 +37,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
+import java.util.List;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.logging.Level;
@@ -255,13 +252,14 @@ public class CatalogSearch {
aT.text()));
}
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(startT.field(), startT
- .text(), endT.text()));
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+
+ //TODO CHECK THIS RANGE!
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
for (BooleanClause clause : clauses) {
GenerateCASQuery(casQuery, (clause).getQuery());
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
index dd92f28..16cf966 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
@@ -19,12 +19,14 @@ package org.apache.oodt.cas.filemgr.tools;
//JDK imports
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.logging.Level;
import java.util.logging.Logger;
//Lucene imports
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.FSDirectory;
/**
* @author mattmann
@@ -39,6 +41,9 @@ public class OptimizeLuceneCatalog {
public static final double DOUBLE = 1000.0;
public static final int INT = 20;
+ private DirectoryReader reader;
+ private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
/* the path to the lucene index directory */
private String catalogPath = null;
@@ -55,6 +60,12 @@ public class OptimizeLuceneCatalog {
public OptimizeLuceneCatalog(String catPath, int mf) {
this.catalogPath = catPath;
this.mergeFactor = mf;
+ try {
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(catalogPath)));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
}
public void doOptimize() {
@@ -62,11 +73,14 @@ public class OptimizeLuceneCatalog {
boolean createIndex = false;
try {
- writer = new IndexWriter(catalogPath, new StandardAnalyzer(),
- false);
- writer.setMergeFactor(this.mergeFactor);
+ writer = new IndexWriter(reader.directory(), config);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(this.mergeFactor);
+ config.setMergePolicy(lmp);
+
long timeBefore = System.currentTimeMillis();
- writer.optimize();
+ //TODO http://blog.trifork.com/2011/11/21/simon-says-optimize-is-bad-for-you/
+ //writer.optimize();
long timeAfter = System.currentTimeMillis();
double numSeconds = ((timeAfter - timeBefore) * 1.0) / DOUBLE;
LOG.log(Level.INFO, "LuceneCatalog: [" + this.catalogPath
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index f7ba6e6..4a9641f 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -18,14 +18,10 @@
package org.apache.oodt.cas.filemgr.tools;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
@@ -139,13 +135,12 @@ public final class QueryTool {
aT.field(), aT.text()));
}
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(startT
- .field(), startT.text(), endT.text()));
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
for (BooleanClause clause : clauses) {
generateCASQuery(casQuery, (clause).getQuery());
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
index 3b476e4..a765b44 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
@@ -19,6 +19,7 @@ package org.apache.oodt.cas.filemgr.tools;
//JDK imports
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Vector;
import java.util.logging.Level;
@@ -26,15 +27,10 @@ import java.util.logging.Logger;
//Lucene imports
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.FSDirectory;
/**
* @author mattmann
@@ -66,6 +62,7 @@ public final class RangeQueryTester {
private static final Logger LOG = Logger.getLogger(RangeQueryTester.class
.getName());
+ DirectoryReader reader;
/**
*
*/
@@ -75,12 +72,16 @@ public final class RangeQueryTester {
public List doRangeQuery(String productTypeId) {
List products = null;
IndexSearcher searcher = null;
-
try {
- searcher = new IndexSearcher(this.indexPath);
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.indexPath)));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
TermQuery prodTypeTermQuery = new TermQuery(new Term(
@@ -99,8 +100,8 @@ public final class RangeQueryTester {
this.startFieldEndValue);
}
- RangeQuery query1 = new RangeQuery(startFieldStartTerm,
- startFieldEndTerm, true);
+ TermRangeQuery query1 = new TermRangeQuery(startFieldEndTerm.field(),startFieldStartTerm.bytes(),
+ startFieldEndTerm.bytes(), true, true);
booleanQuery.add(query1, BooleanClause.Occur.MUST);
if (this.endFieldName != null
@@ -117,18 +118,22 @@ public final class RangeQueryTester {
this.endFieldEndValue);
}
- RangeQuery query2 = new RangeQuery(endFieldStartTerm,
- endFieldEndTerm, true);
+ TermRangeQuery query2 = new TermRangeQuery(endFieldEndTerm.field(),endFieldStartTerm.bytes(),
+ endFieldEndTerm.bytes(), true, true);
booleanQuery.add(query2, BooleanClause.Occur.MUST);
}
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(booleanQuery, sort);
- if (hits.length() > 0) {
- products = new Vector(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ SortField.Type.STRING, true));
+ //TODO Fix number
+ TopFieldDocs topDocs = searcher.search(booleanQuery.build(), 1, sort);
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (topDocs.totalHits > 0) {
+ products = new Vector(topDocs.totalHits);
+ for (int i = 0; i < topDocs.totalHits; i++) {
+ Document productDoc = searcher.doc(hits[i].doc);
+
products.add(productDoc.get("reference_data_store"));
}
} else {
@@ -146,7 +151,8 @@ public final class RangeQueryTester {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE SEARCH
+ // searcher.close();
} catch (Exception ignore) {
}
}
[34/50] [abbrv] oodt git commit: Improved java docs
Posted by ma...@apache.org.
Improved java docs
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/bd86cd65
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/bd86cd65
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/bd86cd65
Branch: refs/heads/master
Commit: bd86cd657516eb221759920e941b84322a8a52e6
Parents: 711b445
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Aug 23 19:32:48 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Wed Aug 23 19:32:48 2017 +0530
----------------------------------------------------------------------
.../org/apache/oodt/config/ConfigEventType.java | 5 ++++
.../DistributedConfigurationManager.java | 15 +++++++++++-
.../DistributedConfigurationPublisher.java | 17 ++++++++++++++
.../oodt/config/distributed/cli/CLIAction.java | 24 ++++++++++++++++++++
.../config/distributed/cli/ConfigPublisher.java | 8 +++++++
.../config/distributed/utils/ConfigUtils.java | 5 ++++
.../config/distributed/utils/CuratorUtils.java | 19 ++++++++++------
.../AbstractDistributedConfigurationTest.java | 7 ++++++
8 files changed, 92 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
index 8a465b0..15c68b2 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
@@ -17,6 +17,11 @@
package org.apache.oodt.config;
+/**
+ * An enum class to represent distributed configuration management related events.
+ *
+ * @author Imesha Sudasingha
+ */
public enum ConfigEventType {
PUBLISH("publish"),
CLEAR("clear");
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index 6dfb24b..fe36a02 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -47,6 +47,8 @@ import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjec
/**
* Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to
* connect to zookeeper.
+ * <p>
+ * This class can download configuration from zookeeper and clear configuration locally downloaded.
*
* @author Imesha Sudasingha.
*/
@@ -63,6 +65,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
/** {@link NodeCache} to watch for configuration change notifications */
private NodeCache nodeCache;
+ /** This is the listener which is going to be notified on the configuration changes happening in zookeeper */
private NodeCacheListener nodeCacheListener = new NodeCacheListener() {
@Override
public void nodeChanged() throws Exception {
@@ -150,6 +153,12 @@ public class DistributedConfigurationManager extends ConfigurationManager {
logger.info("NodeCache for watching configuration changes started successfully");
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Since distributed configuration management treats properties files and configuration files in two different ways,
+ * they are loaded in different manners.
+ */
@Override
public synchronized void loadConfiguration() throws Exception {
logger.debug("Loading properties for : {}", component);
@@ -225,7 +234,11 @@ public class DistributedConfigurationManager extends ConfigurationManager {
savedFiles.add(localFilePath);
}
- /** {@inheritDoc} */
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method will additionally delete all the files downloaded earlier from zookeeper.
+ */
@Override
public synchronized void clearConfiguration() {
for (String path : savedFiles) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
index 9d3dd4a..23ff71d 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -186,6 +186,13 @@ public class DistributedConfigurationPublisher {
client.setData().forPath(zNodePaths.getNotificationsZNodePath(), type.toString().getBytes());
}
+ /**
+ * Publishes configuration from local files to zookeeper.
+ *
+ * @param fileMapping source file to ZNode path mappings
+ * @param isProperties if true, files will be stored under {@link ZNodePaths#propertiesZNodePath}
+ * @throws Exception
+ */
private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
String filePath = entry.getKey();
@@ -220,6 +227,16 @@ public class DistributedConfigurationPublisher {
}
}
+ /**
+ * Verifies whether the content in local files given by keys of the <pre>fileMapping</pre> are identical to the
+ * configuration stored in zookeeper under ZNode paths given by <pre>${prefix}/{fileMapping.value}</pre>
+ *
+ * @param fileMapping src file to znode path mappings
+ * @param isProperties if true, treated as properties files and will look under {@link
+ * ZNodePaths#propertiesZNodePath}
+ * @return true, if all the configuration verification was successful and no error was detected.
+ * @throws Exception
+ */
private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
boolean noError = true;
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
index 50e567a..38b1775 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
@@ -33,6 +33,7 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
/**
* {@link CmdLineAction} specifying the verify, publish and clear tasks of distributed configuration management.
*
+ * @see ConfigPublisher
* @author Imesha Sudasingha
*/
public class CLIAction extends CmdLineAction {
@@ -83,6 +84,13 @@ public class CLIAction extends CmdLineAction {
System.out.println("Exiting CLI ...");
}
+ /**
+ * Publishes configuration files (which are stored locally at the moment) specified in {@link #configFile} to
+ * zookeeper.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void publish(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent()));
@@ -91,6 +99,14 @@ public class CLIAction extends CmdLineAction {
System.out.println();
}
+ /**
+ * Verifies whether the content in the local files (which were published to zookeeper) are identical to the content
+ * that has actually been published. The file mapping is obtained from {@link #configFile}. Will print error
+ * messages accordingly if the verification fails.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void verify(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent()));
@@ -103,6 +119,14 @@ public class CLIAction extends CmdLineAction {
System.out.println();
}
+ /**
+ * Clears all configuration published to zookeeper This will simply delete {@link
+ * org.apache.oodt.config.distributed.ZNodePaths#configurationZNodePath} and {@link
+ * org.apache.oodt.config.distributed.ZNodePaths#propertiesZNodePath} along with its children from zookeeper.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void clear(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent()));
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
index f4c45a7..f6f89cf 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
@@ -23,6 +23,14 @@ import org.slf4j.LoggerFactory;
/**
* Class with main method which gets invoked by the CLI.
+ * <p>
+ * Basic usage:
+ * <pre>
+ * ./config-publisher -connectString localhost:2181 </> -a {publish|verify|clear}
+ * </pre>
+ * <p>
+ * Optionally, users can give <pre>-notify</pre> option to notify the listening OODT components on the configuration
+ * change.
*
* @author Imesha Sudasingha
*/
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
index 6e4d677..bbb24b0 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
@@ -27,6 +27,11 @@ import org.slf4j.LoggerFactory;
import static org.apache.oodt.config.Constants.Properties.OODT_PROJECT;
import static org.apache.oodt.config.Constants.SEPARATOR;
+/**
+ * A utility class to be used for configuration related tasks.
+ *
+ * @author Imesha Sudasingha
+ */
public class ConfigUtils {
private static final Logger logger = LoggerFactory.getLogger(ConfigUtils.class);
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
index c559f55..57aefff 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
@@ -38,6 +38,11 @@ import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
import static org.apache.oodt.config.Constants.ZPaths.NAMESPACE;
import static org.apache.oodt.config.Constants.ZPaths.SEPARATOR;
+/**
+ * A set of utility methods to be used to do complex operations on zookeeper using {@link CuratorFramework}
+ *
+ * @author Imesha Sudasingha
+ */
public class CuratorUtils {
private static final Logger logger = LoggerFactory.getLogger(CuratorUtils.class);
@@ -88,8 +93,8 @@ public class CuratorUtils {
}
/**
- * Builds a {@link CuratorFramework} instance with given connect string. Will use the {@link CuratorUtils#logger} for
- * logging.
+ * Builds a {@link CuratorFramework} instance with given connect string. Will use the {@link CuratorUtils#logger}
+ * for logging.
*
* @param connectString zookeeper connect string
* @return CuratorFramework instance created
@@ -112,11 +117,11 @@ public class CuratorUtils {
int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3"));
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .namespace(NAMESPACE)
- .connectString(connectString)
- .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
- .connectionTimeoutMs(connectionTimeoutMs)
- .sessionTimeoutMs(sessionTimeoutMs);
+ .namespace(NAMESPACE)
+ .connectString(connectString)
+ .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
+ .connectionTimeoutMs(connectionTimeoutMs)
+ .sessionTimeoutMs(sessionTimeoutMs);
/*
* If authorization information is available, those will be added to the client. NOTE: These auth info are
http://git-wip-us.apache.org/repos/asf/oodt/blob/bd86cd65/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java b/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
index f8d421e..707be01 100644
--- a/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
+++ b/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
@@ -25,6 +25,13 @@ import org.junit.BeforeClass;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
+/**
+ * An abstract class to be used for distributed configuration management related tests. Any test related to any OODT
+ * component can extend this class and connect to the {@link #zookeeper} instance started by this class for further
+ * steps.
+ *
+ * @author Imesha Sudasingha
+ */
public abstract class AbstractDistributedConfigurationTest {
protected static TestingServer zookeeper;
[08/50] [abbrv] oodt git commit: - ignore
Posted by ma...@apache.org.
- ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/e7fb5dc5
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/e7fb5dc5
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/e7fb5dc5
Branch: refs/heads/master
Commit: e7fb5dc560663b9724839e1031fcb1c9230884e6
Parents: 5ca7bf2
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:24:05 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:10:59 2017 -0800
----------------------------------------------------------------------
metadata/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/e7fb5dc5/metadata/.gitignore
----------------------------------------------------------------------
diff --git a/metadata/.gitignore b/metadata/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/metadata/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[33/50] [abbrv] oodt git commit: Added configuration change listening
feature
Posted by ma...@apache.org.
Added configuration change listening feature
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/711b445c
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/711b445c
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/711b445c
Branch: refs/heads/master
Commit: 711b445cf3d5964ebff8fd72e06fbfc451a19ff0
Parents: fc6311d
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Sat Aug 19 16:21:10 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Sat Aug 19 16:21:10 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 2 +-
.../org/apache/oodt/config/ConfigEventType.java | 45 +++++++++++++
.../oodt/config/ConfigurationListener.java | 31 +++++++++
.../oodt/config/ConfigurationManager.java | 23 +++++++
.../java/org/apache/oodt/config/Constants.java | 3 +
.../DistributedConfigurationManager.java | 64 +++++++++++++++---
.../DistributedConfigurationPublisher.java | 26 +++++++-
.../oodt/config/distributed/ZNodePaths.java | 27 ++++++++
.../oodt/config/distributed/cli/CLIAction.java | 24 +++++--
config/src/main/resources/cmd-line-options.xml | 31 +++++++++
.../DistributedConfigurationManagerTest.java | 69 ++++++++++++++++++--
core/pom.xml | 2 +-
.../cas/filemgr/system/XmlRpcFileManager.java | 18 ++++-
pge/pom.xml | 2 +-
14 files changed, 343 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/pom.xml
----------------------------------------------------------------------
diff --git a/config/pom.xml b/config/pom.xml
index 3f26499..e34ff44 100644
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -26,7 +26,7 @@
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
new file mode 100644
index 0000000..8a465b0
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config;
+
+public enum ConfigEventType {
+ PUBLISH("publish"),
+ CLEAR("clear");
+
+ private String name;
+
+ ConfigEventType(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public static ConfigEventType parse(String string) {
+ switch (string) {
+ case "publish":
+ return PUBLISH;
+ case "clear":
+ return CLEAR;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java b/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
new file mode 100644
index 0000000..a5abcab
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config;
+
+/**
+ * The interface which should be implemented in order to listen for configuration changes.
+ *
+ * @author Imesha Sudasingha
+ */
+public interface ConfigurationListener {
+
+ /**
+ * This method is invoked when there has been any change in configuration of the interested component
+ */
+ void configurationChanged(ConfigEventType type);
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
index 53aacef..9158fad 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
@@ -17,7 +17,9 @@
package org.apache.oodt.config;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* The abstract class to define functions of the configuration managers.
@@ -28,6 +30,7 @@ public abstract class ConfigurationManager {
protected Component component;
protected String project;
+ private Set<ConfigurationListener> configurationListeners = new HashSet<>(1);
public ConfigurationManager(Component component) {
this(component, Constants.DEFAULT_PROJECT);
@@ -38,6 +41,12 @@ public abstract class ConfigurationManager {
this.project = project;
}
+ /**
+ * Loads configuration required for {@link #component}. If distributed configuration management is enabled, this
+ * will download configuration from zookeeper. Else, this will load properties files specified.
+ *
+ * @throws Exception
+ */
public abstract void loadConfiguration() throws Exception;
/**
@@ -47,6 +56,20 @@ public abstract class ConfigurationManager {
*/
public abstract void clearConfiguration();
+ public synchronized void addConfigurationListener(ConfigurationListener listener) {
+ configurationListeners.add(listener);
+ }
+
+ public synchronized void removeConfigurationListener(ConfigurationListener listener) {
+ configurationListeners.remove(listener);
+ }
+
+ protected synchronized void notifyConfigurationChange(ConfigEventType type) {
+ for (ConfigurationListener listener : configurationListeners) {
+ listener.configurationChanged(type);
+ }
+ }
+
public Component getComponent() {
return component;
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 9cd217f..3824afb 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -93,5 +93,8 @@ public class Constants {
/** Where other configuration files will be stored */
public static final String CONFIGURATION_PATH_NAME = "configuration";
+
+ /** Path to be watched for configuration changes */
+ public static final String NOTIFICATIONS_PATH = "notifications";
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index ae8912a..6dfb24b 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -19,7 +19,10 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.apache.curator.framework.recipes.cache.NodeCacheListener;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.Constants.Properties;
@@ -38,6 +41,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
+import static org.apache.oodt.config.Constants.Properties.ZK_STARTUP_TIMEOUT;
import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjectName;
/**
@@ -57,12 +61,41 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private List<String> savedFiles = new ArrayList<>();
+ /** {@link NodeCache} to watch for configuration change notifications */
+ private NodeCache nodeCache;
+ private NodeCacheListener nodeCacheListener = new NodeCacheListener() {
+ @Override
+ public void nodeChanged() throws Exception {
+ byte[] data = client.getData().forPath(zNodePaths.getNotificationsZNodePath());
+ if (data == null) {
+ return;
+ }
+
+ String event = new String(data);
+ ConfigEventType type = ConfigEventType.parse(event);
+ if (type != null) {
+ logger.info("Configuration changed event of type: '{}' received", type);
+ switch (type) {
+ case PUBLISH:
+ loadConfiguration();
+ break;
+ case CLEAR:
+ clearConfiguration();
+ break;
+ }
+
+ notifyConfigurationChange(type);
+ }
+ }
+ };
+
public DistributedConfigurationManager(Component component) {
super(component, getOODTProjectName());
this.zNodePaths = new ZNodePaths(this.project, this.component.getName());
- if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
+ if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " +
+ ZK_CONNECT_STRING + " to be set");
}
if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
@@ -73,11 +106,11 @@ public class DistributedConfigurationManager extends ConfigurationManager {
}
}
- if (System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
+ if (System.getProperty(ZK_CONNECT_STRING) == null) {
throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
}
- connectString = System.getProperty(Constants.Properties.ZK_CONNECT_STRING);
+ connectString = System.getProperty(ZK_CONNECT_STRING);
logger.info("Using zookeeper connect string : {}", connectString);
startZookeeper();
}
@@ -90,7 +123,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
client = CuratorUtils.newCuratorFrameworkClient(connectString, logger);
client.start();
logger.info("Curator framework start operation invoked");
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000"));
+
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(ZK_STARTUP_TIMEOUT, "30000"));
try {
logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
@@ -103,10 +137,21 @@ public class DistributedConfigurationManager extends ConfigurationManager {
}
logger.info("CuratorFramework client started successfully");
+
+ nodeCache = new NodeCache(client, zNodePaths.getNotificationsZNodePath());
+ nodeCache.getListenable().addListener(nodeCacheListener);
+ try {
+ logger.debug("Starting NodeCache to watch for configuration changes");
+ nodeCache.start(true);
+ } catch (Exception e) {
+ logger.error("Error occurred when start listening for configuration changes", e);
+ throw new IllegalStateException("Unable to start listening for configuration changes", e);
+ }
+ logger.info("NodeCache for watching configuration changes started successfully");
}
@Override
- public void loadConfiguration() throws Exception {
+ public synchronized void loadConfiguration() throws Exception {
logger.debug("Loading properties for : {}", component);
loadProperties();
logger.info("Properties loaded for : {}", component);
@@ -170,9 +215,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private void saveFile(String path, byte[] data) throws IOException {
String localFilePath = ConfigUtils.fixForComponentHome(component, path);
File localFile = new File(localFilePath);
- if (localFile.exists()) {
- logger.warn("Deleting already existing file at {} before writing new content", localFilePath);
- localFile.delete();
+ if (localFile.exists() && localFile.delete()) {
+ logger.warn("Deleted already existing file at {} before writing new content", localFilePath);
}
logger.debug("Storing configuration in file: {}", localFilePath);
@@ -183,7 +227,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
/** {@inheritDoc} */
@Override
- public void clearConfiguration() {
+ public synchronized void clearConfiguration() {
for (String path : savedFiles) {
logger.debug("Removing saved file {}", path);
File file = new File(path);
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
index 6229e96..9d3dd4a 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -20,6 +20,7 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.distributed.utils.CuratorUtils;
import org.apache.zookeeper.data.Stat;
@@ -83,6 +84,14 @@ public class DistributedConfigurationPublisher {
logger.info("Using zookeeper connect string : {}", connectString);
startZookeeper();
+
+ try {
+ logger.debug("Creating ZNode paths");
+ zNodePaths.createZNodes(client);
+ } catch (Exception e) {
+ logger.error("Error occurred when creating initial ZNode paths", e);
+ throw new IllegalStateException("Unable to create ZNode paths", e);
+ }
}
/**
@@ -145,7 +154,8 @@ public class DistributedConfigurationPublisher {
*/
public boolean verifyPublishedConfiguration() {
try {
- return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false);
+ return verifyPublishedConfiguration(propertiesFiles, true) &&
+ verifyPublishedConfiguration(configFiles, false);
} catch (Exception e) {
logger.error("Error occurred when checking published config", e);
return false;
@@ -165,6 +175,17 @@ public class DistributedConfigurationPublisher {
logger.info("Configuration cleared!");
}
+ /**
+ * Notifies the watching {@link org.apache.oodt.config.ConfigurationManager}s about the configuration change
+ *
+ * @param type {@link ConfigEventType}
+ * @throws Exception
+ */
+ public void notifyConfigEvent(ConfigEventType type) throws Exception {
+ logger.info("Notifying event: '{}' to configuration managers of {}", type, component);
+ client.setData().forPath(zNodePaths.getNotificationsZNodePath(), type.toString().getBytes());
+ }
+
private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
String filePath = entry.getKey();
@@ -173,7 +194,8 @@ public class DistributedConfigurationPublisher {
String content = getFileContent(filePath);
- String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) :
+ zNodePaths.getConfigurationZNodePath(relativeZNodePath);
if (client.checkExists().forPath(zNodePath) != null) {
byte[] bytes = client.getData().forPath(zNodePath);
String existingData = new String(bytes);
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
index cf3ca00..6324ac1 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
@@ -17,9 +17,14 @@
package org.apache.oodt.config.distributed;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.oodt.config.distributed.utils.CuratorUtils;
+import org.apache.zookeeper.CreateMode;
+
import static org.apache.oodt.config.Constants.DEFAULT_PROJECT;
import static org.apache.oodt.config.Constants.ZPaths.COMPONENTS_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.CONFIGURATION_PATH_NAME;
+import static org.apache.oodt.config.Constants.ZPaths.NOTIFICATIONS_PATH;
import static org.apache.oodt.config.Constants.ZPaths.PROJECTS_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.PROPERTIES_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.SEPARATOR;
@@ -43,6 +48,9 @@ public class ZNodePaths {
private String configurationZNodePath;
private String configurationZNodeRoot;
+ /** ZNode to be watched for configuration changes. /projects/${project}/components/${component}/notifications */
+ private String notificationsZNodePath;
+
/**
* Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre>
* given.
@@ -70,6 +78,21 @@ public class ZNodePaths {
configurationZNodePath = componentZNodeRoot + CONFIGURATION_PATH_NAME;
configurationZNodeRoot = configurationZNodePath + SEPARATOR;
+
+ notificationsZNodePath = componentZNodeRoot + NOTIFICATIONS_PATH;
+ }
+
+ /**
+ * Creates the initial ZNode structure in zookeeper. Supposed to be called by the {@link
+ * DistributedConfigurationPublisher}.
+ *
+ * @param client {@link CuratorFramework} instance
+ * @throws Exception
+ */
+ public void createZNodes(CuratorFramework client) throws Exception {
+ CuratorUtils.createZNodeIfNotExists(client, propertiesZNodePath, CreateMode.PERSISTENT, new byte[1]);
+ CuratorUtils.createZNodeIfNotExists(client, configurationZNodePath, CreateMode.PERSISTENT, new byte[1]);
+ CuratorUtils.createZNodeIfNotExists(client, notificationsZNodePath, CreateMode.PERSISTENT, new byte[1]);
}
public String getComponentZNodePath() {
@@ -99,4 +122,8 @@ public class ZNodePaths {
public String getLocalPropertiesFilePath(String zNodePath) {
return zNodePath.substring(propertiesZNodeRoot.length());
}
+
+ public String getNotificationsZNodePath() {
+ return notificationsZNodePath;
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
index 1ab1b30..50e567a 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
@@ -19,6 +19,7 @@ package org.apache.oodt.config.distributed.cli;
import org.apache.oodt.cas.cli.action.CmdLineAction;
import org.apache.oodt.cas.cli.exception.CmdLineActionException;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.distributed.DistributedConfigurationPublisher;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@@ -36,12 +37,9 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
*/
public class CLIAction extends CmdLineAction {
- public enum Action {
- PUBLISH, VERIFY, CLEAR
- }
-
private String connectString;
private String configFile = DEFAULT_CONFIG_PUBLISHER_XML;
+ private boolean notify = false;
private Action action;
@@ -60,12 +58,18 @@ public class CLIAction extends CmdLineAction {
switch (action) {
case PUBLISH:
publish(publisher);
+ if (notify) {
+ publisher.notifyConfigEvent(ConfigEventType.PUBLISH);
+ }
break;
case VERIFY:
verify(publisher);
break;
case CLEAR:
clear(publisher);
+ if (notify) {
+ publisher.notifyConfigEvent(ConfigEventType.CLEAR);
+ }
break;
}
publisher.destroy();
@@ -123,4 +127,16 @@ public class CLIAction extends CmdLineAction {
public void setConfigFile(String configFile) {
this.configFile = configFile;
}
+
+ public boolean isNotify() {
+ return notify;
+ }
+
+ public void setNotify(boolean notify) {
+ this.notify = notify;
+ }
+
+ public enum Action {
+ PUBLISH, VERIFY, CLEAR
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/main/resources/cmd-line-options.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/cmd-line-options.xml b/config/src/main/resources/cmd-line-options.xml
index 698ad32..3aa506b 100644
--- a/config/src/main/resources/cmd-line-options.xml
+++ b/config/src/main/resources/cmd-line-options.xml
@@ -82,4 +82,35 @@
</bean>
</property>
</bean>
+
+ <bean id="notifyConfigChange" class="org.apache.oodt.cas.cli.option.AdvancedCmdLineOption">
+ <property name="shortOption" value="n"/>
+ <property name="longOption" value="notify"/>
+ <property name="description" value="Notify the configuration managers' about the configuration change done"/>
+ <property name="hasArgs" value="false"/>
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="publish" p:relation="OPTIONAL"/>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="clear" p:relation="OPTIONAL"/>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="verify" p:relation="OPTIONAL"/>
+ </list>
+ </property>
+ <property name="handler">
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToActionHandler">
+ <property name="applyToActions">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="publish" p:methodName="setNotify"/>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="verify" p:methodName="setNotify"/>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="clear" p:methodName="setNotify"/>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
index 0bf2dde..b553643 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
@@ -17,6 +17,7 @@
package org.apache.oodt.config.distributed;
+import org.apache.oodt.config.Component;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.distributed.cli.ConfigPublisher;
import org.apache.oodt.config.distributed.utils.ConfigUtils;
@@ -32,6 +33,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -50,6 +52,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
private static final String CONFIG_PUBLISHER_XML = "config-publisher.xml";
private List<DistributedConfigurationPublisher> publishers;
+ private Map<Component, ConfigurationManager> configurationManagers;
@Before
public void setUpTest() throws Exception {
@@ -65,21 +68,24 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(CONFIG_PUBLISHER_XML);
Map distributedConfigurationPublishers = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
- publishers = new ArrayList<>(distributedConfigurationPublishers.values().size());
+ publishers = new ArrayList<>();
+ configurationManagers = new HashMap<>();
for (Object bean : distributedConfigurationPublishers.values()) {
DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+ System.setProperty(OODT_PROJECT, publisher.getProject());
System.setProperty(publisher.getComponent().getHome(), ".");
publishers.add(publisher);
+ configurationManagers.put(publisher.getComponent(), new DistributedConfigurationManager(publisher.getComponent()));
+ System.clearProperty(OODT_PROJECT);
}
}
@Test
public void loadConfigurationTest() throws Exception {
for (DistributedConfigurationPublisher publisher : publishers) {
- System.setProperty(OODT_PROJECT, publisher.getProject());
- ConfigurationManager configurationManager = new DistributedConfigurationManager(publisher.getComponent());
+ ConfigurationManager configurationManager = configurationManagers.get(publisher.getComponent());
configurationManager.loadConfiguration();
// Checking for configuration files
@@ -117,8 +123,63 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
File file = new File(localFile);
Assert.assertFalse(file.exists());
}
+ }
+ }
- System.clearProperty(OODT_PROJECT);
+ @Test
+ public void notifyConfigurationChangeTest() throws Exception {
+ // First publish config. Then check if config has downloaded locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "publish"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(true);
+
+ // Now clear config. Then check if config has deleted locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "clear"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(false);
+
+ // First publish config. Then check if config has downloaded locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "publish"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(true);
+ }
+
+ private void checkFileExistence(boolean exists) {
+ for (DistributedConfigurationPublisher publisher : publishers) {
+ for (String fileName : publisher.getPropertiesFiles().values()) {
+ fileName = ConfigUtils.fixForComponentHome(publisher.getComponent(), fileName);
+ File file = new File(fileName);
+ if (exists) {
+ Assert.assertTrue(file.exists());
+ } else {
+ Assert.assertFalse(file.exists());
+ }
+ }
+
+ for (String fileName : publisher.getConfigFiles().values()) {
+ fileName = ConfigUtils.fixForComponentHome(publisher.getComponent(), fileName);
+ File file = new File(fileName);
+ if (exists) {
+ Assert.assertTrue(file.exists());
+ } else {
+ Assert.assertFalse(file.exists());
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 5e84d83..f6e0fc3 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -244,7 +244,7 @@ the License.
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
index 4ac48bb..37d32d2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
@@ -54,6 +54,8 @@ import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
import org.apache.oodt.commons.date.DateUtils;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
+import org.apache.oodt.config.ConfigurationListener;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.ConfigurationManagerFactory;
import org.apache.xmlrpc.WebServer;
@@ -112,6 +114,18 @@ public class XmlRpcFileManager {
/** Configuration Manager instance which will handle the configuration aspect in distributed/standalone manner */
private ConfigurationManager configurationManager;
+ private ConfigurationListener configurationListener = new ConfigurationListener() {
+ @Override
+ public void configurationChanged(ConfigEventType type) {
+ switch (type) {
+ case PUBLISH:
+ refreshConfigAndPolicy();
+ break;
+ case CLEAR:
+ // TODO: 8/19/17 What should we do if the config has been cleared?
+ }
+ }
+ };
/**
* <p> Creates a new XmlRpcFileManager with the given metadata store factory, and the given data store factory, on the
@@ -135,8 +149,9 @@ public class XmlRpcFileManager {
}
configurationManager = ConfigurationManagerFactory.getConfigurationManager(Component.FILE_MANAGER, propertiesFiles);
-
+ configurationManager.addConfigurationListener(configurationListener);
this.loadConfiguration();
+
LOG.log(Level.INFO, "File Manager started by " + System.getProperty("user.name", "unknown"));
}
@@ -1224,6 +1239,7 @@ public class XmlRpcFileManager {
}
public boolean shutdown() {
+ configurationManager.removeConfigurationListener(configurationListener);
configurationManager.clearConfiguration();
if (this.webServer != null) {
this.webServer.shutdown();
http://git-wip-us.apache.org/repos/asf/oodt/blob/711b445c/pge/pom.xml
----------------------------------------------------------------------
diff --git a/pge/pom.xml b/pge/pom.xml
index 6fd4d72..72a33d9 100644
--- a/pge/pom.xml
+++ b/pge/pom.xml
@@ -27,7 +27,7 @@ the License.
<artifactId>cas-pge</artifactId>
<name>CAS PGE Adaptor Framework</name>
<description>Allows data processing jobs not written in conformance with the
- PCS PGE interface to be run within the PCS.</description>
+ PCS PGE (Production Generation Executive) interface to be run within the PCS.</description>
<!-- All dependencies should be listed in core/pom.xml and be ordered alphabetically by package and artifact.
Once the dependency is in the core pom, it can then be used in other modules without the version tags.
For example, within core/pom.xml:
[04/50] [abbrv] oodt git commit: Fix for OODT-929: EnvUtilities
should only load environment variables one time and statically cache them.
Posted by ma...@apache.org.
Fix for OODT-929: EnvUtilities should only load environment variables one time and statically cache them.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/26116853
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/26116853
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/26116853
Branch: refs/heads/master
Commit: 261168532b918eacb41a3689b20f1fe196b3411b
Parents: e55f2cd
Author: Chris Mattmann <ma...@apache.org>
Authored: Tue Jul 5 10:57:46 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:09:17 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 9 ++++++++-
.../org/apache/oodt/commons/exec/EnvUtilities.java | 16 ++++++++++------
.../apache/oodt/commons/exec/TestEnvUtilities.java | 9 +++++++++
3 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/26116853/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index d88b954..7d0b1b2 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,6 +1,13 @@
Apache OODT Change Log
======================
-Release 0.13 - 06/21/2016
+
+Release 1.1 - Current Development
+
+* OODT-929 EnvUtilities should only load environment variables one time and
+ statically cache them (mattmann)
+
+
+Release 1.0 - 06/21/2016
* OODT-925 - Update schema.xml field defs for SOLR 5.x
* OODT-926 - Fix pom versioning
http://git-wip-us.apache.org/repos/asf/oodt/blob/26116853/commons/src/main/java/org/apache/oodt/commons/exec/EnvUtilities.java
----------------------------------------------------------------------
diff --git a/commons/src/main/java/org/apache/oodt/commons/exec/EnvUtilities.java b/commons/src/main/java/org/apache/oodt/commons/exec/EnvUtilities.java
index 5e630a7..18fbf39 100644
--- a/commons/src/main/java/org/apache/oodt/commons/exec/EnvUtilities.java
+++ b/commons/src/main/java/org/apache/oodt/commons/exec/EnvUtilities.java
@@ -48,6 +48,14 @@ public final class EnvUtilities {
private EnvUtilities() throws InstantiationException {
throw new InstantiationException("Don't construct utility classes!");
}
+
+ private final static Properties environment = new Properties();
+ static{
+ System.getenv().entrySet();
+ for (Map.Entry<String, String> entry : System.getenv().entrySet()) {
+ environment.setProperty(entry.getKey(), entry.getValue());
+ }
+ }
/**
* This method returns a particular named environment variable from the user's
@@ -57,7 +65,7 @@ public final class EnvUtilities {
* @return The environment variable value, as a String.
*/
public static String getEnv(String envVarName) {
- return System.getenv(envVarName);
+ return environment.getProperty(envVarName);
}
/**
@@ -67,11 +75,7 @@ public final class EnvUtilities {
* @return The user's current environment, in a {@link Properties} object.
*/
public static Properties getEnv() {
- Properties envProps = new Properties();
- for (Map.Entry<String, String> entry : System.getenv().entrySet()) {
- envProps.setProperty(entry.getKey(), entry.getValue());
- }
- return envProps;
+ return environment;
}
/**
http://git-wip-us.apache.org/repos/asf/oodt/blob/26116853/commons/src/test/java/org/apache/oodt/commons/exec/TestEnvUtilities.java
----------------------------------------------------------------------
diff --git a/commons/src/test/java/org/apache/oodt/commons/exec/TestEnvUtilities.java b/commons/src/test/java/org/apache/oodt/commons/exec/TestEnvUtilities.java
index 61d02dc..eaef9d9 100644
--- a/commons/src/test/java/org/apache/oodt/commons/exec/TestEnvUtilities.java
+++ b/commons/src/test/java/org/apache/oodt/commons/exec/TestEnvUtilities.java
@@ -124,4 +124,13 @@ public class TestEnvUtilities extends TestCase {
assertEquals(userNameTest1,userNameTest2);
}
}
+
+ public void testStaticEnvironment(){
+ if(SystemUtils.IS_OS_UNIX){
+ Properties env = EnvUtilities.getEnv();
+ Properties env2 = EnvUtilities.getEnv();
+
+ assertEquals(env, env2);
+ }
+ }
}
[19/50] [abbrv] oodt git commit: fix some counts
Posted by ma...@apache.org.
fix some counts
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/2820d5d1
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/2820d5d1
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/2820d5d1
Branch: refs/heads/master
Commit: 2820d5d120ef4bb50f128c45cc16cc651e90ba63
Parents: d1b0a7b
Author: Tom Barber <to...@analytical-labs.com>
Authored: Fri Jul 8 22:21:07 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:35:11 2017 -0800
----------------------------------------------------------------------
.../apache/oodt/cas/filemgr/catalog/LuceneCatalog.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/2820d5d1/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index 0a5564b..b54e8f7 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -1179,22 +1179,22 @@ public class LuceneCatalog implements Catalog {
doc.add(new Field("product_id", product.getProductId(),
StringField.TYPE_STORED));
doc.add(new Field("product_name", product.getProductName(),
- TextField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc.add(new Field("product_structure", product.getProductStructure(),
- TextField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc
.add(new Field("product_transfer_status", product
- .getTransferStatus(), TextField.TYPE_STORED));
+ .getTransferStatus(), StringField.TYPE_STORED));
// product type
doc
.add(new Field("product_type_id", product.getProductType()
.getProductTypeId(), StringField.TYPE_STORED));
doc.add(new Field("product_type_name", product.getProductType()
- .getName(), TextField.TYPE_STORED));
+ .getName(), StringField.TYPE_STORED));
doc.add(new Field("product_type_desc", product.getProductType()
.getDescription() != null ? product.getProductType()
- .getDescription() : "", TextField.TYPE_STORED));
+ .getDescription() : "", StringField.TYPE_STORED));
doc.add(new Field("product_type_repoPath", product.getProductType()
.getProductRepositoryPath() != null ? product.getProductType()
.getProductRepositoryPath() : "", StringField.TYPE_STORED));
@@ -1260,7 +1260,7 @@ public class LuceneCatalog implements Catalog {
// add special field for all products
// then can use that field to retrieve back all products
- doc.add(new Field("myfield", "myvalue", TextField.TYPE_STORED));
+ doc.add(new Field("myfield", "myvalue", StringField.TYPE_STORED));
return doc;
}
[25/50] [abbrv] oodt git commit: OODT-938 fix netcdf so it uses
non-fat jar
Posted by ma...@apache.org.
OODT-938 fix netcdf so it uses non-fat jar
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/b1e39db9
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/b1e39db9
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/b1e39db9
Branch: refs/heads/master
Commit: b1e39db96d686633c6bff7b66f44fcf7aae4fcf3
Parents: a5d6730
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Nov 10 15:25:07 2016 +0000
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:42:19 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 4 ++++
core/pom.xml | 9 +++++++--
opendapps/pom.xml | 6 +++++-
.../java/org/apache/oodt/opendapps/DatasetExtractor.java | 5 +++++
4 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/b1e39db9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e779398..9510096 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,10 @@ Apache OODT Change Log
Release 1.1 - Current Development
+* OODT-938 update netcdf (magicaltrout)
+
+* OODT-943 update javadoc and site plugin versions (magicaltrout)
+
* OODT-931 CAS Crawler and metadata preconditions should say what preconditions
failed (mattmann)
http://git-wip-us.apache.org/repos/asf/oodt/blob/b1e39db9/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 55392c4..3b7b857 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -154,8 +154,13 @@ the License.
</dependency>
<dependency>
<groupId>edu.ucar</groupId>
- <artifactId>netcdf</artifactId>
- <version>4.2</version>
+ <artifactId>netcdf4</artifactId>
+ <version>4.5.5</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>opendap</artifactId>
+ <version>2.2.2</version>
</dependency>
<dependency>
<groupId>hsqldb</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/b1e39db9/opendapps/pom.xml
----------------------------------------------------------------------
diff --git a/opendapps/pom.xml b/opendapps/pom.xml
index 7f7f051..0d8b27c 100644
--- a/opendapps/pom.xml
+++ b/opendapps/pom.xml
@@ -51,7 +51,11 @@
<dependencies>
<dependency>
<groupId>edu.ucar</groupId>
- <artifactId>netcdf</artifactId>
+ <artifactId>netcdf4</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>opendap</artifactId>
</dependency>
<dependency>
<groupId>org.apache.oodt</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/b1e39db9/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
----------------------------------------------------------------------
diff --git a/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java b/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
index 192494c..ba22df8 100644
--- a/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
+++ b/opendapps/src/main/java/org/apache/oodt/opendapps/DatasetExtractor.java
@@ -107,6 +107,11 @@ public class DatasetExtractor {
LOG.log(Level.WARNING, msg);
}
+ @Override
+ public void setProgress(String s, int i) {
+
+ }
+
};
LOG.log(Level.FINE, "catalogURL: " + this.mainCatalogURL);
[20/50] [abbrv] oodt git commit: finish lucene 6 test compatability
for fmgr
Posted by ma...@apache.org.
finish lucene 6 test compatability for fmgr
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/2f79f68f
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/2f79f68f
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/2f79f68f
Branch: refs/heads/master
Commit: 2f79f68f3243c04402572b74532b9a20622b5c16
Parents: 2820d5d
Author: Tom Barber <to...@analytical-labs.com>
Authored: Fri Jul 8 23:00:20 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:35:22 2017 -0800
----------------------------------------------------------------------
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 66 +++++++++++---------
.../cli/action/LuceneQueryCliAction.java | 10 ++-
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 5 +-
3 files changed, 46 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/2f79f68f/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index b54e8f7..4c9ed96 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -448,20 +448,25 @@ public class LuceneCatalog implements Catalog {
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
- TopDocs topDocs = searcher.search(query, check.totalHits, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
- ScoreDoc[] hits = topDocs.scoreDocs;
+ ScoreDoc[] hits = topDocs.scoreDocs;
- // should be > 0 hits
- if (hits.length > 0) {
- // just get the first hit back
- Document productDoc = searcher.doc(hits[0].doc);
- CompleteProduct prod = toCompleteProduct(productDoc, getRefs,
- false);
- return prod.getProduct();
- } else {
- LOG.log(Level.FINEST, "Request for product by name: ["
- + productName + "] returned no results");
+ // should be > 0 hits
+ if (hits.length > 0) {
+ // just get the first hit back
+ Document productDoc = searcher.doc(hits[0].doc);
+ CompleteProduct prod = toCompleteProduct(productDoc, getRefs,
+ false);
+ return prod.getProduct();
+ } else {
+ LOG.log(Level.FINEST, "Request for product by name: ["
+ + productName + "] returned no results");
+ return null;
+ }
+ }
+ else{
return null;
}
@@ -732,22 +737,27 @@ public class LuceneCatalog implements Catalog {
+ "]");
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(booleanQuery.build(), 1, sort);
- TopDocs topDocs = searcher.search(booleanQuery.build(), check.totalHits, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(booleanQuery.build(), check.totalHits, sort);
- ScoreDoc[] hits = topDocs.scoreDocs;
+ ScoreDoc[] hits = topDocs.scoreDocs;
- if (hits.length > 0) {
- products = new Vector<Product>(n);
- int i = 0;
- while (products.size() < Math.min(n, hits.length)) {
- Document productDoc = searcher.doc(hits[i].doc);
- CompleteProduct prod = toCompleteProduct(productDoc, false,
- false);
- products.add(prod.getProduct());
- i++;
+ if (hits.length > 0) {
+ products = new Vector<Product>(n);
+ int i = 0;
+ while (products.size() < Math.min(n, hits.length)) {
+ Document productDoc = searcher.doc(hits[i].doc);
+ CompleteProduct prod = toCompleteProduct(productDoc, false,
+ false);
+ products.add(prod.getProduct());
+ i++;
+ }
+ } else {
+ LOG.log(Level.WARNING, "Top N query produced no products!");
}
- } else {
- LOG.log(Level.WARNING, "Top N query produced no products!");
+ }
+ else{
+ return null;
}
} catch (IOException e) {
@@ -1468,15 +1478,11 @@ public class LuceneCatalog implements Catalog {
String startVal = ((RangeQueryCriteria) queryCriteria).getStartValue();
String endVal = ((RangeQueryCriteria) queryCriteria).getEndValue();
boolean inclusive = ((RangeQueryCriteria) queryCriteria).getInclusive();
- Term startTerm = null, endTerm = null;
+ Term startTerm = null;
if (!startVal.equals("")) {
startTerm = new Term(queryCriteria.getElementName(), startVal);
}
- if (!endVal.equals("")) {
- endTerm = new Term(queryCriteria.getElementName(), endVal);
- }
-
return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive);
}else {
throw new CatalogException("Invalid QueryCriteria ["
http://git-wip-us.apache.org/repos/asf/oodt/blob/2f79f68f/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
index 85e06cd..c91790c 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
@@ -20,6 +20,7 @@ package org.apache.oodt.cas.filemgr.cli.action;
import com.google.common.collect.Lists;
import org.apache.commons.lang.Validate;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
@@ -80,8 +81,7 @@ public class LuceneQueryCliAction extends AbstractQueryCliAction {
private Query parseQuery(String query) throws ParseException {
// note that "__FREE__" is a control work for free text searching
- return (Query) new QueryParser(FREE_TEXT_BLOCK, new CASAnalyzer())
- .parse(query);
+ return (Query) new QueryParser(FREE_TEXT_BLOCK, new WhitespaceAnalyzer()).parse(query);
}
private QueryCriteria generateCASQuery(Query luceneQuery)
@@ -110,8 +110,12 @@ public class LuceneQueryCliAction extends AbstractQueryCliAction {
BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
//TODO CHECK Inclusive
+ boolean inc = false;
+ if(((TermRangeQuery) luceneQuery).includesLower() && ((TermRangeQuery) luceneQuery).includesUpper()){
+ inc = true;
+ }
return new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT
- .utf8ToString(), endT.utf8ToString()/*, ((TermRangeQuery) luceneQuery).isInclusive()*/);
+ .utf8ToString(), endT.utf8ToString(), inc);
} else if (luceneQuery instanceof BooleanQuery) {
List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
BooleanQueryCriteria bqc = new BooleanQueryCriteria();
http://git-wip-us.apache.org/repos/asf/oodt/blob/2f79f68f/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
index 9470c62..bfe2384 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
@@ -78,14 +78,15 @@ public class CASAnalyzer extends Analyzer {
}
/** Builds an analyzer with the given stop words. */
public CASAnalyzer(CharArraySet stopWords) {
- //stopSet = StopFilter.makeStopSet(stopWords);
+ stopSet = StopFilter.makeStopSet(stopWords.toArray(new String[stopWords.size()]));
+
}
/**
* Constructs a {@link org.apache.lucene.analysis.standard.StandardTokenizer} filtered by a {@link
* StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
*/
- /*public TokenStream tokenStream(String fieldName, Reader reader) {
+ /*4public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new WhitespaceTokenizer(reader);
result = new StandardFilter(result);
result = new StopFilter(result, stopSet);
[16/50] [abbrv] oodt git commit: back to pertest in fmgr
Posted by ma...@apache.org.
back to pertest in fmgr
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/3939d0a3
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/3939d0a3
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/3939d0a3
Branch: refs/heads/master
Commit: 3939d0a3ecf74fc475b696f8f1cfcbb76abf139e
Parents: fa82510
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Jul 7 10:09:44 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:32:23 2017 -0800
----------------------------------------------------------------------
filemgr/pom.xml | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/3939d0a3/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index cc9a6e5..6e102dc 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -276,7 +276,6 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkMode>pertest</forkMode>
- <!--forkMode>pertest</forkMode-->
<!-- so we can DEBUG -->
<argLine>-enableassertions</argLine>
<useSystemClassLoader>false</useSystemClassLoader>
[23/50] [abbrv] oodt git commit: downgrade solrj
Posted by ma...@apache.org.
downgrade solrj
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/89d6bf13
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/89d6bf13
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/89d6bf13
Branch: refs/heads/master
Commit: 89d6bf13a6168236dd7d43224560f666b8f13839
Parents: 4c3e94a
Author: Tom Barber <to...@analytical-labs.com>
Authored: Mon Jul 11 19:38:58 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:36:06 2017 -0800
----------------------------------------------------------------------
core/pom.xml | 2 +-
filemgr/pom.xml | 1 -
2 files changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/89d6bf13/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 7eee1db..83bf104 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -65,7 +65,7 @@ the License.
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>6.1.0</version>
+ <version>1.3.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/89d6bf13/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 54b4e31..a30ba3a 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -284,7 +284,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <forkMode>never</forkMode>
<!--forkMode>pertest</forkMode-->
<forkMode>pertest</forkMode>
<!-- so we can DEBUG -->
[29/50] [abbrv] oodt git commit: fix merge
Posted by ma...@apache.org.
fix merge
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ab366e2a
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ab366e2a
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ab366e2a
Branch: refs/heads/master
Commit: ab366e2a648df7c7ac37aaa1c49d1d98a1aeb20c
Parents: 78ce45e 07d9030
Author: Tom Barber <to...@analytical-labs.com>
Authored: Wed Jul 26 13:16:15 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Wed Jul 26 13:16:15 2017 +0100
----------------------------------------------------------------------
CHANGES.txt | 21 +-
app/fmbrowser/pom.xml | 1 +
.../cas/filemgr/browser/model/QueryBuilder.java | 30 +-
app/weditor/.gitignore | 1 +
catalog/.gitignore | 1 +
catalog/pom.xml | 15 +
commons/.gitignore | 1 +
.../apache/oodt/commons/exec/EnvUtilities.java | 16 +-
.../oodt/commons/exec/TestEnvUtilities.java | 9 +
core/pom.xml | 50 ++-
crawler/.gitignore | 1 +
crawler/pom.xml | 14 +
.../cas/crawl/MetExtractorProductCrawler.java | 37 +-
.../oodt/cas/crawl/StdProductCrawler.java | 16 +-
.../cas/crawl/action/SolrIndexingAction.java | 18 +-
curator/webapp/pom.xml | 12 -
docker/Dockerfile | 6 +-
filemgr/pom.xml | 11 +-
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 430 +++++++++++--------
.../filemgr/catalog/LuceneCatalogFactory.java | 18 +-
.../cli/action/LuceneQueryCliAction.java | 33 +-
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 37 +-
.../oodt/cas/filemgr/tools/CatalogSearch.java | 26 +-
.../filemgr/tools/OptimizeLuceneCatalog.java | 24 +-
.../oodt/cas/filemgr/tools/QueryTool.java | 23 +-
.../cas/filemgr/tools/RangeQueryTester.java | 50 ++-
.../cas/filemgr/catalog/TestLuceneCatalog.java | 4 +-
metadata/.gitignore | 1 +
opendapps/pom.xml | 6 +-
.../apache/oodt/opendapps/DatasetExtractor.java | 5 +
pcs/input/.gitignore | 1 +
product/.gitignore | 1 +
protocol/api/.gitignore | 1 +
protocol/ftp/.gitignore | 1 +
protocol/http/.gitignore | 1 +
protocol/imaps/.gitignore | 1 +
protocol/sftp/.gitignore | 1 +
workflow/pom.xml | 4 +
.../LuceneWorkflowInstanceRepository.java | 320 ++++++++------
...LuceneWorkflowInstanceRepositoryFactory.java | 25 +-
.../workflow/tools/TestInstanceRepoCleaner.java | 2 +-
workflow/src/test/resources/testinstrepo/_4.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_4.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_4.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/_43.cfs | Bin 14148 -> 0 bytes
workflow/src/test/resources/testinstrepo/_5.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_5.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_5.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/deletable | Bin 4 -> 0 bytes
.../resources/testinstrepo/pending_segments_b | Bin 0 -> 198 bytes
.../src/test/resources/testinstrepo/segments | Bin 28 -> 0 bytes
.../src/test/resources/testinstrepo/segments_a | Bin 0 -> 136 bytes
.../src/test/resources/testinstrepo/write.lock | 0
xmlps/.gitignore | 1 +
xmlquery/.gitignore | 1 +
55 files changed, 825 insertions(+), 452 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/catalog/pom.xml
----------------------------------------------------------------------
diff --cc catalog/pom.xml
index 07805a3,fa13c3a..cc73c07
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@@ -60,9 -60,24 +60,24 @@@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-lucene-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/crawler/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/filemgr/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ab366e2a/workflow/pom.xml
----------------------------------------------------------------------
[10/50] [abbrv] oodt git commit: - ignore
Posted by ma...@apache.org.
- ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ed2f4c54
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ed2f4c54
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ed2f4c54
Branch: refs/heads/master
Commit: ed2f4c5494466872112a1e00d5c352879b3914af
Parents: 17d7245
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:31:05 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:11:15 2017 -0800
----------------------------------------------------------------------
protocol/ftp/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ed2f4c54/protocol/ftp/.gitignore
----------------------------------------------------------------------
diff --git a/protocol/ftp/.gitignore b/protocol/ftp/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/protocol/ftp/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[47/50] [abbrv] oodt git commit: [OODT-964] Removed empty
test/workflow.properties which caused wmgr tests fail
Posted by ma...@apache.org.
[OODT-964] Removed empty test/workflow.properties which caused wmgr tests fail
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/b52f235f
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/b52f235f
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/b52f235f
Branch: refs/heads/master
Commit: b52f235f799502ed04dc11da200f9cbe2650fced
Parents: c7d0842
Author: Imesha Sudasingha <im...@apache.org>
Authored: Wed Oct 11 22:54:03 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Oct 12 08:07:15 2017 +0530
----------------------------------------------------------------------
workflow/src/test/resources/workflow.properties | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/b52f235f/workflow/src/test/resources/workflow.properties
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/workflow.properties b/workflow/src/test/resources/workflow.properties
deleted file mode 100755
index e69de29..0000000
[36/50] [abbrv] oodt git commit: Merge master into development
AvroRPC branch.
Posted by ma...@apache.org.
Merge master into development AvroRPC branch.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/423444cb
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/423444cb
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/423444cb
Branch: refs/heads/master
Commit: 423444cb9147f21f211a45e20a5fb74203d88666
Parents: e43c6c0 871803a
Author: Chris Mattmann <ma...@apache.org>
Authored: Sat Sep 2 08:34:57 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sat Sep 2 08:34:57 2017 -0700
----------------------------------------------------------------------
26.patch | 3506 ------------------
CHANGES.txt | 45 +-
CHANGES.txt.orig | 1189 ------
KEYS | 39 +
README.md | 6 +-
agility/oodt/conf/fm.conf | 43 +
agility/oodt/conf/setclasspath.sh | 17 +
agility/oodt/filemgr.py | 554 +++
agility/oodt/metadata.py | 29 +
agility/oodt/workflow.py | 134 +
agility/setup.py | 2 +-
app/fmbrowser/pom.xml | 81 -
app/fmbrowser/src/main/bin/filemgr-browser | 37 -
.../browser/controller/TableListener.java | 172 -
.../browser/controller/WindowListener.java | 166 -
.../oodt/cas/filemgr/browser/model/CasDB.java | 188 -
.../cas/filemgr/browser/model/QueryBuilder.java | 102 -
.../oodt/cas/filemgr/browser/model/Results.java | 68 -
.../browser/system/XmlRpcFilemgrBrowser.java | 35 -
.../cas/filemgr/browser/view/GuiParams.java | 28 -
.../cas/filemgr/browser/view/MainWindow.java | 70 -
.../cas/filemgr/browser/view/menus/MenuBar.java | 98 -
.../browser/view/menus/RightClickMenu.java | 51 -
.../filemgr/browser/view/panels/BottomPane.java | 64 -
.../cas/filemgr/browser/view/panels/Cell.java | 73 -
.../filemgr/browser/view/panels/HeaderCell.java | 49 -
.../filemgr/browser/view/panels/HeaderRow.java | 99 -
.../browser/view/panels/HeaderSpacer.java | 48 -
.../filemgr/browser/view/panels/MiddlePane.java | 52 -
.../filemgr/browser/view/panels/QueryField.java | 97 -
.../filemgr/browser/view/panels/QueryPane.java | 69 -
.../cas/filemgr/browser/view/panels/Row.java | 83 -
.../filemgr/browser/view/panels/TablePane.java | 150 -
.../browser/view/prompts/ConnectPrompt.java | 105 -
.../view/prompts/QueryBuilderPrompt.java | 444 ---
.../browser/view/prompts/SortPrompt.java | 153 -
app/weditor/.gitignore | 1 -
app/weditor/pom.xml | 110 -
app/weditor/src/main/assembly/assembly.xml | 72 -
app/weditor/src/main/bin/weditor | 22 -
.../oodt/cas/workflow/gui/WorkflowGUI.java | 325 --
.../oodt/cas/workflow/gui/menu/EditMenu.java | 48 -
.../oodt/cas/workflow/gui/menu/FileMenu.java | 82 -
.../oodt/cas/workflow/gui/model/ModelGraph.java | 307 --
.../oodt/cas/workflow/gui/model/ModelNode.java | 375 --
.../model/repo/XmlWorkflowModelRepository.java | 614 ---
.../repo/XmlWorkflowModelRepositoryFactory.java | 49 -
.../gui/perspective/MultiStatePerspective.java | 103 -
.../workflow/gui/perspective/Perspective.java | 87 -
.../gui/perspective/build/BuildPerspective.java | 436 ---
.../gui/perspective/view/MultiStateView.java | 46 -
.../cas/workflow/gui/perspective/view/View.java | 97 -
.../gui/perspective/view/ViewChange.java | 97 -
.../gui/perspective/view/ViewListener.java | 32 -
.../gui/perspective/view/ViewState.java | 271 --
.../perspective/view/impl/DefaultPropView.java | 862 -----
.../perspective/view/impl/DefaultTreeView.java | 496 ---
.../perspective/view/impl/GlobalConfigView.java | 270 --
.../gui/perspective/view/impl/GraphView.java | 1140 ------
.../perspective/view/impl/IdentifiableEdge.java | 92 -
.../view/impl/JungJGraphModelAdapter.java | 246 --
.../perspective/view/impl/TreeProjectView.java | 155 -
.../oodt/cas/workflow/gui/toolbox/Tool.java | 125 -
.../oodt/cas/workflow/gui/toolbox/ToolBox.java | 76 -
.../oodt/cas/workflow/gui/util/GuiUtils.java | 377 --
.../oodt/cas/workflow/gui/util/IconLoader.java | 83 -
.../apache/oodt/cas/workflow/gui/util/Line.java | 79 -
.../gui/util/exceptions/WorkflowException.java | 27 -
app/weditor/src/main/resources/REMOVE.log | 18 -
.../src/main/resources/examples/GranuleMaps.xml | 238 --
.../main/resources/examples/mspi-workflow.xml | 124 -
.../resources/examples/shared-properties.xml | 85 -
.../src/main/resources/examples/test.xml | 78 -
.../src/main/resources/icons/cursor-zoom.png | Bin 2967 -> 0 bytes
.../src/main/resources/icons/delete-sel.jpg | Bin 2686 -> 0 bytes
app/weditor/src/main/resources/icons/delete.jpg | Bin 2219 -> 0 bytes
.../src/main/resources/icons/edit-sel.jpg | Bin 2276 -> 0 bytes
app/weditor/src/main/resources/icons/edit.jpg | Bin 2375 -> 0 bytes
.../src/main/resources/icons/move-sel.jpg | Bin 1924 -> 0 bytes
app/weditor/src/main/resources/icons/move.jpg | Bin 2021 -> 0 bytes
.../src/main/resources/icons/zoom-in-sel.jpg | Bin 2438 -> 0 bytes
.../src/main/resources/icons/zoom-in.jpg | Bin 2192 -> 0 bytes
.../src/main/resources/icons/zoom-out-sel.jpg | Bin 2356 -> 0 bytes
.../src/main/resources/icons/zoom-out.jpg | Bin 2137 -> 0 bytes
.../src/main/resources/logging.properties | 40 -
balance/etc/skel/.htaccess | 45 -
balance/etc/skel/README.txt | 90 -
balance/etc/skel/config.ini | 82 -
balance/etc/skel/hooks.php | 98 -
balance/etc/skel/index.php | 48 -
.../skel/scripts/widgets/BreadcrumbsWidget.php | 101 -
balance/etc/skel/static/css/balance/balance.css | 119 -
balance/etc/skel/static/css/site.css | 19 -
balance/etc/skel/views/common/footer.php | 38 -
balance/etc/skel/views/common/header.php | 56 -
balance/etc/skel/views/errors/403.php | 30 -
balance/etc/skel/views/errors/404.php | 29 -
balance/etc/skel/views/index.php | 35 -
balance/lib/pear/Boot/bootstrap.php | 59 -
balance/lib/pear/Core/Application.class.php | 427 ---
.../pear/Core/ApplicationDataResponse.class.php | 81 -
.../lib/pear/Core/ApplicationRequest.class.php | 171 -
.../lib/pear/Core/ApplicationResponse.class.php | 311 --
.../IApplicationAuthenticationProvider.php | 162 -
.../IApplicationAuthorizationProvider.php | 62 -
.../Interfaces/IApplicationDataProvider.php | 125 -
.../Interfaces/IApplicationErrorProvider.php | 39 -
.../lib/pear/Interfaces/IApplicationWidget.php | 67 -
.../LDAPAuthenticationProvider.class.php | 170 -
.../LDAPAuthorizationProvider.class.php | 109 -
.../Providers/Data/MDB2DataProvider.class.php | 137 -
.../Providers/Data/MySqlDataProvider.class.php | 128 -
.../Providers/Data/PDODataProvider.class.php | 142 -
.../Error/DefaultErrorProvider.class.php | 38 -
balance/lib/pear/README | 34 -
balance/lib/pear/package.xml | 136 -
.../cas-browser/classes/CasBrowser.class.php | 455 ---
.../modules/cas-browser/classes/Utils.class.php | 388 --
balance/modules/cas-browser/config.ini | 121 -
.../modules/cas-browser/element-ordering.ini | 64 -
.../modules/cas-browser/element-visibility.ini | 49 -
balance/modules/cas-browser/hooks.php | 74 -
.../cas-browser/scripts/allTypeScript.php | 64 -
.../scripts/crossTypeQueryScript.php | 246 --
.../modules/cas-browser/scripts/pageScript.php | 102 -
.../cas-browser/scripts/productTypeFilter.php | 70 -
.../modules/cas-browser/scripts/queryScript.php | 155 -
.../scripts/widgets/BasicSearchWidget.php | 42 -
.../scripts/widgets/CrossTypeSearchWidget.php | 125 -
.../scripts/widgets/CustomSearchWidget.php | 178 -
.../scripts/widgets/FilterWidget.php | 136 -
.../scripts/widgets/MetadataDisplayWidget.php | 90 -
.../scripts/widgets/ProductDownloadWidget.php | 96 -
.../scripts/widgets/ProductPageWidget.php | 133 -
.../scripts/widgets/ProductTypeListWidget.php | 92 -
.../cas-browser/static/css/cas-browser.css | 289 --
.../cas-browser/static/css/dataTables.css | 469 ---
.../static/img/dataTables/Sorting icons.psd | Bin 27490 -> 0 bytes
.../static/img/dataTables/back_disabled.png | Bin 1361 -> 0 bytes
.../static/img/dataTables/back_enabled.png | Bin 1379 -> 0 bytes
.../img/dataTables/back_enabled_hover.png | Bin 1375 -> 0 bytes
.../static/img/dataTables/favicon.ico | Bin 894 -> 0 bytes
.../static/img/dataTables/forward_disabled.png | Bin 1363 -> 0 bytes
.../static/img/dataTables/forward_enabled.png | Bin 1380 -> 0 bytes
.../img/dataTables/forward_enabled_hover.png | Bin 1379 -> 0 bytes
.../static/img/dataTables/sort_asc.png | Bin 1118 -> 0 bytes
.../static/img/dataTables/sort_asc_disabled.png | Bin 1050 -> 0 bytes
.../static/img/dataTables/sort_both.png | Bin 1136 -> 0 bytes
.../static/img/dataTables/sort_desc.png | Bin 1127 -> 0 bytes
.../img/dataTables/sort_desc_disabled.png | Bin 1045 -> 0 bytes
.../cas-browser/static/img/download-icon.gif | Bin 556 -> 0 bytes
.../modules/cas-browser/static/img/loading.gif | Bin 7247 -> 0 bytes
.../cas-browser/static/img/zip-icon-smaller.gif | Bin 1877 -> 0 bytes
.../modules/cas-browser/static/img/zip-icon.gif | Bin 3476 -> 0 bytes
.../static/js/crosstypesearchwidget.js | 106 -
.../cas-browser/static/js/customsearchwidget.js | 78 -
.../cas-browser/static/js/filterwidget.js | 98 -
.../cas-browser/static/js/jcorner.jquery.js | 247 --
.../cas-browser/static/js/jquery-1.4.2-min.js | 154 -
.../static/js/jquery.dataTables.min.js | 388 --
.../cas-browser/static/js/querywidget.js | 314 --
balance/modules/cas-browser/views/dataset.php | 100 -
balance/modules/cas-browser/views/index.php | 87 -
balance/modules/cas-browser/views/product.php | 84 -
balance/modules/cas-browser/views/products.php | 156 -
balance/modules/cas-browser/views/search.php | 89 -
.../modules/cas-browser/views/typesearch.php | 121 -
balance/modules/profile/README.txt | 88 -
.../LDAPAuthenticationProvider.class.php | 263 --
.../classes/LDAPAuthorizationProvider.class.php | 113 -
.../NullAuthenticationProvider.class.php | 96 -
.../classes/NullAuthorizationProvider.class.php | 45 -
balance/modules/profile/config.ini | 180 -
balance/modules/profile/hooks.php | 91 -
balance/modules/profile/scripts/access.php | 107 -
balance/modules/profile/scripts/changePwd.php | 69 -
balance/modules/profile/scripts/login.php | 46 -
balance/modules/profile/scripts/logout.php | 36 -
balance/modules/profile/scripts/manage.php | 59 -
.../scripts/widgets/UserStatusWidget.php | 91 -
balance/modules/profile/static/css/profile.css | 150 -
.../profile/static/js/additional-methods.js | 259 --
.../profile/static/js/jquery.validate.min.js | 16 -
.../profile/static/js/jquery.validate.pack.js | 15 -
balance/modules/profile/views/changePwd.php | 76 -
balance/modules/profile/views/createUser.php | 184 -
balance/modules/profile/views/groups.php | 82 -
balance/modules/profile/views/index.php | 81 -
balance/modules/profile/views/login.php | 71 -
balance/modules/profile/views/manage.php | 87 -
.../profile/views/passwordChangeConfirmed.php | 44 -
balance/modules/puny/README.txt | 115 -
balance/modules/puny/classes/Puny.class.php | 166 -
.../puny/classes/Puny_Container.class.php | 85 -
.../puny/classes/Puny_Resource.class.php | 163 -
.../puny/classes/data/Puny_DataStore.class.php | 69 -
.../data/Puny_LocalFileDataStore.class.php | 79 -
.../classes/data/Puny_MongoDataStore.class.php | 75 -
.../classes/data/Puny_PdoDataStore.class.php | 111 -
.../parsers/Puny_MarkdownParser.class.php | 36 -
.../puny/classes/parsers/Puny_Parser.class.php | 40 -
balance/modules/puny/config.ini | 75 -
balance/modules/puny/hooks.php | 17 -
balance/modules/puny/libs/markdown.php | 2949 ---------------
balance/modules/puny/schemas/schema.pdo.mysql | 26 -
balance/modules/puny/scripts/login.php | 52 -
balance/modules/puny/scripts/logout.php | 32 -
balance/modules/puny/scripts/store.php | 40 -
balance/modules/puny/static/css/defaults.css | 39 -
balance/modules/puny/static/css/dialog.css | 157 -
balance/modules/puny/static/css/editor.css | 567 ---
balance/modules/puny/static/css/github.css | 677 ----
balance/modules/puny/static/css/puny.css | 32 -
balance/modules/puny/static/img/icon-sprite.png | Bin 9542 -> 0 bytes
balance/modules/puny/static/js/editable.js | 1090 ------
.../puny/static/js/editor/gollum.editor.js | 1090 ------
.../puny/static/js/editor/langs/markdown.js | 228 --
balance/modules/puny/static/js/gollum.dialog.js | 279 --
balance/modules/puny/static/js/gollum.js | 178 -
.../puny/static/js/gollum.placeholder.js | 71 -
.../modules/puny/static/js/jquery-1.7.2.min.js | 4 -
balance/modules/puny/static/js/puny.js | 56 -
balance/modules/puny/views/edit.php | 141 -
balance/modules/puny/views/error.php | 31 -
balance/modules/puny/views/login.php | 61 -
balance/modules/puny/views/logout.php | 32 -
balance/modules/puny/views/preview.php | 36 -
catalog/.gitignore | 1 -
catalog/pom.xml | 258 --
catalog/src/main/assembly/assembly.xml | 77 -
catalog/src/main/avro/CommunicationChannel.avpr | 481 ---
catalog/src/main/bin/catserv-client | 33 -
catalog/src/main/bin/catserv-server | 33 -
.../cli/action/CatalogServiceCliAction.java | 77 -
.../action/GetSupportedCatalogIdsCliAction.java | 40 -
.../action/LoadCatalogRepositoryCliAction.java | 80 -
.../cli/action/LoadCatalogsCliAction.java | 70 -
.../catalog/cli/action/PagedQueryCliAction.java | 116 -
.../cas/catalog/cli/action/QueryCliAction.java | 95 -
.../cli/action/ReducedPagedQueryCliAction.java | 134 -
.../cli/action/ReducedQueryCliAction.java | 110 -
.../catalog/cli/action/ShutdownCliAction.java | 38 -
.../exception/CatalogDictionaryException.java | 41 -
.../cas/catalog/exception/CatalogException.java | 40 -
.../exception/CatalogIndexException.java | 40 -
.../exception/CatalogRepositoryException.java | 40 -
.../exception/CatalogServiceException.java | 40 -
.../exception/IngestServiceException.java | 40 -
.../exception/QueryServiceException.java | 40 -
.../catalog/mapping/DataSourceIngestMapper.java | 259 --
.../mapping/DataSourceIngestMapperFactory.java | 91 -
.../catalog/mapping/InMemoryIngestMapper.java | 41 -
.../mapping/InMemoryIngestMapperFactory.java | 52 -
.../oodt/cas/catalog/mapping/IngestMapper.java | 157 -
.../catalog/mapping/IngestMapperFactory.java | 31 -
.../cas/catalog/mapping/LuceneIngestMapper.java | 114 -
.../mapping/LuceneIngestMapperFactory.java | 35 -
.../mapping/MemoryBasedIngestMapper.java | 281 --
.../mapping/MemoryBasedIngestMapperFactory.java | 34 -
.../catalog/metadata/TransactionalMetadata.java | 73 -
.../oodt/cas/catalog/page/CatalogReceipt.java | 83 -
.../oodt/cas/catalog/page/IndexPager.java | 77 -
.../oodt/cas/catalog/page/IngestReceipt.java | 56 -
.../org/apache/oodt/cas/catalog/page/Page.java | 77 -
.../apache/oodt/cas/catalog/page/PageInfo.java | 49 -
.../cas/catalog/page/ProcessedPageInfo.java | 47 -
.../oodt/cas/catalog/page/QueryPager.java | 73 -
.../cas/catalog/page/TransactionReceipt.java | 91 -
.../query/ComparisonQueryExpression.java | 84 -
.../catalog/query/CustomQueryExpression.java | 73 -
.../query/CustomWrapperQueryExpression.java | 58 -
.../catalog/query/FreeTextQueryExpression.java | 103 -
.../cas/catalog/query/NotQueryExpression.java | 54 -
.../oodt/cas/catalog/query/QueryExpression.java | 53 -
.../cas/catalog/query/QueryLogicalGroup.java | 117 -
.../cas/catalog/query/StdQueryExpression.java | 50 -
.../cas/catalog/query/TermQueryExpression.java | 44 -
.../catalog/query/WrapperQueryExpression.java | 57 -
.../catalog/query/filter/FilterAlgorithm.java | 35 -
.../cas/catalog/query/filter/QueryFilter.java | 60 -
.../query/filter/time/MetadataTimeEvent.java | 41 -
.../time/MetadataTimeEventFilterAlgorithm.java | 57 -
.../time/MetadataTimeEventQueryFilter.java | 112 -
.../conv/AsciiSortableVersionConverter.java | 39 -
.../filter/time/conv/VersionConverter.java | 32 -
.../query/parser/JJTQueryParserState.java | 123 -
.../oodt/cas/catalog/query/parser/Node.java | 36 -
.../catalog/query/parser/ParseException.java | 190 -
.../cas/catalog/query/parser/QueryParser.java | 1000 -----
.../cas/catalog/query/parser/QueryParser.jj | 497 ---
.../cas/catalog/query/parser/QueryParser.jjt | 214 --
.../query/parser/QueryParserConstants.java | 65 -
.../query/parser/QueryParserTokenManager.java | 806 ----
.../query/parser/QueryParserTreeConstants.java | 29 -
.../catalog/query/parser/SimpleCharStream.java | 480 ---
.../cas/catalog/query/parser/SimpleNode.java | 79 -
.../oodt/cas/catalog/query/parser/Token.java | 131 -
.../cas/catalog/query/parser/TokenMgrError.java | 146 -
.../catalog/repository/CatalogRepository.java | 77 -
.../repository/CatalogRepositoryFactory.java | 34 -
.../MemoryBasedCatalogRepository.java | 120 -
.../MemoryBasedCatalogRepositoryFactory.java | 38 -
.../repository/SerializedCatalogRepository.java | 212 --
.../SerializedCatalogRepositoryFactory.java | 54 -
.../repository/SpringCatalogRepository.java | 87 -
.../SpringCatalogRepositoryFactory.java | 43 -
.../server/CatalogServiceCommandLineClient.java | 46 -
.../server/CatalogServiceServerLauncher.java | 51 -
.../AbstractCommunicationChannelClient.java | 42 -
.../AbstractCommunicationChannelServer.java | 404 --
.../channel/CommunicationChannelClient.java | 127 -
.../CommunicationChannelClientFactory.java | 35 -
.../channel/CommunicationChannelServer.java | 154 -
.../CommunicationChannelServerFactory.java | 40 -
.../AvrorpcCommunicationChannelClient.java | 485 ---
...vrorpcCommunicationChannelClientFactory.java | 63 -
.../AvrorpcCommunicationChannelServer.java | 432 ---
...vrorpcCommunicationChannelServerFactory.java | 52 -
.../rmi/RmiCommunicationChannelClient.java | 271 --
.../RmiCommunicationChannelClientInterface.java | 109 -
.../rmi/RmiCommunicationChannelClientMBean.java | 27 -
.../rmi/RmiCommunicationChannelServer.java | 275 --
.../XmlRpcCommunicationChannelClient.java | 639 ----
...XmlRpcCommunicationChannelClientFactory.java | 93 -
.../XmlRpcCommunicationChannelServer.java | 261 --
...XmlRpcCommunicationChannelServerFactory.java | 77 -
.../action/CatalogServiceServerCliAction.java | 82 -
.../cli/action/LaunchXmlRpcServerCliAction.java | 68 -
.../oodt/cas/catalog/struct/Dictionary.java | 65 -
.../cas/catalog/struct/DictionaryFactory.java | 29 -
.../apache/oodt/cas/catalog/struct/Index.java | 65 -
.../oodt/cas/catalog/struct/IndexFactory.java | 29 -
.../oodt/cas/catalog/struct/IngestService.java | 74 -
.../oodt/cas/catalog/struct/QueryService.java | 80 -
.../oodt/cas/catalog/struct/TransactionId.java | 72 -
.../catalog/struct/TransactionIdFactory.java | 31 -
.../dictionary/WorkflowManagerDictionary.java | 71 -
.../WorkflowManagerDictionaryFactory.java | 34 -
.../struct/impl/index/DataSourceIndex.java | 549 ---
.../impl/index/DataSourceIndexFactory.java | 67 -
.../struct/impl/index/InMemoryIndex.java | 45 -
.../struct/impl/index/InMemoryIndexFactory.java | 47 -
.../index/WorkflowManagerDataSourceIndex.java | 323 --
.../WorkflowManagerDataSourceIndexFactory.java | 59 -
.../transaction/LongTransactionIdFactory.java | 52 -
.../transaction/StringTransactionIdFactory.java | 51 -
.../transaction/UuidTransactionIdFactory.java | 55 -
.../apache/oodt/cas/catalog/system/Catalog.java | 399 --
.../oodt/cas/catalog/system/CatalogFactory.java | 80 -
.../oodt/cas/catalog/system/CatalogService.java | 130 -
.../catalog/system/CatalogServiceFactory.java | 31 -
.../system/impl/CatalogServiceClient.java | 407 --
.../impl/CatalogServiceClientFactory.java | 66 -
.../system/impl/CatalogServiceLocal.java | 1336 -------
.../system/impl/CatalogServiceLocalFactory.java | 122 -
.../apache/oodt/cas/catalog/term/Bucket.java | 47 -
.../org/apache/oodt/cas/catalog/term/Term.java | 163 -
.../oodt/cas/catalog/term/TermBucket.java | 103 -
.../util/CasPropertyPlaceholderConfigurer.java | 62 -
.../cas/catalog/util/PluginClassLoader.java | 100 -
.../apache/oodt/cas/catalog/util/PluginURL.java | 47 -
.../oodt/cas/catalog/util/QueryUtils.java | 91 -
.../oodt/cas/catalog/util/Serializer.java | 99 -
.../oodt/cas/catalog/util/SpringUtils.java | 54 -
catalog/src/main/resources/REMOVE.log | 19 -
catalog/src/main/resources/logging.properties | 56 -
.../oracle/create-transaction-id-map-table.sql | 29 -
.../oracle/create-transaction-index-table.sql | 31 -
.../src/main/resources/policy/catserv-beans.xml | 71 -
.../policy/catserv-client-action-beans.xml | 51 -
.../policy/catserv-client-cmd-line-beans.xml | 302 --
.../resources/policy/catserv-client-config.xml | 29 -
.../resources/policy/catserv-properties.xml | 43 -
.../policy/catserv-server-cmd-line-beans.xml | 51 -
.../resources/policy/catserv-server-config.xml | 28 -
.../main/resources/policy/cmd-line-actions.xml | 51 -
.../main/resources/policy/cmd-line-options.xml | 317 --
.../policy/cmd-line-server-actions.xml | 30 -
.../policy/cmd-line-server-options.xml | 207 --
.../resources/policy/in-memory-index-cat.sql | 32 -
.../policy/in-memory-ingest-mapper.sql | 26 -
.../resources/policy/workflow-catalog-beans.xml | 56 -
.../TestAvrorpcCommunicationChannelServer.java | 145 -
.../system/impl/TestCatalogServiceLocal.java | 204 -
catalog/src/test/resources/test-index-cat.sql | 32 -
catalog/src/test/resources/test-mapper-cat.sql | 26 -
cli/pom.xml | 2 +-
commons/pom.xml | 2 +-
.../org/apache/oodt/commons/util/XMLTest.java | 4 +-
config/pom.xml | 140 +
config/src/main/assembly/assembly.xml | 97 +
.../java/org/apache/oodt/config/Component.java | 51 +
.../oodt/config/ConfigurationManager.java | 66 +
.../config/ConfigurationManagerFactory.java | 86 +
.../java/org/apache/oodt/config/Constants.java | 97 +
.../DistributedConfigurationManager.java | 230 ++
.../DistributedConfigurationPublisher.java | 273 ++
.../oodt/config/distributed/ZNodePaths.java | 102 +
.../oodt/config/distributed/cli/CLIAction.java | 126 +
.../config/distributed/cli/ConfigPublisher.java | 37 +
.../config/distributed/utils/ConfigUtils.java | 75 +
.../config/distributed/utils/CuratorUtils.java | 193 +
.../StandaloneConfigurationManager.java | 67 +
.../AbstractDistributedConfigurationTest.java | 51 +
config/src/main/resources/cmd-line-actions.xml | 38 +
config/src/main/resources/cmd-line-options.xml | 85 +
config/src/main/resources/config-publisher.xml | 80 +
config/src/main/resources/log4j.xml | 53 +
config/src/main/scripts/conf-publisher | 40 +
.../DistributedConfigurationManagerTest.java | 140 +
.../DistributedConfigurationPublisherTest.java | 128 +
.../oodt/config/distributed/TestServer.java | 45 +
config/src/test/resources/config-publisher.xml | 76 +
core/pom.xml | 45 +-
crawler/pom.xml | 2 +-
curator/services/pom.xml | 4 +-
.../cas/curation/service/IngestionResource.java | 300 +-
curator/sso/pom.xml | 74 +
.../sso/AbstractWebBasedSingleSignOn.java | 90 +
.../org/apache/oodt/security/sso/DummyImpl.java | 102 +
.../apache/oodt/security/sso/OpenSSOImpl.java | 191 +
.../apache/oodt/security/sso/SingleSignOn.java | 85 +
.../oodt/security/sso/SingleSignOnFactory.java | 68 +
.../security/sso/opensso/IdentityDetails.java | 136 +
.../oodt/security/sso/opensso/SSOMetKeys.java | 81 +
.../oodt/security/sso/opensso/SSOProxy.java | 354 ++
.../sso/opensso/SingleSignOnException.java | 11 +
.../oodt/security/sso/opensso/UserDetails.java | 97 +
curator/webapp/pom.xml | 4 +-
.../org/apache/oodt/cas/curation/HomePage.html | 2 +-
.../apache/oodt/cas/curation/HomePage_edrn.html | 2 +-
docker/Dockerfile | 22 -
filemgr/pom.xml | 15 +-
filemgr/src/main/assembly/assembly.xml | 49 +-
filemgr/src/main/bin/filemgr | 28 +-
.../filemgr/metadata/FileAttributesMetKeys.java | 53 +
.../examples/FileAttributesExtractor.java | 82 +
.../oodt/cas/filemgr/structs/Element.java | 10 +
.../oodt/cas/filemgr/structs/ProductPage.java | 2 +
.../oodt/cas/filemgr/structs/ProductType.java | 14 +
.../cas/filemgr/system/XmlRpcFileManager.java | 43 +-
.../oodt/cas/filemgr/tools/CatalogSearch.java | 1 -
.../oodt/cas/filemgr/tools/MetadataDumper.java | 1 -
filemgr/src/main/python/fm.conf | 43 -
filemgr/src/main/python/fm.py | 554 ---
filemgr/src/main/python/setclasspath.sh | 17 -
filemgr/src/main/resources/log4j.xml | 40 +
.../cas/filemgr/catalog/TestLuceneCatalog.java | 10 +-
.../examples/TestFileAttributesExtractor.java | 167 +
.../TestDistributedXmlRpcFileManager.java | 230 ++
.../distributed/config/config-publisher.xml | 45 +
.../distributed/config/filemgr.properties | 157 +
filemgr/src/test/resources/filemgr.properties | 155 -
grid/pom.xml | 138 -
.../grid/AuthenticationRequiredException.java | 30 -
.../java/org/apache/oodt/grid/ConfigBean.java | 177 -
.../org/apache/oodt/grid/ConfigServlet.java | 309 --
.../org/apache/oodt/grid/Configuration.java | 463 ---
.../java/org/apache/oodt/grid/GridServlet.java | 126 -
.../java/org/apache/oodt/grid/LoginServlet.java | 65 -
.../apache/oodt/grid/ProductQueryServlet.java | 226 --
.../org/apache/oodt/grid/ProductServer.java | 65 -
.../apache/oodt/grid/ProfileQueryServlet.java | 141 -
.../org/apache/oodt/grid/ProfileServer.java | 71 -
.../java/org/apache/oodt/grid/QueryServlet.java | 274 --
.../oodt/grid/RestfulProductQueryServlet.java | 88 -
.../main/java/org/apache/oodt/grid/Server.java | 166 -
.../main/java/org/apache/oodt/grid/Utility.java | 82 -
grid/src/main/webapp/META-INF/context.xml | 23 -
grid/src/main/webapp/WEB-INF/jetty-web.xml | 21 -
grid/src/main/webapp/WEB-INF/web.xml | 86 -
grid/src/main/webapp/config.jsp | 205 -
grid/src/main/webapp/error.jsp | 60 -
grid/src/main/webapp/images/webapp-large.png | Bin 3529 -> 0 bytes
grid/src/main/webapp/images/webapp-small.png | Bin 1222 -> 0 bytes
grid/src/main/webapp/index.html | 32 -
grid/src/main/webapp/style.css | 161 -
grid/src/site/resources/images/web-grid.jpg | Bin 45879 -> 0 bytes
grid/src/site/resources/images/web-grid.psd | Bin 741413 -> 0 bytes
grid/src/site/resources/slides.pdf | Bin 631046 -> 0 bytes
grid/src/site/site.xml | 30 -
grid/src/site/xdoc/index.xml | 73 -
.../org/apache/oodt/grid/ConfigurationTest.java | 180 -
metadata/pom.xml | 2 +-
mvn/archetypes/opsui/pom.xml | 2 +-
mvn/archetypes/pom.xml | 2 +-
mvn/archetypes/radix/pom.xml | 2 +-
.../radix/src/main/resources/bin/radix | 2 +-
mvn/plugins/cas-install/pom.xml | 2 +-
opendapps/pom.xml | 99 -
.../org/apache/oodt/opendapps/DapNames.java | 37 -
.../apache/oodt/opendapps/DatasetCrawler.java | 126 -
.../apache/oodt/opendapps/DatasetExtractor.java | 188 -
.../OpendapProfileElementExtractor.java | 175 -
.../oodt/opendapps/OpendapProfileHandler.java | 186 -
.../org/apache/oodt/opendapps/Profiler.java | 155 -
.../oodt/opendapps/config/ConstantSpec.java | 86 -
.../apache/oodt/opendapps/config/DapRoot.java | 88 -
.../oodt/opendapps/config/DatasetMetElem.java | 68 -
.../oodt/opendapps/config/OpendapConfig.java | 128 -
.../opendapps/config/OpendapConfigMetKeys.java | 87 -
.../opendapps/config/OpendapConfigReader.java | 133 -
.../opendapps/config/OpendapProfileMetKeys.java | 36 -
.../config/ProcessingInstructions.java | 102 -
.../oodt/opendapps/config/RewriteSpec.java | 86 -
.../extractors/DasMetadataExtractor.java | 186 -
.../opendapps/extractors/MetadataExtractor.java | 43 -
.../extractors/NcmlMetadataExtractor.java | 55 -
.../extractors/ThreddsMetadataExtractor.java | 315 --
.../opendapps/util/OpendapURLEvaluator.java | 48 -
.../oodt/opendapps/util/ProfileChecker.java | 186 -
.../oodt/opendapps/util/ProfileSerializer.java | 109 -
.../oodt/opendapps/util/ProfileUtils.java | 417 ---
opendapps/src/main/resources/logging.properties | 35 -
opendapps/src/main/resources/opendap.config.xml | 192 -
pcs/core/pom.xml | 2 +-
.../apache/oodt/pcs/util/FileManagerUtils.java | 113 +-
pcs/input/pom.xml | 2 +-
pcs/opsui/pom.xml | 2 +-
pcs/services/pom.xml | 2 +-
.../oodt/pcs/services/HealthResource.java | 18 +-
pge/pom.xml | 2 +-
.../cas/pge/staging/FileManagerFileStager.java | 1 +
.../oodt/cas/pge/TestPGETaskInstance.java | 3 +-
.../pge/staging/TestFileManagerFileStager.java | 41 +
pom.xml | 23 +-
product/.gitignore | 1 -
product/pom.xml | 138 -
product/src/main/conf/displayableTypes.xml | 50 -
product/src/main/conf/ofsn-ps.xml | 201 -
.../oodt/product/LargeProductQueryHandler.java | 57 -
.../org/apache/oodt/product/QueryHandler.java | 38 -
.../handlers/ofsn/AbstractCrawlLister.java | 116 -
.../ofsn/DirListNonRecursiveHandler.java | 60 -
.../handlers/ofsn/DirListRecursiveHandler.java | 63 -
.../ofsn/FileListNonRecursiveHandler.java | 56 -
.../handlers/ofsn/FileListRecursiveHandler.java | 55 -
.../product/handlers/ofsn/MD5GetHandler.java | 143 -
.../product/handlers/ofsn/OFSNFileHandler.java | 255 --
.../ofsn/OFSNFileHandlerConfiguration.java | 131 -
.../OFSNFileHandlerConfigurationReader.java | 93 -
.../product/handlers/ofsn/OFSNGetHandler.java | 73 -
.../handlers/ofsn/OFSNHandlerConfig.java | 121 -
.../product/handlers/ofsn/OFSNListHandler.java | 61 -
.../handlers/ofsn/RawSizeListHandler.java | 60 -
.../handlers/ofsn/SingleFileListHandler.java | 64 -
.../handlers/ofsn/SingleZipFileListHandler.java | 83 -
.../handlers/ofsn/StdOFSNGetHandler.java | 112 -
.../product/handlers/ofsn/URLGetHandler.java | 252 --
.../handlers/ofsn/metadata/OFSNMetKeys.java | 41 -
.../ofsn/metadata/OFSNXMLConfigMetKeys.java | 55 -
.../handlers/ofsn/metadata/OFSNXMLMetKeys.java | 43 -
.../handlers/ofsn/metadata/OODTMetKeys.java | 31 -
.../handlers/ofsn/metadata/XMLQueryMetKeys.java | 32 -
.../handlers/ofsn/util/OFSNObjectFactory.java | 112 -
.../product/handlers/ofsn/util/OFSNUtils.java | 197 -
.../xmlquery/ChunkedProductInputStream.java | 215 --
.../org/apache/oodt/xmlquery/LargeResult.java | 166 -
.../src/site/resources/images/delegation.png | Bin 59132 -> 0 bytes
.../src/site/resources/images/delegation.psd | Bin 277484 -> 0 bytes
.../src/site/resources/images/grid-product.jpg | Bin 55049 -> 0 bytes
product/src/site/site.xml | 32 -
product/src/site/xdoc/servlet/index.xml | 243 --
product/src/site/xdoc/tutorials/index.xml | 36 -
product/src/site/xdoc/tutorials/lh/index.xml | 718 ----
product/src/site/xdoc/tutorials/ps/index.xml | 481 ---
product/src/site/xdoc/tutorials/qh/index.xml | 704 ----
.../handlers/ofsn/util/OFSNUtilsTest.java | 59 -
.../xmlquery/ChunkedProductInputStreamTest.java | 170 -
.../apache/oodt/xmlquery/LargeResultTest.java | 54 -
profile/pom.xml | 155 -
profile/src/main/dtd/prof.dtd | 86 -
.../org/apache/oodt/profile/DefaultFactory.java | 56 -
.../oodt/profile/EnumeratedProfileElement.java | 106 -
.../org/apache/oodt/profile/ObjectFactory.java | 109 -
.../java/org/apache/oodt/profile/Profile.java | 401 --
.../apache/oodt/profile/ProfileAttributes.java | 344 --
.../org/apache/oodt/profile/ProfileElement.java | 586 ---
.../apache/oodt/profile/ProfileException.java | 57 -
.../oodt/profile/ProfileSAXException.java | 37 -
.../oodt/profile/ProfileSQLException.java | 34 -
.../oodt/profile/RangedProfileElement.java | 115 -
.../apache/oodt/profile/ResourceAttributes.java | 602 ---
.../oodt/profile/UnspecifiedProfileElement.java | 89 -
.../java/org/apache/oodt/profile/Utility.java | 254 --
.../profile/activity/ConnectedToServer.java | 26 -
.../activity/ProfileClientConstructed.java | 26 -
.../oodt/profile/activity/QueriedHandlers.java | 26 -
.../oodt/profile/activity/ReceivedQuery.java | 26 -
.../apache/oodt/profile/gui/LeafListener.java | 288 --
.../oodt/profile/gui/ProfileBuilderGUI.form | 435 ---
.../oodt/profile/gui/ProfileBuilderGUI.java | 962 -----
.../apache/oodt/profile/gui/profileTree.java | 41 -
.../gui/pstructs/ProfileAttributesPrinter.java | 75 -
.../gui/pstructs/ProfileElementPrinter.java | 64 -
.../profile/gui/pstructs/ProfilePrinter.java | 77 -
.../gui/pstructs/ResourceAttributesPrinter.java | 118 -
.../handlers/DatabaseProfileManager.java | 277 --
.../oodt/profile/handlers/ProfileHandler.java | 64 -
.../oodt/profile/handlers/ProfileManager.java | 149 -
.../profile/handlers/cas/CASProfileHandler.java | 292 --
.../profile/handlers/cas/util/ProfileUtils.java | 107 -
.../handlers/lightweight/AndExpression.java | 67 -
.../lightweight/ConstantExpression.java | 65 -
.../handlers/lightweight/FalseResult.java | 50 -
.../handlers/lightweight/Intersection.java | 60 -
.../lightweight/LightweightProfileServer.java | 306 --
.../handlers/lightweight/MatchingResult.java | 61 -
.../handlers/lightweight/NotExpression.java | 59 -
.../lightweight/OperatorExpression.java | 140 -
.../handlers/lightweight/OrExpression.java | 67 -
.../profile/handlers/lightweight/Result.java | 35 -
.../SearchableEnumeratedProfileElement.java | 50 -
.../lightweight/SearchableObjectFactory.java | 62 -
.../handlers/lightweight/SearchableProfile.java | 44 -
.../lightweight/SearchableProfileElement.java | 36 -
.../SearchableRangedProfileElement.java | 66 -
.../SearchableResourceAttributes.java | 142 -
.../SearchableUnspecifiedProfileElement.java | 39 -
.../profile/handlers/lightweight/Union.java | 58 -
.../handlers/lightweight/WhereExpression.java | 55 -
.../profile/handlers/lightweight/package.html | 32 -
profile/src/site/resources/examples/src.jar | Bin 7973 -> 0 bytes
profile/src/site/resources/images/class.png | Bin 120990 -> 0 bytes
profile/src/site/resources/images/class.psd | Bin 499375 -> 0 bytes
.../src/site/resources/images/delegation.png | Bin 48258 -> 0 bytes
.../src/site/resources/images/delegation.psd | Bin 254613 -> 0 bytes
.../src/site/resources/images/grid-profile.jpg | Bin 54712 -> 0 bytes
profile/src/site/resources/images/stack.png | Bin 44350 -> 0 bytes
profile/src/site/resources/images/stack.psd | Bin 256409 -> 0 bytes
profile/src/site/resources/images/tree.png | Bin 27643 -> 0 bytes
profile/src/site/resources/images/tree.psd | Bin 129734 -> 0 bytes
profile/src/site/site.xml | 37 -
profile/src/site/xdoc/adv/index.xml | 1072 ------
profile/src/site/xdoc/handler/index.xml | 595 ---
profile/src/site/xdoc/info/index.xml | 848 -----
profile/src/site/xdoc/querying/index.xml | 203 -
profile/src/site/xdoc/rep/index.xml | 334 --
.../profile/EnumeratedProfileElementTest.java | 130 -
.../oodt/profile/ProfileAttributesTest.java | 167 -
.../oodt/profile/ProfileElementTestCase.java | 187 -
.../org/apache/oodt/profile/ProfileTest.java | 130 -
.../oodt/profile/RangedProfileElementTest.java | 82 -
.../oodt/profile/ResourceAttributesTest.java | 178 -
.../LightweightProfileServerTest.java | 158 -
.../handlers/lightweight/lightweightTest.xml | 119 -
.../resources/org/apache/oodt/profile/test.xml | 71 -
profile/src/test/resources/test.xml | 71 -
protocol/api/.gitignore | 1 -
protocol/api/pom.xml | 147 -
protocol/api/src/main/assembly/assembly.xml | 94 -
protocol/api/src/main/bin/protocol | 29 -
.../org/apache/oodt/cas/protocol/Protocol.java | 60 -
.../oodt/cas/protocol/ProtocolFactory.java | 31 -
.../apache/oodt/cas/protocol/ProtocolFile.java | 142 -
.../oodt/cas/protocol/auth/Authentication.java | 30 -
.../cas/protocol/auth/BasicAuthentication.java | 46 -
.../cas/protocol/auth/NoAuthentication.java | 34 -
.../cli/action/BasicVerifyCliAction.java | 83 -
.../action/CrossProtocolTransferCliAction.java | 123 -
.../action/DeleteEmptyDirectoriesCliAction.java | 70 -
.../protocol/cli/action/DownloadCliAction.java | 81 -
.../action/GetSupportedFactoriesCliAction.java | 43 -
.../protocol/cli/action/ProtocolCliAction.java | 84 -
.../cas/protocol/config/ProtocolConfig.java | 39 -
.../protocol/config/SpringProtocolConfig.java | 77 -
.../protocol/exceptions/ProtocolException.java | 34 -
.../protocol/system/ProtocolCommandLine.java | 31 -
.../cas/protocol/system/ProtocolManager.java | 116 -
.../cas/protocol/util/ProtocolFileFilter.java | 32 -
.../protocol/verify/BasicProtocolVerifier.java | 84 -
.../verify/BasicProtocolVerifierFactory.java | 47 -
.../cas/protocol/verify/ProtocolVerifier.java | 35 -
.../verify/ProtocolVerifierFactory.java | 28 -
.../main/resources/policy/cmd-line-actions.xml | 42 -
.../main/resources/policy/cmd-line-options.xml | 166 -
.../main/resources/policy/protocol-config.xml | 31 -
.../apache/oodt/cas/protocol/MockProtocol.java | 130 -
.../oodt/cas/protocol/MockProtocolFactory.java | 51 -
.../oodt/cas/protocol/TestProtocolFile.java | 53 -
.../protocol/auth/TestBasicAuthentication.java | 49 -
.../cas/protocol/auth/TestNoAuthentication.java | 35 -
.../cli/action/MockProtocolCliAction.java | 33 -
.../cli/action/TestBasicVerifyCliAction.java | 53 -
.../cli/action/TestDownloadCliAction.java | 76 -
.../cli/action/TestProtocolCliAction.java | 41 -
.../config/MockSpringProtocolConfig.java | 44 -
.../protocol/system/TestProtocolManager.java | 95 -
.../src/test/resources/test-protocol-config.xml | 53 -
protocol/ftp/.gitignore | 1 -
protocol/ftp/pom.xml | 122 -
.../cas/protocol/ftp/CogJGlobusFtpProtocol.java | 226 --
.../ftp/CogJGlobusFtpProtocolFactory.java | 39 -
.../cas/protocol/ftp/CommonsNetFtpProtocol.java | 246 --
.../ftp/CommonsNetFtpProtocolFactory.java | 38 -
.../resources/policy/ftp-protocol-config.xml | 32 -
.../protocol/ftp/TestCogJGlobusFtpProtocol.java | 137 -
.../ftp/src/test/resources/users.properties | 26 -
protocol/http/.gitignore | 1 -
protocol/http/pom.xml | 105 -
.../apache/oodt/cas/protocol/http/HttpFile.java | 51 -
.../oodt/cas/protocol/http/HttpProtocol.java | 242 --
.../cas/protocol/http/HttpProtocolFactory.java | 37 -
.../oodt/cas/protocol/http/util/HttpUtils.java | 130 -
.../resources/policy/http-protocol-config.xml | 31 -
.../oodt/cas/protocol/http/TestHttpFile.java | 53 -
.../cas/protocol/http/TestHttpProtocol.java | 103 -
.../cas/protocol/http/util/TestHttpUtils.java | 173 -
protocol/imaps/.gitignore | 1 -
protocol/imaps/pom.xml | 114 -
.../oodt/cas/protocol/imaps/ImapsProtocol.java | 385 --
.../protocol/imaps/ImapsProtocolFactory.java | 43 -
.../resources/policy/imaps-protocol-config.xml | 31 -
.../cas/protocol/imaps/TestImapsProtocol.java | 128 -
protocol/sftp/.gitignore | 1 -
protocol/sftp/pom.xml | 140 -
.../cas/protocol/sftp/JschSftpProtocol.java | 211 --
.../protocol/sftp/JschSftpProtocolFactory.java | 48 -
.../sftp/auth/HostKeyAuthentication.java | 49 -
.../resources/policy/sftp-protocol-config.xml | 23 -
.../sftp/DummyAuthenticationProvider.java | 71 -
.../cas/protocol/sftp/TestJschSftpProtocol.java | 298 --
.../sftp/UnsupportedShellProcessProvider.java | 114 -
.../sftp/auth/TestHostKeyAuthentication.java | 36 -
.../sftp/src/test/resources/authorization.xml | 24 -
protocol/sftp/src/test/resources/platform.xml | 35 -
protocol/sftp/src/test/resources/sample-dsa.key | 12 -
protocol/sftp/src/test/resources/sample-dsa.pub | 1 -
protocol/sftp/src/test/resources/server.xml | 28 -
.../src/test/resources/sshTestDir/sshTestFile | 18 -
pushpull/pom.xml | 181 -
pushpull/src/main/assembly/assembly.xml | 85 -
pushpull/src/main/bin/pushpull | 32 -
pushpull/src/main/bin/url-downloader | 21 -
.../apache/oodt/cas/pushpull/config/Config.java | 405 --
.../oodt/cas/pushpull/config/ConfigMetKeys.java | 71 -
.../pushpull/config/ConfigParserMetKeys.java | 113 -
.../oodt/cas/pushpull/config/DaemonInfo.java | 131 -
.../oodt/cas/pushpull/config/DataFilesInfo.java | 47 -
.../oodt/cas/pushpull/config/DownloadInfo.java | 86 -
.../oodt/cas/pushpull/config/ParserInfo.java | 104 -
.../oodt/cas/pushpull/config/PropFilesInfo.java | 164 -
.../oodt/cas/pushpull/config/ProtocolInfo.java | 168 -
.../oodt/cas/pushpull/config/RemoteSpecs.java | 364 --
.../oodt/cas/pushpull/config/SiteInfo.java | 120 -
.../apache/oodt/cas/pushpull/daemon/Daemon.java | 576 ---
.../cas/pushpull/daemon/DaemonController.java | 208 --
.../cas/pushpull/daemon/DaemonLauncher.java | 194 -
.../pushpull/daemon/DaemonLauncherMBean.java | 38 -
.../cas/pushpull/daemon/DaemonListener.java | 40 -
.../oodt/cas/pushpull/daemon/DaemonMBean.java | 95 -
.../oodt/cas/pushpull/daemon/DaemonManager.java | 123 -
.../cas/pushpull/daemon/DaemonRmiInterface.java | 97 -
.../exceptions/AlreadyInDatabaseException.java | 41 -
.../pushpull/exceptions/ConfigException.java | 45 -
.../exceptions/CrawlDaemonException.java | 41 -
.../pushpull/exceptions/CrawlerException.java | 45 -
.../exceptions/FileRestrictionsException.java | 41 -
.../exceptions/MetExtractorException.java | 41 -
.../pushpull/exceptions/MethodException.java | 41 -
.../pushpull/exceptions/ParserException.java | 45 -
.../exceptions/ProtocolDeletionError.java | 41 -
.../pushpull/exceptions/ProtocolException.java | 42 -
.../exceptions/ProtocolFileException.java | 42 -
.../exceptions/PushPullFrameworkException.java | 57 -
.../exceptions/RemoteConnectionException.java | 45 -
.../exceptions/RetrievalMethodException.java | 45 -
.../exceptions/ThreadEvaluatorException.java | 41 -
.../ToManyFailedDownloadsException.java | 41 -
.../exceptions/UndefinedTypeException.java | 41 -
.../pushpull/expressions/GlobalVariables.java | 40 -
.../oodt/cas/pushpull/expressions/Method.java | 301 --
.../oodt/cas/pushpull/expressions/Operator.java | 45 -
.../cas/pushpull/expressions/Punctuation.java | 45 -
.../cas/pushpull/expressions/ValidInput.java | 36 -
.../oodt/cas/pushpull/expressions/Variable.java | 126 -
.../filerestrictions/FileRestrictions.java | 518 ---
.../cas/pushpull/filerestrictions/Parser.java | 43 -
.../pushpull/filerestrictions/VirtualFile.java | 315 --
.../filerestrictions/VirtualFileStructure.java | 66 -
.../parsers/ClassNoaaEmailParser.java | 123 -
.../parsers/DirStructXmlParser.java | 346 --
.../parsers/FileListParser.java | 59 -
.../parsers/GenericEmailParser.java | 168 -
.../renamingconventions/RenamingConvention.java | 168 -
.../objectfactory/PushPullObjectFactory.java | 76 -
.../cas/pushpull/protocol/ProtocolHandler.java | 603 ---
.../cas/pushpull/protocol/ProtocolPath.java | 142 -
.../oodt/cas/pushpull/protocol/RemoteSite.java | 133 -
.../cas/pushpull/protocol/RemoteSiteFile.java | 87 -
.../pushpull/retrievalmethod/ListRetriever.java | 111 -
.../pushpull/retrievalmethod/RemoteCrawler.java | 174 -
.../retrievalmethod/RetrievalMethod.java | 45 -
.../DataFileToPropFileLinker.java | 183 -
.../retrievalsystem/DownloadListener.java | 41 -
.../DownloadThreadEvaluator.java | 164 -
.../retrievalsystem/DownloadingFileInfo.java | 66 -
.../retrievalsystem/FileRetrievalSystem.java | 966 -----
.../pushpull/retrievalsystem/RemoteFile.java | 97 -
.../retrievalsystem/RemoteFileMetKeys.java | 57 -
.../retrievalsystem/RetrievalSetup.java | 267 --
.../retrievalsystem/TimeAndThreadCount.java | 48 -
.../pushpull/url/handlers/imaps/Handler.java | 41 -
.../url/handlers/imaps/ImapsURLConnection.java | 46 -
.../cas/pushpull/url/handlers/sftp/Handler.java | 41 -
.../url/handlers/sftp/SftpURLConnection.java | 44 -
.../cas/pushpull/util/ExpressionValidator.java | 131 -
pushpull/src/main/resources/REMOVE.log | 18 -
pushpull/src/main/resources/default.properties | 21 -
.../Push-Pull-User-Manual_2008-02-12.doc | Bin 431616 -> 0 bytes
.../ClassNoaaEmailParserFiles/IasiEmail | 129 -
.../DirStructXmlParserFiles/AIRABRAD.xml | 26 -
.../DirStructXmlParserFiles/AIRS3ST8.xml | 23 -
.../DirStructXmlParserFiles/AIRS3STD.xml | 23 -
.../DirStructXmlParserFiles/CLASS_Email.xml | 22 -
.../CLASS_Text_Email.xml | 26 -
.../DirStructXmlParserFiles/CloudSAT.xml | 20 -
.../examples/DirStructXmlParserFiles/GFS.xml | 30 -
.../examples/DirStructXmlParserFiles/IASI.xml | 22 -
.../examples/DirStructXmlParserFiles/IASI_2.xml | 22 -
.../DirStructXmlParserFiles/IBIBLIO.xml | 25 -
.../examples/DirStructXmlParserFiles/MODIS.xml | 23 -
.../DirStructXmlParserFiles/RTG_SST.xml | 24 -
.../ExternalSourcesFiles/ExternalSources.xml | 75 -
.../FileListParserFiles/DownloadList.txt | 7 -
.../examples/ProcmailForwarding/[dot]forward | 1 -
.../ProcmailForwarding/[dot]procmail/general.rc | 25 -
.../examples/ProcmailForwarding/[dot]procmailrc | 7 -
.../examples/RemoteSpecsFiles/RemoteSpecs.xml | 181 -
pushpull/src/main/resources/jssecacerts | Bin 113900 -> 0 bytes
pushpull/src/main/resources/logging.properties | 43 -
.../policy/ParserToRetrievalMethodMap.xml | 30 -
.../resources/policy/ProtocolFactoryInfo.xml | 45 -
.../src/main/resources/policy/mimetypes.xml | 60 -
.../resources/push_pull_framework.properties | 106 -
.../src/site/resources/images/cas-pushpull.jpg | Bin 72076 -> 0 bytes
.../src/site/resources/images/cas-pushpull.psd | Bin 1013848 -> 0 bytes
.../resources/images/pp_extension_points.png | Bin 50656 -> 0 bytes
.../site/resources/images/pp_object_model.png | Bin 15808 -> 0 bytes
pushpull/src/site/site.xml | 32 -
.../src/site/xdoc/development/developer.xml | 286 --
pushpull/src/site/xdoc/development/maven.xml | 176 -
pushpull/src/site/xdoc/user/advanced.xml | 50 -
pushpull/src/site/xdoc/user/basic.xml | 50 -
pushpull/src/site/xdoc/user/index.xml | 49 -
.../parsers/GenericEmailParserTest.java | 75 -
pushpull/src/test/resources/TestEmail.txt | 16 -
resource/pom.xml | 10 +-
.../resource/jobqueue/FifoMappedJobQueue.java | 510 +++
.../jobqueue/FifoMappedJobQueueFactory.java | 75 +
.../cas/resource/jobqueue/MappedJobQueue.java | 132 +
.../apache/oodt/cas/resource/structs/Job.java | 12 +
.../oodt/cas/resource/structs/JobInput.java | 8 +
.../cas/resource/structs/NameValueJobInput.java | 17 +
.../resource/system/XmlRpcResourceManager.java | 48 +-
.../jobqueue/TestFifoMappedJobQueue.java | 101 +
.../system/TestXmlRpcResourceManager.java | 19 +-
.../TestDistributedXmlRpcResourceManager.java | 83 +
.../config/distributed/config-publisher.xml | 40 +
.../config/distributed/resource.properties | 75 +
sso/pom.xml | 74 -
.../sso/AbstractWebBasedSingleSignOn.java | 90 -
.../org/apache/oodt/security/sso/DummyImpl.java | 102 -
.../apache/oodt/security/sso/OpenSSOImpl.java | 190 -
.../apache/oodt/security/sso/SingleSignOn.java | 85 -
.../oodt/security/sso/SingleSignOnFactory.java | 68 -
.../security/sso/opensso/IdentityDetails.java | 136 -
.../oodt/security/sso/opensso/SSOMetKeys.java | 81 -
.../oodt/security/sso/opensso/SSOProxy.java | 354 --
.../sso/opensso/SingleSignOnException.java | 11 -
.../oodt/security/sso/opensso/UserDetails.java | 97 -
sso/src/main/php/pear/Config.php | 65 -
sso/src/main/php/pear/SingleSignOn.php | 202 -
sso/src/main/php/pear/package.xml | 94 -
tools/pdi_plugin/build-res/subfloor-pkg.xml | 367 --
tools/pdi_plugin/build-res/subfloor.xml | 2026 ----------
tools/pdi_plugin/build.properties | 13 -
tools/pdi_plugin/build.xml | 68 -
tools/pdi_plugin/build/build.properties | 3 -
tools/pdi_plugin/build/build.xml | 83 -
tools/pdi_plugin/ivy.xml | 37 -
tools/pdi_plugin/ivysettings.xml | 32 -
tools/pdi_plugin/package-ivy.xml | 15 -
tools/pdi_plugin/plugins | Bin 592575 -> 0 bytes
.../oodt/filemgrcheck/FilemgrCheckStep.java | 199 -
.../oodt/filemgrcheck/FilemgrCheckStepData.java | 55 -
.../filemgrcheck/FilemgrCheckStepDialog.java | 307 --
.../oodt/filemgrcheck/FilemgrCheckStepMeta.java | 345 --
.../apache/oodt/filemgrcheck/OODTConfig.java | 98 -
.../apache/oodt/filemgrcheck/OODTProcesses.java | 36 -
.../filemgrcheck/PushPullObjectFactory.java | 74 -
.../messages/messages_en_US.properties | 6 -
.../oodt/filemgrcheck/resources/check/oodt.jpg | Bin 25487 -> 0 bytes
.../oodt/filemgrcheck/resources/check/oodt.png | 0
.../oodt/filemgrdelete/FilemgrDeleteStep.java | 195 -
.../filemgrdelete/FilemgrDeleteStepData.java | 56 -
.../filemgrdelete/FilemgrDeleteStepDialog.java | 309 --
.../filemgrdelete/FilemgrDeleteStepMeta.java | 349 --
.../apache/oodt/filemgrdelete/OODTConfig.java | 57 -
.../oodt/filemgrdelete/OODTProcesses.java | 57 -
.../filemgrdelete/PushPullObjectFactory.java | 74 -
.../messages/messages_en_US.properties | 11 -
.../apache/oodt/filemgrget/FilemgrGetStep.java | 244 --
.../oodt/filemgrget/FilemgrGetStepData.java | 50 -
.../oodt/filemgrget/FilemgrGetStepDialog.java | 438 ---
.../oodt/filemgrget/FilemgrGetStepMeta.java | 410 --
.../org/apache/oodt/filemgrget/OODTConfig.java | 60 -
.../apache/oodt/filemgrget/OODTProcesses.java | 130 -
.../src/org/apache/oodt/filemgrget/Process.java | 37 -
.../src/org/apache/oodt/filemgrget/Search.java | 37 -
.../messages/messages_en_US.properties | 9 -
.../oodt/filemgrget/resources/get/oodt.jpg | Bin 25487 -> 0 bytes
.../oodt/filemgrget/resources/get/oodt.png | 0
.../oodt/filemgringest/FilemgrIngestStep.java | 216 --
.../filemgringest/FilemgrIngestStepData.java | 56 -
.../filemgringest/FilemgrIngestStepDialog.java | 388 --
.../filemgringest/FilemgrIngestStepMeta.java | 358 --
.../apache/oodt/filemgringest/OODTConfig.java | 98 -
.../oodt/filemgringest/OODTProcesses.java | 86 -
.../filemgringest/PushPullObjectFactory.java | 75 -
.../messages/messages_en_US.properties | 9 -
.../oodt/filemgringest/resources/oodt.jpg | Bin 25487 -> 0 bytes
.../oodt/filemgringest/resources/oodt.png | 0
vagrant/radix/README.txt | 51 -
vagrant/radix/Vagrantfile | 129 -
vagrant/radix/vagrant/conf/terminal/bashrc | 110 -
vagrant/radix/vagrant/env.sh | 28 -
vagrant/radix/vagrant/provision.sh | 70 -
webapp/components/pom.xml | 2 +-
.../curation/workbench/Workbench.js | 4 +
.../webcomponents/filemgr/FileManagerConn.java | 42 +-
webapp/fmbrowser/pom.xml | 4 +-
webapp/fmprod/pom.xml | 25 +-
webapp/fmprod/src/conf/mime.properties | 407 ++
webapp/fmprod/src/dtd/query.dtd | 100 +
.../fmprod/src/main/conf/displayableTypes.xml | 50 +
webapp/fmprod/src/main/conf/mime.properties | 407 ++
webapp/fmprod/src/main/conf/ofsn-ps.xml | 201 +
webapp/fmprod/src/main/dtd/query.dtd | 100 +
.../cas/product/data/DataDeliveryServlet.java | 16 +-
.../oodt/cas/product/rss/RSSProductServlet.java | 5 +
.../oodt/product/HttpRedirectException.java | 40 +
.../oodt/product/LargeProductQueryHandler.java | 57 +
.../apache/oodt/product/ProductException.java | 62 +
.../org/apache/oodt/product/QueryHandler.java | 38 +
.../java/org/apache/oodt/product/Retriever.java | 46 +
.../handlers/ofsn/AbstractCrawlLister.java | 116 +
.../ofsn/DirListNonRecursiveHandler.java | 60 +
.../handlers/ofsn/DirListRecursiveHandler.java | 63 +
.../ofsn/FileListNonRecursiveHandler.java | 56 +
.../handlers/ofsn/FileListRecursiveHandler.java | 55 +
.../product/handlers/ofsn/MD5GetHandler.java | 143 +
.../product/handlers/ofsn/OFSNFileHandler.java | 265 ++
.../ofsn/OFSNFileHandlerConfiguration.java | 131 +
.../OFSNFileHandlerConfigurationReader.java | 93 +
.../product/handlers/ofsn/OFSNGetHandler.java | 73 +
.../handlers/ofsn/OFSNHandlerConfig.java | 121 +
.../product/handlers/ofsn/OFSNListHandler.java | 61 +
.../handlers/ofsn/RawSizeListHandler.java | 60 +
.../handlers/ofsn/SingleFileListHandler.java | 64 +
.../handlers/ofsn/SingleZipFileListHandler.java | 83 +
.../handlers/ofsn/StdOFSNGetHandler.java | 112 +
.../product/handlers/ofsn/URLGetHandler.java | 252 ++
.../handlers/ofsn/metadata/OFSNMetKeys.java | 41 +
.../ofsn/metadata/OFSNXMLConfigMetKeys.java | 55 +
.../handlers/ofsn/metadata/OFSNXMLMetKeys.java | 43 +
.../handlers/ofsn/metadata/OODTMetKeys.java | 31 +
.../handlers/ofsn/metadata/XMLQueryMetKeys.java | 32 +
.../handlers/ofsn/util/OFSNObjectFactory.java | 112 +
.../product/handlers/ofsn/util/OFSNUtils.java | 197 +
.../apache/oodt/xmlquery/ByteArrayCodec.java | 63 +
.../xmlquery/ChunkedProductInputStream.java | 215 ++
.../java/org/apache/oodt/xmlquery/Codec.java | 85 +
.../org/apache/oodt/xmlquery/CodecFactory.java | 60 +
.../oodt/xmlquery/CompressedObjectCodec.java | 110 +
.../oodt/xmlquery/CompressedStringCodec.java | 91 +
.../java/org/apache/oodt/xmlquery/Header.java | 211 ++
.../org/apache/oodt/xmlquery/LargeResult.java | 166 +
.../org/apache/oodt/xmlquery/ObjectCodec.java | 106 +
.../org/apache/oodt/xmlquery/QueryElement.java | 148 +
.../apache/oodt/xmlquery/QueryException.java | 37 +
.../org/apache/oodt/xmlquery/QueryHeader.java | 304 ++
.../org/apache/oodt/xmlquery/QueryResult.java | 151 +
.../java/org/apache/oodt/xmlquery/Result.java | 447 +++
.../java/org/apache/oodt/xmlquery/Results.java | 60 +
.../org/apache/oodt/xmlquery/Statistic.java | 155 +
.../org/apache/oodt/xmlquery/StringCodec.java | 51 +
.../oodt/xmlquery/UnsupportedMimeTypeCodec.java | 44 +
.../java/org/apache/oodt/xmlquery/XMLQuery.java | 981 +++++
.../xmlquery/activity/QueryConstructed.java | 23 +
.../oodt/xmlquery/activity/ResultsReceived.java | 23 +
.../java/org/apache/oodt/xmlquery/package.html | 26 +
.../src/site/resources/images/delegation.png | Bin 0 -> 59132 bytes
.../src/site/resources/images/delegation.psd | Bin 0 -> 277484 bytes
.../src/site/resources/images/edm-query.jpg | Bin 0 -> 57650 bytes
.../src/site/resources/images/grid-product.jpg | Bin 0 -> 55049 bytes
.../src/site/resources/images/large-stack.png | Bin 0 -> 41424 bytes
.../src/site/resources/images/large-stack.psd | Bin 0 -> 282874 bytes
.../src/site/resources/images/results.png | Bin 0 -> 53580 bytes
.../src/site/resources/images/results.psd | Bin 0 -> 275480 bytes
.../src/site/resources/images/small-stack.png | Bin 0 -> 12368 bytes
.../src/site/resources/images/small-stack.psd | Bin 0 -> 89725 bytes
.../src/site/resources/images/xmlquery.png | Bin 0 -> 85734 bytes
.../src/site/resources/images/xmlquery.psd | Bin 0 -> 352069 bytes
webapp/fmprod/src/site/site.xml | 33 +
webapp/fmprod/src/site/xdoc/disquery/index.xml | 106 +
webapp/fmprod/src/site/xdoc/servlet/index.xml | 640 ++++
webapp/fmprod/src/site/xdoc/tutorial/index.xml | 659 ++++
webapp/fmprod/src/site/xdoc/tutorials/index.xml | 36 +
.../fmprod/src/site/xdoc/tutorials/lh/index.xml | 718 ++++
.../fmprod/src/site/xdoc/tutorials/ps/index.xml | 481 +++
.../fmprod/src/site/xdoc/tutorials/qh/index.xml | 704 ++++
.../apache/oodt/xmlquery/CodecFactoryTest.java | 61 +
.../org/apache/oodt/xmlquery/CodecTest.java | 77 +
.../xmlquery/CompressedObjectCodecTest.java | 42 +
.../xmlquery/CompressedStringCodecTest.java | 37 +
.../oodt/xmlquery/EmptyByteArrayCodecTest.java | 51 +
.../org/apache/oodt/xmlquery/HeaderTest.java | 142 +
.../apache/oodt/xmlquery/ObjectCodecTest.java | 42 +
.../apache/oodt/xmlquery/QueryElementTest.java | 93 +
.../org/apache/oodt/xmlquery/ResultTest.java | 166 +
.../apache/oodt/xmlquery/StringCodecTest.java | 37 +
.../org/apache/oodt/xmlquery/XMLQueryTest.java | 225 ++
.../handlers/ofsn/util/OFSNUtilsTest.java | 59 +
.../xmlquery/ChunkedProductInputStreamTest.java | 170 +
.../apache/oodt/xmlquery/LargeResultTest.java | 54 +
webapp/wmonitor/pom.xml | 4 +-
webapp/wmservices/pom.xml | 90 +
.../cas/wmservices/client/WmServicesClient.java | 132 +
.../repository/PackagedWorkflowManager.java | 275 ++
.../AbstractWorkflowServiceResource.java | 88 +
.../wmservices/resources/WorkflowResource.java | 108 +
.../wmservices/servlets/WmServicesServlet.java | 114 +
.../src/main/webapp/META-INF/context.xml | 21 +
.../wmservices/src/main/webapp/WEB-INF/web.xml | 48 +
workflow/pom.xml | 10 +-
.../DataSourceWorkflowInstanceRepository.java | 54 +
.../MemoryWorkflowInstanceRepository.java | 6 +
.../instrepo/WorkflowInstanceRepository.java | 8 +
.../workflow/policy/EnvSavingConfiguration.java | 62 +
.../policy/EnvVarSavingConfigReader.java | 53 +
.../policy/PolicyAwareWorkflowTask.java | 117 +
.../cas/workflow/policy/TaskPolicyMetKeys.java | 52 +
.../cas/workflow/policy/TaskPolicyReader.java | 212 ++
.../cas/workflow/policy/TaskPolicyWriter.java | 222 ++
.../cas/workflow/policy/TaskRequestFactory.java | 130 +
.../policy/TaskRequestProcessorMetKeys.java | 38 +
.../RollbackableWorkflowTaskInstance.java | 75 +
.../oodt/cas/workflow/structs/TaskJobInput.java | 37 +
.../workflow/system/XmlRpcWorkflowManager.java | 42 +-
.../system/XmlRpcWorkflowManagerClient.java | 6 +
.../main/python/xmlrpc-wrapper/oodt_metadata.py | 29 -
.../xmlrpc-wrapper/oodt_workflow_xmlrpc.py | 134 -
.../TestAsynchronousLocalEngineRunner.java | 35 +-
.../TestLuceneWorkflowInstanceRepository.java | 19 +
.../system/TestXmlRpcWorkflowManagerClient.java | 72 +-
.../TestDistributedXmlRpcWorkflowManager.java | 136 +
.../config/distributed/config-publisher.xml | 56 +
.../config/distributed/workflow.properties | 87 +
xmlps/.gitignore | 1 -
xmlps/pom.xml | 146 -
xmlps/src/main/conf/example-ps.xml | 71 -
xmlps/src/main/conf/example.db.properties | 21 -
.../oodt/xmlps/exceptions/XmlpsException.java | 27 -
.../oodt/xmlps/mapping/DatabaseTable.java | 118 -
.../oodt/xmlps/mapping/DatabaseTableGroup.java | 82 -
.../apache/oodt/xmlps/mapping/FieldScope.java | 35 -
.../apache/oodt/xmlps/mapping/FieldType.java | 35 -
.../org/apache/oodt/xmlps/mapping/Mapping.java | 163 -
.../apache/oodt/xmlps/mapping/MappingField.java | 251 --
.../oodt/xmlps/mapping/MappingReader.java | 206 -
.../xmlps/mapping/MappingReaderMetKeys.java | 65 -
.../oodt/xmlps/mapping/funcs/MappingFunc.java | 64 -
.../oodt/xmlps/mapping/funcs/ReplaceFunc.java | 77 -
.../apache/oodt/xmlps/product/DBMSExecutor.java | 68 -
.../oodt/xmlps/product/XMLPSProductHandler.java | 390 --
.../apache/oodt/xmlps/profile/DBMSExecutor.java | 169 -
.../oodt/xmlps/profile/XMLPSProfileHandler.java | 200 -
.../oodt/xmlps/queryparser/AndExpression.java | 30 -
.../xmlps/queryparser/ContainsExpression.java | 30 -
.../xmlps/queryparser/EqualsExpression.java | 30 -
.../oodt/xmlps/queryparser/Expression.java | 34 -
.../GreaterThanEqualsExpression.java | 31 -
.../queryparser/GreaterThanExpression.java | 31 -
.../xmlps/queryparser/HandlerQueryParser.java | 132 -
.../queryparser/LessThanEqualsExpression.java | 33 -
.../xmlps/queryparser/LessThanExpression.java | 33 -
.../apache/oodt/xmlps/queryparser/Literal.java | 39 -
.../oodt/xmlps/queryparser/LogOpExpression.java | 79 -
.../oodt/xmlps/queryparser/OrExpression.java | 30 -
.../oodt/xmlps/queryparser/ParseConstants.java | 65 -
.../oodt/xmlps/queryparser/RelOpExpression.java | 75 -
.../oodt/xmlps/queryparser/WildcardLiteral.java | 48 -
.../apache/oodt/xmlps/structs/CDEResult.java | 181 -
.../xmlps/structs/CDEResultInputStream.java | 121 -
.../org/apache/oodt/xmlps/structs/CDERow.java | 70 -
.../org/apache/oodt/xmlps/structs/CDEValue.java | 74 -
.../xmlps/util/GenericCDEObjectFactory.java | 70 -
.../apache/oodt/xmlps/util/XMLQueryHelper.java | 38 -
.../apache/oodt/xmlps/util/XMLQueryKeys.java | 30 -
.../oodt/xmlps/mapping/TestMappingReader.java | 165 -
.../xmlps/product/TestXMLPSProductHandler.java | 182 -
.../xmlps/profile/TestXMLPSProfileHandler.java | 71 -
.../queryparser/TestHandlerQueryParser.java | 111 -
.../oodt/xmlps/structs/TestCDEResult.java | 122 -
.../xmlps/structs/TestCDEResultInputStream.java | 137 -
xmlps/src/test/resources/loging.properties | 34 -
xmlps/src/test/resources/test-ps.xml | 120 -
.../test/resources/test-required-tables-ps.xml | 71 -
.../test/resources/test-same-col-name-ps.xml | 63 -
xmlquery/.gitignore | 1 -
xmlquery/oodt-xmlquery.iml.orig | 41 -
xmlquery/oodt-xmlquery.iml.rej | 66 -
xmlquery/pom.xml | 128 -
xmlquery/src/main/conf/mime.properties | 407 --
xmlquery/src/main/dtd/query.dtd | 100 -
.../oodt/product/HttpRedirectException.java | 40 -
.../apache/oodt/product/ProductException.java | 62 -
.../java/org/apache/oodt/product/Retriever.java | 46 -
.../apache/oodt/xmlquery/ByteArrayCodec.java | 63 -
.../java/org/apache/oodt/xmlquery/Codec.java | 85 -
.../org/apache/oodt/xmlquery/CodecFactory.java | 60 -
.../oodt/xmlquery/CompressedObjectCodec.java | 110 -
.../oodt/xmlquery/CompressedStringCodec.java | 91 -
.../java/org/apache/oodt/xmlquery/Header.java | 211 --
.../org/apache/oodt/xmlquery/ObjectCodec.java | 106 -
.../org/apache/oodt/xmlquery/QueryElement.java | 148 -
.../apache/oodt/xmlquery/QueryException.java | 37 -
.../org/apache/oodt/xmlquery/QueryHeader.java | 304 --
.../org/apache/oodt/xmlquery/QueryResult.java | 151 -
.../java/org/apache/oodt/xmlquery/Result.java | 447 ---
.../java/org/apache/oodt/xmlquery/Results.java | 60 -
.../org/apache/oodt/xmlquery/Statistic.java | 155 -
.../org/apache/oodt/xmlquery/StringCodec.java | 51 -
.../oodt/xmlquery/UnsupportedMimeTypeCodec.java | 44 -
.../java/org/apache/oodt/xmlquery/XMLQuery.java | 981 -----
.../xmlquery/activity/QueryConstructed.java | 23 -
.../oodt/xmlquery/activity/ResultsReceived.java | 23 -
.../java/org/apache/oodt/xmlquery/package.html | 26 -
.../src/site/resources/images/edm-query.jpg | Bin 57650 -> 0 bytes
.../src/site/resources/images/large-stack.png | Bin 41424 -> 0 bytes
.../src/site/resources/images/large-stack.psd | Bin 282874 -> 0 bytes
xmlquery/src/site/resources/images/results.png | Bin 53580 -> 0 bytes
xmlquery/src/site/resources/images/results.psd | Bin 275480 -> 0 bytes
.../src/site/resources/images/small-stack.png | Bin 12368 -> 0 bytes
.../src/site/resources/images/small-stack.psd | Bin 89725 -> 0 bytes
xmlquery/src/site/resources/images/xmlquery.png | Bin 85734 -> 0 bytes
xmlquery/src/site/resources/images/xmlquery.psd | Bin 352069 -> 0 bytes
xmlquery/src/site/site.xml | 33 -
xmlquery/src/site/xdoc/disquery/index.xml | 106 -
xmlquery/src/site/xdoc/servlet/index.xml | 640 ----
xmlquery/src/site/xdoc/tutorial/index.xml | 659 ----
.../apache/oodt/xmlquery/CodecFactoryTest.java | 61 -
.../org/apache/oodt/xmlquery/CodecTest.java | 77 -
.../xmlquery/CompressedObjectCodecTest.java | 42 -
.../xmlquery/CompressedStringCodecTest.java | 37 -
.../oodt/xmlquery/EmptyByteArrayCodecTest.java | 51 -
.../org/apache/oodt/xmlquery/HeaderTest.java | 142 -
.../apache/oodt/xmlquery/ObjectCodecTest.java | 42 -
.../apache/oodt/xmlquery/QueryElementTest.java | 93 -
.../org/apache/oodt/xmlquery/ResultTest.java | 166 -
.../apache/oodt/xmlquery/StringCodecTest.java | 37 -
.../org/apache/oodt/xmlquery/XMLQueryTest.java | 225 --
1163 files changed, 21456 insertions(+), 119210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index ed72a94,a688c31..5abaf3a
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,7 -1,49 +1,50 @@@
Apache OODT Change Log
======================
+
- Release 1.1 - Current Development
+ Release 1.2 - 08/14/2017
+
+ * OODT-955 Fix numhits error in Lucene Catalog
+ * OODT-956 Add connection checks to components
+
+ Release 1.1 - 07/18/2017
+
+ *** NOTE: A change was introduced in this version that causes a forced upgrade to your
+ Lucene Workflow Instance repository catalog. If you are using Lucene in the workflow manager,
+ the best bet is to blow away your old workflow instance catalog, and then allow it to be
+ rebuilt automatically. You can detect if you are affected by this issue if you see a message
+ akin to the below.
+ Jul 19, 2017 10:45:02 AM org.apache.oodt.cas.workflow.instrepo.LuceneWorkflowInstanceRepository addWorkflowInstanceToCatalog
+ WARNING: Unable to index workflow instance: [d8dad567-6c90-11e7-9e4e-7574e3a12106]: Message: Format version is not supported
+ (resource BufferedChecksumIndexInput(MMapIndexInput(path="/project/oodt/dev/jedi/data/workflow/slothrop/segments"))): -1 (needs
+ to be between 1071082519 and 1071082519). This version of Lucene only supports indexes created with release 5.0 and later.
+ java.lang.NullPointerException
+ ****
+
+ * OODT-954 CAS-PGE FileManagerFileStager has NPE b/c it does not set Product Structure (mattmann)
+
+ * OODT-526 Add base functionality in Curator for Task List persistence (mjoyce, mattmann)
+
+ * OODT-525 Export IngestTaskList from Curator (mjoyce, mattmann)
+
+ * OODT-304 Add a Getter method to get the file handler configuration (Michael Cayanan via mattmann)
+
+ * OODT-877 Fix 2 failures in o.a.o.commons.util.XMLTest on Windows 2008 Server (lewismc, mattmann)
+
+ * OODT-305 Make the Resource Manager manage different queues of jobs independently (resneck, mattmann)
+
+ * OODT-753 Move FM and WM Python APIs into "agility" component (kelly, mattmann)
+
+ * OODT-563 Task editing and Workflow execution (varun, mattmann)
+
+ * OODT-212 Rollback capability for Workflows (sherylj via mattmann)
+
+ * OODT-836, OODT-837 Remove Non-Essential Components (mattmann, mdstarch, magicaltrout)
+
+ * OODT-213 Reading/Writing API for XML-based Workflow Task policy (mattmann)
+
+ * OODT-775 XSD's for pushpull's mimetypes.xml (Lewis John McGibbney via mattmann)
+
+ * OODT-847 New FilemgrMetExtractor to extract File System Attributes (Aditya Dhulipala, rverma via mattmann)
* OODT-948 Fix dockerfile (magicaltrout/R B Krishna)
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/commons/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/core/pom.xml
----------------------------------------------------------------------
diff --cc core/pom.xml
index cbb6195,5e84d83..96444d5
--- a/core/pom.xml
+++ b/core/pom.xml
@@@ -249,9 -243,19 +249,23 @@@ the License
<version>8.7</version>
</dependency>
<dependency>
++<<<<<<< HEAD
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.12</version>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>3.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>3.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.5.1-alpha</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
@@@ -308,6 -317,6 +322,18 @@@
</dependency>
<dependency>
++ <groupId>org.apache.lucene</groupId>
++ <artifactId>lucene-queryparser</artifactId>
++ <version>6.1.0</version>
++ </dependency>
++
++ <dependency>
++ <groupId>org.apache.lucene</groupId>
++ <artifactId>lucene-analyzers-common</artifactId>
++ <version>6.1.0</version>
++ </dependency>
++
++ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
<version>${project.parent.version}</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/crawler/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/curator/sso/pom.xml
----------------------------------------------------------------------
diff --cc curator/sso/pom.xml
index 0000000,e5025f0..2fb29ef
mode 000000,100644..100644
--- a/curator/sso/pom.xml
+++ b/curator/sso/pom.xml
@@@ -1,0 -1,75 +1,74 @@@
+ <?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE.txt file distributed with this work for
+ additional information regarding copyright ownership. The ASF licenses this
+ file to you under the Apache License, Version 2.0 (the "License"); you may not
+ use this file except in compliance with the License. You may obtain a copy of
+ the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ License for the specific language governing permissions and limitations under
+ the License.
+ -->
+ <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>oodt-core</artifactId>
+ <version>1.3-SNAPSHOT</version>
+ <relativePath>../../core/pom.xml</relativePath>
+ </parent>
+ <artifactId>curator-sso</artifactId>
+ <packaging>jar</packaging>
+ <name>OODT CAS Curator Single Sign On Security Package</name>
+ <properties />
+ <!-- All dependencies should be listed in core/pom.xml and be ordered alphabetically by package and artifact.
+ Once the dependency is in the core pom, it can then be used in other modules without the version tags.
+ For example, within core/pom.xml:
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.4</version>
+ </dependency>
+
+ Elsewhere in the platform:
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ </dependency>
+
+ Where possible the same dependency version should be used across the whole platform but if required the version
+ can be overridden in a specific pom and should have a comment explaing why the version has been overridden
+ -->
+ <dependencies>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.3</version>
+ </dependency>
+ <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
++ <groupId>org.apache.httpcomponents</groupId>
++ <artifactId>httpclient</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ <version>2.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oodt</groupId>
+ <artifactId>cas-metadata</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins />
+ </build>
+ <distributionManagement />
+ </project>
[28/50] [abbrv] oodt git commit: update Changes
Posted by ma...@apache.org.
update Changes
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/07d9030d
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/07d9030d
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/07d9030d
Branch: refs/heads/master
Commit: 07d9030d303e27c55f336a7c54af060bb9be09b7
Parents: e09d368
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Mar 9 15:03:24 2017 +0000
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:42:46 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/07d9030d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9510096..ed72a94 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,8 @@ Apache OODT Change Log
Release 1.1 - Current Development
+* OODT-948 Fix dockerfile (magicaltrout/R B Krishna)
+
* OODT-938 update netcdf (magicaltrout)
* OODT-943 update javadoc and site plugin versions (magicaltrout)
[06/50] [abbrv] oodt git commit: - git ignore for crawler
Posted by ma...@apache.org.
- git ignore for crawler
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/6538bdc6
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/6538bdc6
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/6538bdc6
Branch: refs/heads/master
Commit: 6538bdc6fa85e2cc982d9d80bcfb9aa23519d08b
Parents: 363d4be
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:20:19 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:10:40 2017 -0800
----------------------------------------------------------------------
crawler/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/6538bdc6/crawler/.gitignore
----------------------------------------------------------------------
diff --git a/crawler/.gitignore b/crawler/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/crawler/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[15/50] [abbrv] oodt git commit: Fix for OODT-931 CAS Crawler and
metadata preconditions should say what preconditions failed.
Posted by ma...@apache.org.
Fix for OODT-931 CAS Crawler and metadata preconditions should say what preconditions failed.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/fa825105
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/fa825105
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/fa825105
Branch: refs/heads/master
Commit: fa8251058b55b5703eda622d6bc64df2c89c8e52
Parents: fcb214f
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 16:08:04 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:12:41 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../cas/crawl/MetExtractorProductCrawler.java | 37 ++++++++++++++++----
.../oodt/cas/crawl/StdProductCrawler.java | 16 +++++++--
3 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/fa825105/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 22147f5..e779398 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,9 @@ Apache OODT Change Log
Release 1.1 - Current Development
+* OODT-931 CAS Crawler and metadata preconditions should say what preconditions
+ failed (mattmann)
+
* OODT-932 OODT-Core forces javadoc to run and fails on linting - it shouldn't
do that (mattmann)
http://git-wip-us.apache.org/repos/asf/oodt/blob/fa825105/crawler/src/main/java/org/apache/oodt/cas/crawl/MetExtractorProductCrawler.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/MetExtractorProductCrawler.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/MetExtractorProductCrawler.java
index d663e84..978798a 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/MetExtractorProductCrawler.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/MetExtractorProductCrawler.java
@@ -22,12 +22,14 @@ import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
import org.apache.oodt.cas.metadata.exceptions.NamingConventionException;
import org.apache.oodt.cas.metadata.filenaming.NamingConvention;
+import org.apache.oodt.cas.metadata.preconditions.PreCondEvalUtils;
import org.apache.oodt.cas.metadata.preconditions.PreConditionComparator;
import org.springframework.beans.factory.annotation.Required;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Level;
/**
@@ -58,14 +60,29 @@ public class MetExtractorProductCrawler extends ProductCrawler {
@Override
protected boolean passesPreconditions(File product) {
if (this.getPreCondIds() != null) {
- for (String preCondId : this.getPreCondIds()) {
- if (!((PreConditionComparator<?>) this.getApplicationContext()
- .getBean(preCondId)).passes(product)) {
- return false;
- }
- }
+ PreCondEvalUtils evalUtils = new PreCondEvalUtils(
+ this.getApplicationContext());
+ if (!evalUtils.eval(this.getPreCondIds(), product)) {
+ return false;
+ }
}
- return product.exists() && product.length() > 0;
+
+ String preCondComparatorId = "ProductFileExistsCheck";
+ String preCondComparatorIdLength = "ProductLengthGreaterZeroCheck";
+
+ boolean existsFlag = true, lengthFlag = true;
+
+ if (!product.exists()){
+ existsFlag = false;
+ logConditionMessage(existsFlag, preCondComparatorId, product);
+ }
+
+ if (product.length() == 0){
+ lengthFlag = false;
+ logConditionMessage(lengthFlag, preCondComparatorIdLength, product);
+ }
+
+ return existsFlag && lengthFlag;
}
@Override
@@ -120,4 +137,10 @@ public class MetExtractorProductCrawler extends ProductCrawler {
public String getNamingConventionId() {
return namingConventionId;
}
+
+ private void logConditionMessage(boolean flag, String preCondComparatorId, File product){
+ String startMsg = flag ? "Passed":"Failed";
+ LOG.log(Level.INFO, startMsg + " precondition comparator id "
+ + preCondComparatorId+" file: "+product);
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/fa825105/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
----------------------------------------------------------------------
diff --git a/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java b/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
index 00e0c91..dcc5b68 100644
--- a/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
+++ b/crawler/src/main/java/org/apache/oodt/cas/crawl/StdProductCrawler.java
@@ -23,6 +23,7 @@ import org.apache.oodt.cas.metadata.extractors.MetReaderExtractor;
//JDK imports
import java.io.File;
+import java.util.logging.Level;
/**
* A generic Product Crawler for Products. The Crawler is given a root Product
@@ -58,8 +59,19 @@ public class StdProductCrawler extends ProductCrawler {
@Override
protected boolean passesPreconditions(File product) {
- return new File(product.getAbsolutePath() + "." + this.metFileExtension)
- .exists();
+ String metFilePath = product.getAbsolutePath() + "." + this.metFileExtension;
+ boolean flag = new File(metFilePath).exists();
+ String preCondComparatorId = "MetFileExistsCheck";
+
+ if (!flag){
+ LOG.log(Level.INFO, "Failed precondition comparator id "
+ + preCondComparatorId+" file: "+metFilePath+" does not exist.");
+ }
+ else{
+ LOG.log(Level.INFO, "Passed precondition comparator id "
+ + preCondComparatorId+" file: "+metFilePath+" exists.");
+ }
+ return flag;
}
@Override
[40/50] [abbrv] oodt git commit: Fix ZK test with new Avro layer in
FM.
Posted by ma...@apache.org.
Fix ZK test with new Avro layer in FM.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/4382d9d0
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/4382d9d0
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/4382d9d0
Branch: refs/heads/master
Commit: 4382d9d0fa477fd9f4f65c8a01221364565c0321
Parents: 9b0b61e
Author: Chris Mattmann <ma...@apache.org>
Authored: Sun Sep 3 13:06:01 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sun Sep 3 13:06:01 2017 -0700
----------------------------------------------------------------------
core/pom.xml | 4 +--
.../oodt/cas/filemgr/system/FileManager.java | 31 ++++++++++++++------
.../TestDistributedXmlRpcFileManager.java | 13 ++++++--
3 files changed, 35 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/4382d9d0/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 91f95ce..ba8e69b 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -429,12 +429,12 @@ the License.
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
- <version>6.2.1</version>
+ <version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
- <version>6.2.1</version>
+ <version>1.3.0</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/4382d9d0/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java
index 6e706ab..fc080e4 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/FileManager.java
@@ -39,6 +39,9 @@ import org.apache.oodt.cas.filemgr.versioning.VersioningUtils;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
import org.apache.oodt.commons.date.DateUtils;
+import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigurationManager;
+import org.apache.oodt.config.ConfigurationManagerFactory;
//JDK imports
import java.util.*;
@@ -74,8 +77,19 @@ public class FileManager {
/* whether or not to expand a product instance into met */
private boolean expandProductMet;
+
+ /** Configuration Manager instance which will handle the configuration aspect in distributed/standalone manner */
+ private ConfigurationManager configurationManager;
public FileManager() throws Exception {
+ List<String> propertiesFiles = new ArrayList<>();
+
+ // set up the configuration, if there is any
+ if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
+ propertiesFiles.add(System.getProperty("org.apache.oodt.cas.filemgr.properties"));
+ }
+
+ configurationManager = ConfigurationManagerFactory.getConfigurationManager(Component.FILE_MANAGER, propertiesFiles);
LOG.log(Level.INFO, "File Manager started by "
+ System.getProperty("user.name", "unknown"));
}
@@ -1080,16 +1094,15 @@ public class FileManager {
}
public void loadConfiguration() throws FileNotFoundException, IOException {
- // set up the configuration, if there is any
- if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
- String configFile = System
- .getProperty("org.apache.oodt.cas.filemgr.properties");
- LOG.log(Level.INFO,
- "Loading File Manager Configuration Properties from: [" + configFile
- + "]");
- System.getProperties().load(new FileInputStream(new File(configFile)));
+
+ try{
+ this.configurationManager.loadConfiguration();
}
-
+ catch(Exception e){
+ e.printStackTrace();
+ throw new IOException(e.getLocalizedMessage());
+ }
+
String metaFactory = null, dataFactory = null;
metaFactory = System.getProperty("filemgr.catalog.factory",
http://git-wip-us.apache.org/repos/asf/oodt/blob/4382d9d0/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/distributed/TestDistributedXmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/distributed/TestDistributedXmlRpcFileManager.java b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/distributed/TestDistributedXmlRpcFileManager.java
index 938a89b..bb4ad61 100644
--- a/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/distributed/TestDistributedXmlRpcFileManager.java
+++ b/filemgr/src/test/java/org/apache/oodt/cas/filemgr/system/distributed/TestDistributedXmlRpcFileManager.java
@@ -22,8 +22,10 @@ import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.metadata.ProductMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.oodt.cas.filemgr.system.AvroFileManagerServer;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManager;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerServer;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
@@ -61,13 +63,15 @@ public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigu
private static final String CONF_PUBLISHER_XML = "distributed/config/config-publisher.xml";
private static final String TRANSFER_SERVICE_FACTORY_CLASS = "org.apache.oodt.cas.filemgr.datatransfer.LocalDataTransferFactory";
- private XmlRpcFileManager fileManager;
+ private XmlRpcFileManagerServer fileManager;
@Before
public void setUpTest() throws Exception {
System.setProperty("org.apache.oodt.cas.cli.action.spring.config", "../config/src/main/resources/cmd-line-actions.xml");
System.setProperty("org.apache.oodt.cas.cli.option.spring.config", "../config/src/main/resources/cmd-line-options.xml");
System.setProperty(ENABLE_DISTRIBUTED_CONFIGURATION, "true");
+ System.setProperty("filemgr.client", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerClientFactory");
+ System.setProperty("filemgr.server", "org.apache.oodt.cas.filemgr.system.rpc.XmlRpcFileManagerServerFactory");
ConfigPublisher.main(new String[]{
"-connectString", zookeeper.getConnectString(),
@@ -76,8 +80,12 @@ public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigu
});
try {
- fileManager = new XmlRpcFileManager(FM_PORT);
+ fileManager = new XmlRpcFileManagerServer(FM_PORT);
+ if(!fileManager.startUp()){
+ throw new Exception("File Manager startup failed.");
+ }
} catch (Exception e) {
+ e.printStackTrace();
fail(e.getMessage());
}
@@ -188,6 +196,7 @@ public class TestDistributedXmlRpcFileManager extends AbstractDistributedConfigu
prodMet.addMetadata(CoreMetKeys.PRODUCT_TYPE, "GenericFile");
ingester.ingest(new URL("http://localhost:" + FM_PORT), new File(refUrl.getFile()), prodMet);
} catch (Exception e) {
+ e.printStackTrace();
fail(e.getMessage());
}
}
[49/50] [abbrv] oodt git commit: Fix merge conflicts.
Posted by ma...@apache.org.
Fix merge conflicts.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/c7b8c989
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/c7b8c989
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/c7b8c989
Branch: refs/heads/master
Commit: c7b8c9896c6def11370077333bf577a3397582ae
Parents: 444a5a9 ebaae1f
Author: Chris Mattmann <ch...@jpl.nasa.gov>
Authored: Thu Oct 12 10:36:49 2017 -0700
Committer: Chris Mattmann <ch...@jpl.nasa.gov>
Committed: Thu Oct 12 10:36:49 2017 -0700
----------------------------------------------------------------------
CHANGES.txt | 8 ++++++++
core/pom.xml | 25 +++++++++++++++++++++++--
2 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7b8c989/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 5abaf3a,5058970..4e50f80
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,6 -1,11 +1,14 @@@
Apache OODT Change Log
======================
+
++=======
+ Release 1.3 - In Progress
+
++* OODT-964, OODT-963 Fix issues merging Distributed ZK Config module into Avro-RPC branch (imesha,mattmann)
+ * OODT-957 Add CVE reporting plugin to maven build chain
+ * OODT-958 Update commons-collections
+ * OODT-959 Update tika to 1.16
+
Release 1.2 - 08/14/2017
* OODT-955 Fix numhits error in Lucene Catalog
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7b8c989/core/pom.xml
----------------------------------------------------------------------
[02/50] [abbrv] oodt git commit: update some dependenceies
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 5cba7d7..44ee93a 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -27,6 +27,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
@@ -34,14 +35,12 @@ import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ConnectionException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
+import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
-import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.SolrInputDocument;
import org.springframework.util.StringUtils;
@@ -74,31 +73,31 @@ public class SolrIndexer {
private final static String ACCESS_KEY = "access.key";
private final static String ACCESS_URL = "access.url";
private final static String PRODUCT_NAME = "CAS.ProductName";
+ private final HttpSolrClient server;
private IndexerConfig config = null;
- private final SolrServer server;
private String fmUrl;
private String solrUrl;
private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
private final static SimpleDateFormat solrFormat = new SimpleDateFormat(
- "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+ "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
/**
* Constructor reads in the configuration and initiates the connection to the
* Solr instance.
- *
+ *
* @param solrUrl
* URL for the Solr instance.
* @param fmUrl
* URL for the File Manager instance.
*/
public SolrIndexer(String solrUrl, String fmUrl)
- throws InstantiationException {
+ throws InstantiationException {
InputStream input = null;
String filename;
try {
LOG.info("System property " + SOLR_INDEXER_CONFIG + " set to "
- + System.getProperty(SOLR_INDEXER_CONFIG));
+ + System.getProperty(SOLR_INDEXER_CONFIG));
filename = System.getProperty(SOLR_INDEXER_CONFIG);
if (filename != null) {
LOG.info("Reading config from " + filename);
@@ -110,7 +109,7 @@ public class SolrIndexer {
config = new IndexerConfig(input);
} catch (IOException e) {
LOG
- .severe("Could not read in configuration for indexer from classpath or file");
+ .severe("Could not read in configuration for indexer from classpath or file");
throw new InstantiationException(e.getMessage());
} finally {
if (input != null) {
@@ -133,13 +132,10 @@ public class SolrIndexer {
}
LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl);
+ SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ server = new HttpSolrClient(this.solrUrl, httpClient);
- try {
- server = new CommonsHttpSolrServer(this.solrUrl);
- } catch (MalformedURLException e) {
- LOG.severe("Could not connect to Solr server " + this.solrUrl);
- throw new InstantiationException(e.getMessage());
- }
+// server = new SolrClient(this.solrUrl);
}
@@ -166,7 +162,7 @@ public class SolrIndexer {
/**
* This method transforms the product metadata into a Solr document.
- *
+ *
* @param metadata
* The metadata object for the product to index.
* @return Returns the SolrInputDocument containing product metadata.
@@ -174,6 +170,8 @@ public class SolrIndexer {
private SolrInputDocument getSolrDocument(Metadata metadata) {
SolrInputDocument doc = new SolrInputDocument();
// Only grab metadata which have a mapping in the indexer.properties
+ String official = "";
+
for (Object objKey : config.getMapProperties().keySet()) {
// The key in the metadata object
String key = (String) objKey;
@@ -184,19 +182,21 @@ public class SolrIndexer {
for (String value : values) {
// Add each metadata value into the
if (value != null && !config.getIgnoreValues().contains(value.trim())) {
- LOG.fine("Adding field: " + fieldName + " value: " + value);
+ official += " Key: " + key + " Value: " + value;
doc.addField(fieldName, value);
}
}
}
}
+ // LOG.info("KEYS WERE:" +official);
+
return doc;
}
/**
* This method adds a single product extracted from a metadata file to the
* Solr index.
- *
+ *
* @param file
* The file containing product metadata.
* @param delete
@@ -206,24 +206,24 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexMetFile(File file, boolean delete)
- throws
- SolrServerException {
+ throws
+ SolrServerException {
LOG.info("Attempting to index product from metadata file.");
try {
SerializableMetadata metadata = new SerializableMetadata("UTF-8", false);
metadata.loadMetadataFromXmlStream(new FileInputStream(file));
metadata.addMetadata("id", metadata.getMetadata("CAS."
- + CoreMetKeys.PRODUCT_ID));
+ + CoreMetKeys.PRODUCT_ID));
metadata.addMetadata(config.getProperty(ACCESS_KEY), config
- .getProperty(ACCESS_URL)
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .getProperty(ACCESS_URL)
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
if (delete) {
server
- .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
}
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: "
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
} catch (InstantiationException e) {
LOG.severe("Could not instantiate metadata object: " + e.getMessage());
} catch (FileNotFoundException e) {
@@ -236,7 +236,7 @@ public class SolrIndexer {
/**
* This method indexes all product types retrieved from the File Manager to
* the Solr index.
- *
+ *
* @param delete
* Flag indicating whether each product type retrieved from the File
* Manager should be deleted from the index.
@@ -246,7 +246,8 @@ public class SolrIndexer {
public void indexProductTypes(boolean delete) {
LOG.info("Indexing product types...");
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -255,9 +256,9 @@ public class SolrIndexer {
metadata.addMetadata("id", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeId", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeDescription", type
- .getDescription());
+ .getDescription());
metadata.addMetadata("CAS.ProductTypeRepositoryPath", type
- .getProductRepositoryPath());
+ .getProductRepositoryPath());
metadata.addMetadata("CAS.ProductTypeVersioner", type.getVersioner());
metadata.addMetadata("CAS.ProductTypeName", type.getName());
metadata.addMetadata("ProductType", "ProductType");
@@ -267,7 +268,7 @@ public class SolrIndexer {
server.deleteById(type.getProductTypeId());
} catch (Exception e) {
LOG.severe("Could not delete product type " + type.getName()
- + " from index: " + e.getMessage());
+ + " from index: " + e.getMessage());
}
}
try {
@@ -276,7 +277,7 @@ public class SolrIndexer {
LOG.info("Indexed product type: " + type.getName());
} catch (Exception e) {
LOG.severe("Could not index " + type.getName() + ": "
- + e.getMessage());
+ + e.getMessage());
}
} else {
LOG.info("Ignoring product type: " + type.getName());
@@ -288,15 +289,15 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
LOG.info("Finished indexing product types.");
}
-
+
/**
- * Suppresses exception that occurred with older file managers.
+ * Suppresses exception that occurred with older file managers.
*/
- private ProductPage safeFirstPage(FileManagerClient fmClient, ProductType type) {
+ private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) {
ProductPage page = null;
try {
page = fmClient.getFirstPage(type);
@@ -310,7 +311,7 @@ public class SolrIndexer {
* This method indexes all products retrieved from the File Manager to the
* Solr index. Metadata from the product's associated ProductType is also
* included.
- *
+ *
* @param delete
* Flag indicating whether each product retrieved from the File
* Manager should be deleted from the index.
@@ -320,28 +321,30 @@ public class SolrIndexer {
public void indexAll(boolean delete) {
LOG.info("Indexing products...");
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
if (!config.getIgnoreTypes().contains(type.getName().trim())) {
+ ProductPage page = safeFirstPage(fmClient, type);
LOG.info("Paging through products for product type: "
- + type.getName());
- ProductPage page = safeFirstPage(fmClient, type);
+ + type.getName()+" Total pages are "+page.getTotalPages());
while (page != null) {
- for (Product product : page.getPageProducts()) {
+ for (Product product : page.getPageProducts()) {
try {
- this.indexProduct(product.getProductId(), fmClient
- .getMetadata(product), type.getTypeMetadata());
+ String p = product.getProductId();
+ Metadata m = fmClient.getMetadata(product);
+ this.indexProduct(p, m, type.getTypeMetadata());
} catch (Exception e) {
LOG.severe("Could not index " + product.getProductId() + ": "
- + e.getMessage());
+ + e.getLocalizedMessage());
}
}
- if (page.isLastPage()) {
- break;
- }
- page = fmClient.getNextPage(type, page);
+ if (page.getPageNum() >= page.getTotalPages() || page.isLastPage()) {
+ break;
+ }
+ page = fmClient.getNextPage(type, page);
}
}
}
@@ -352,10 +355,10 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve products from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
}
@@ -363,38 +366,50 @@ public class SolrIndexer {
* This method adds a single product retrieved from the File Manager by its
* product identifier to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productId
* The identifier of the product (CAS.ProductId).
* @throws SolrServerException
* When an error occurs communicating with the Solr server instance.
*/
public void indexProduct(String productId)
- throws SolrServerException {
+ throws SolrServerException {
LOG.info("Attempting to index product: " + productId);
try {
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
Product product = fmClient.getProductById(productId);
Metadata productMetadata = fmClient.getMetadata(product);
+ List<String> keys = productMetadata.getAllKeys();
+ String logger = "";
+ for(String k :keys){
+ logger+=" key: "+keys;
+ List<String> values = productMetadata.getAllValues(k);
+
+ for(String v : values){
+ logger+=" value: "+v + ", ";
+ }
+ }
+ LOG.info("Metadata: "+ logger);
indexProduct(product.getProductId(), productMetadata, product
- .getProductType().getTypeMetadata());
+ .getProductType().getTypeMetadata());
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
-
+
/**
* This method adds a single product retrieved from the File Manager by its
* product name to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productName
* The identifier of the product (CAS.ProductId).
* @param delete
@@ -404,7 +419,7 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexProductByName(String productName, boolean delete) throws SolrServerException {
-
+
LOG.info("Attempting to index product: " + productName);
try {
@@ -423,27 +438,28 @@ public class SolrIndexer {
}
}
- FileManagerClient fmClient = RpcCommunicationFactory.createClient(new URL(this.fmUrl));
+ XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
+ this.fmUrl));
Product product = fmClient.getProductByName(productName);
Metadata productMetadata = fmClient.getMetadata(product);
// NOTE: delete (by id) is now false
indexProduct(product.getProductId(), productMetadata, product.getProductType().getTypeMetadata());
-
+
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
private void indexProduct(String productId, Metadata productMetadata,
- Metadata typeMetadata) throws SolrServerException,
- java.text.ParseException {
+ Metadata typeMetadata) throws SolrServerException,
+ java.text.ParseException {
Metadata metadata = new Metadata();
metadata.addMetadata("id", productId);
// Add in product type metadata
@@ -469,7 +485,7 @@ public class SolrIndexer {
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: " + productId);
} catch (IOException e) {
- LOG.severe("Could not index product: " + productId);
+ LOG.severe("Could not index product: " + productId+ "Exception:"+e.getLocalizedMessage());
}
} else {
LOG.info("Could not find metadata for product: " + productId);
@@ -478,7 +494,7 @@ public class SolrIndexer {
/**
* This method deletes a single product identified by a productID from the Solr index
- *
+ *
* @param productId
* @throws IOException
* @throws SolrServerException
@@ -490,7 +506,7 @@ public class SolrIndexer {
/**
* This method deletes a product(s) from the Solr index identified by a given name
- *
+ *
* @param productName
* @throws IOException
* @throws SolrServerException
@@ -514,16 +530,16 @@ public class SolrIndexer {
private void deleteProductFromIndex(String productId) throws IOException, SolrServerException {
server.deleteById(productId);
}
-
+
/**
* Quick helper method to do substitution on the keys specified in the config
- *
+ *
* @param metadata
* to substitute on
* @throws java.text.ParseException
*/
private void performSubstitution(Metadata metadata)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Do metadata replacement
for (String key : config.getReplacementKeys()) {
List<String> values = metadata.getAllMetadata(key);
@@ -543,10 +559,12 @@ public class SolrIndexer {
List<String> values = metadata.getAllMetadata(keyString);
if (values != null) {
List<String> newValues = new ArrayList<String>();
- SimpleDateFormat format = new SimpleDateFormat(config
- .getFormatProperties().getProperty(keyString).trim());
+
+ /* SimpleDateFormat format = new SimpleDateFormat(config
+ .getFormatProperties().getProperty(keyString).trim());*/
for (String value : values) {
- newValues.add(formatDate(format, value));
+ String d = value.trim();
+ newValues.add(d);
}
metadata.removeMetadata(keyString);
metadata.addMetadata(keyString, newValues);
@@ -556,17 +574,17 @@ public class SolrIndexer {
}
private String formatDate(SimpleDateFormat format, String value)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Ignore formating if its an ignore value
if (config.getIgnoreValues().contains(value.trim())) {
- return value;
+ return value;
}
return solrFormat.format(format.parse(value));
}
/**
* This method builds the command-line options.
- *
+ *
* @return Returns the supported Options.
*/
@SuppressWarnings("static-access")
@@ -575,31 +593,31 @@ public class SolrIndexer {
options.addOption(new Option("h", "help", false, "Print this message"));
options.addOption(new Option("o", "optimize", false,
- "Optimize the Solr index"));
+ "Optimize the Solr index"));
options.addOption(new Option("d", "delete", false,
- "Delete item before indexing"));
+ "Delete item before indexing"));
options.addOption(OptionBuilder.withArgName("Solr URL").hasArg()
- .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
- .create("su"));
+ .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
+ .create("su"));
options.addOption(OptionBuilder.withArgName("Filemgr URL").hasArg()
- .withDescription("URL to the File Manager").withLongOpt("fmUrl")
- .create("fmu"));
+ .withDescription("URL to the File Manager").withLongOpt("fmUrl")
+ .create("fmu"));
OptionGroup group = new OptionGroup();
Option all = new Option("a", "all", false,
- "Index all products from the File Manager");
+ "Index all products from the File Manager");
Option product = OptionBuilder.withArgName("productId").hasArg()
- .withDescription("Index the product from the File Manager")
- .withLongOpt("product").create("p");
+ .withDescription("Index the product from the File Manager")
+ .withLongOpt("product").create("p");
Option met = OptionBuilder.withArgName("file").hasArg().withDescription(
- "Index the product from a metadata file").withLongOpt("metFile")
- .create("mf");
+ "Index the product from a metadata file").withLongOpt("metFile")
+ .create("mf");
Option read = new Option("r", "read", false,
- "Index all products based on a list of product identifiers passed in");
+ "Index all products based on a list of product identifiers passed in");
Option types = new Option("t", "types", false,
- "Index all product types from the File Manager");
+ "Index all product types from the File Manager");
Option deleteAll = new Option("da", "deleteAll", false,
- "Delete all products/types from the Solr index");
+ "Delete all products/types from the Solr index");
group.addOption(all);
group.addOption(product);
@@ -614,7 +632,7 @@ public class SolrIndexer {
/**
* The main method. Execution without argument displays help message.
- *
+ *
* @param args
* Command-line arguments.
*/
@@ -633,8 +651,8 @@ public class SolrIndexer {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("java " + SolrIndexer.class.getName(), options);
} else if (line.hasOption("all") || line.hasOption("product")
- || line.hasOption("metFile") || line.hasOption("read")
- || line.hasOption("types") || line.hasOption("deleteAll")) {
+ || line.hasOption("metFile") || line.hasOption("read")
+ || line.hasOption("types") || line.hasOption("deleteAll")) {
SolrIndexer indexer;
String solrUrl = null;
String fmUrl = null;
@@ -652,7 +670,7 @@ public class SolrIndexer {
indexer.indexProduct(line.getOptionValue("product"));
} else if (line.hasOption("metFile")) {
indexer.indexMetFile(new File(line.getOptionValue("metFile")), line
- .hasOption("delete"));
+ .hasOption("delete"));
} else if (line.hasOption("read")) {
for (String productId : readProductIdsFromStdin()) {
indexer.indexProduct(productId);
@@ -671,14 +689,14 @@ public class SolrIndexer {
} catch (Exception e) {
LOG.severe("An error occurred indexing: " + e.getMessage());
LOG
- .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
+ .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
}
}
}
/**
* This method reads product identifiers from the standard input.
- *
+ *
* @return Returns a List of product identifiers.
*/
private static List<String> readProductIdsFromStdin() {
@@ -694,12 +712,12 @@ public class SolrIndexer {
}
} catch (IOException e) {
LOG.severe("Error reading product id: line: [" + line + "]: Message: "
- + e.getMessage());
+ + e.getMessage());
} finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
return productIds;
}
@@ -731,32 +749,32 @@ public class SolrIndexer {
String key = (String) objKey;
if (key.startsWith(PREFIX_CONFIG)) {
properties.put(key.substring(PREFIX_CONFIG.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_MET)) {
mapProperties.put(key.substring(PREFIX_MET.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_FORMAT)) {
formatProperties.put(key.substring(PREFIX_FORMAT.length()), props
- .getProperty(key));
+ .getProperty(key));
}
}
if (properties.getProperty(IGNORE_TYPES) != null) {
String[] values = properties.getProperty(IGNORE_TYPES).trim()
- .split(",");
- Collections.addAll(ignoreTypes, values);
+ .split(",");
+ Collections.addAll(ignoreTypes, values);
}
if (properties.getProperty(IGNORE_VALUES) != null) {
String[] values = properties.getProperty(IGNORE_VALUES).trim().split(
- ",");
- Collections.addAll(ignoreValues, values);
+ ",");
+ Collections.addAll(ignoreValues, values);
}
if (properties.getProperty(REPLACEMENT_KEYS) != null) {
String[] values = properties.getProperty(REPLACEMENT_KEYS).trim()
- .split(",");
- Collections.addAll(replacementKeys, values);
+ .split(",");
+ Collections.addAll(replacementKeys, values);
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/profile/pom.xml
----------------------------------------------------------------------
diff --git a/profile/pom.xml b/profile/pom.xml
index 3d67496..68f8fa9 100644
--- a/profile/pom.xml
+++ b/profile/pom.xml
@@ -62,9 +62,8 @@
<version>1.3</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0-alpha1</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/protocol/http/pom.xml
----------------------------------------------------------------------
diff --git a/protocol/http/pom.xml b/protocol/http/pom.xml
index 4aaf834..19b5685 100644
--- a/protocol/http/pom.xml
+++ b/protocol/http/pom.xml
@@ -41,8 +41,8 @@
-->
<dependencies>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/resource/pom.xml
----------------------------------------------------------------------
diff --git a/resource/pom.xml b/resource/pom.xml
index 6ccb724..3e1bc8e 100644
--- a/resource/pom.xml
+++ b/resource/pom.xml
@@ -73,8 +73,8 @@ the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/sso/pom.xml
----------------------------------------------------------------------
diff --git a/sso/pom.xml b/sso/pom.xml
index cdb59a2..82979ec 100644
--- a/sso/pom.xml
+++ b/sso/pom.xml
@@ -53,9 +53,8 @@ the License.
<version>1.3</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.0</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
----------------------------------------------------------------------
diff --git a/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java b/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
index 5d77083..d0d399f 100755
--- a/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
+++ b/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
@@ -18,30 +18,37 @@
package org.apache.oodt.security.sso.opensso;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.PostMethod;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
+
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
- *
+ *
* A client class to the services provided by the <a
* href="https://opensso.dev.java.net/">OpenSSO</a> project. The descriptions of
* these services are <a
* href="http://developers.sun.com/identity/reference/techart/id-svcs.html"
* >here</a>.
- *
+ *
* @author mattmann
* @version $Revision$
- *
+ *
*/
public class SSOProxy implements SSOMetKeys {
@@ -54,53 +61,63 @@ public class SSOProxy implements SSOMetKeys {
private static final String IDENT_ATTR_ENDPOINT_KEY = "IDENT_ATTR_ENDPOINT";
private static final String LOG_ENDPOINT;
private static final String LOG_ENDPOINT_KEY = "LOG_ENDPOINT";
-
+
static {
- if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
- AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
- } else {
- AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
- }
- if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
- IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
- } else {
- IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
- }
- if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
- IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
- } else {
- IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
- }
- if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
- LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
- } else {
- LOG_ENDPOINT = LOGOUT_ENDPOINT;
- }
-
- LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
- LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
- LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
- LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
+ if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
+ AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
+ } else {
+ AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
+ }
+ if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
+ IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
+ } else {
+ IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
+ }
+ if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
+ IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
+ } else {
+ IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
+ }
+ if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
+ LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
+ } else {
+ LOG_ENDPOINT = LOGOUT_ENDPOINT;
+ }
+
+ LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
+ LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
+ LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
+ LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
}
public String authenticate(String username, String password) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(AUTH_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(AUTH_ENDPOINT);
+ //PostMethod post = new PostMethod(AUTH_ENDPOINT);
String response;
String ssoToken = null;
- NameValuePair[] data = { new NameValuePair("username", username),
- new NameValuePair("password", password),
- new NameValuePair("uri", "realm/lmmp") };
+ NameValuePair[] data = { new BasicNameValuePair("username", username),
+ new BasicNameValuePair("password", password),
+ new BasicNameValuePair("uri", "realm/lmmp") };
- post.setRequestBody(data);
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new HttpException(response1.getStatusLine().toString());
}
- response = post.getResponseBodyAsString().trim();
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ response = handler.handleResponse(response1);
ssoToken = response.substring(9);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
@@ -112,56 +129,85 @@ public class SSOProxy implements SSOMetKeys {
}
public IdentityDetails readIdentity(String username, String token)
- throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_READ_ENDPOINT);
+ throws IOException, SingleSignOnException {
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
LOG.log(Level.INFO, "Obtaining identity: username: [" + username
- + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
- + "]");
- NameValuePair[] data = { new NameValuePair("name", username),
- new NameValuePair("admin", token) };
+ + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
+ + "]");
+ NameValuePair[] data = { new BasicNameValuePair("name", username),
+ new BasicNameValuePair("admin", token) };
- post.setRequestBody(data);
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new SingleSignOnException(response1.getStatusLine().toString());
}
- return parseIdentityDetails(post.getResponseBodyAsString().trim());
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+
+ return parseIdentityDetails(handler.handleResponse(response1).trim());
}
public UserDetails getUserAttributes(String token) throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_ATTR_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
LOG.log(Level.INFO, "Obtaining user attributes: token: [" + token
- + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
+ + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
+ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
+
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
- post.setRequestBody(data);
+ post.setEntity(entity);
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new SingleSignOnException(response1.getStatusLine().toString());
}
- return parseUserDetails(post.getResponseBodyAsString().trim());
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+
+ return parseUserDetails(handler.handleResponse(response1).trim());
}
public void logout(String token) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(LOG_ENDPOINT);
+ HttpClient httpClient = new DefaultHttpClient();
+ HttpPost post = new HttpPost(LOG_ENDPOINT);
LOG.log(Level.INFO, "Logging out: token: [" + token + "]: REST url: ["
- + LOG_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
- post.setRequestBody(data);
+ + LOG_ENDPOINT + "]");
+ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
+
+
+ UrlEncodedFormEntity entity = null;
+ try {
+ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ post.setEntity(entity);
try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
+ HttpResponse response1 = httpClient.execute(post);
+ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+ throw new HttpException(response1.getStatusLine().toString());
}
} catch (HttpException e) {
// TODO Auto-generated catch block
@@ -176,7 +222,7 @@ public class SSOProxy implements SSOMetKeys {
private IdentityDetails parseIdentityDetails(String serviceResponse) {
ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
+ .getBytes());
BufferedReader br = new BufferedReader(new InputStreamReader(is));
IdentityDetails details = new IdentityDetails();
String line = null, lastAttrKeyRead = null;
@@ -214,7 +260,7 @@ public class SSOProxy implements SSOMetKeys {
} catch (IOException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
+ + "]: Message: " + e.getMessage());
} finally {
try {
is.close();
@@ -233,7 +279,7 @@ public class SSOProxy implements SSOMetKeys {
private UserDetails parseUserDetails(String serviceResponse) {
ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
+ .getBytes());
BufferedReader br = new BufferedReader(new InputStreamReader(is));
UserDetails details = new UserDetails();
String line = null, lastAttrKeyRead = null;
@@ -264,7 +310,7 @@ public class SSOProxy implements SSOMetKeys {
} catch (IOException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
+ + "]: Message: " + e.getMessage());
} finally {
try {
is.close();
@@ -283,8 +329,8 @@ public class SSOProxy implements SSOMetKeys {
public static void main(String[] args) throws IOException, SingleSignOnException {
String usage = "SSOProxy <cmd> [args]\n\n" + "Where cmd is one of:\n"
- + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
- + "attributes <token>\nlogout <token>\n";
+ + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
+ + "attributes <token>\nlogout <token>\n";
if (args.length < 2 || args.length > 3) {
System.err.println(usage);
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/workflow/pom.xml
----------------------------------------------------------------------
diff --git a/workflow/pom.xml b/workflow/pom.xml
index 72fb549..1fe5189 100644
--- a/workflow/pom.xml
+++ b/workflow/pom.xml
@@ -76,8 +76,8 @@ the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
[44/50] [abbrv] oodt git commit: Merge branch
'feature/zookeeper-config' of https://github.com/IMS94/oodt into development
Posted by ma...@apache.org.
Merge branch 'feature/zookeeper-config' of https://github.com/IMS94/oodt into development
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/54089cae
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/54089cae
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/54089cae
Branch: refs/heads/master
Commit: 54089cae98a30b7953d848117bd3d1994185f5b1
Parents: ce89ad1 07aea0d
Author: Imesha Sudasingha <im...@apache.org>
Authored: Wed Oct 11 15:30:34 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Oct 12 08:07:14 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 2 +-
.../org/apache/oodt/config/ConfigEventType.java | 50 +++++++++++++
.../oodt/config/ConfigurationListener.java | 31 ++++++++
.../oodt/config/ConfigurationManager.java | 23 ++++++
.../java/org/apache/oodt/config/Constants.java | 3 +
.../DistributedConfigurationManager.java | 79 +++++++++++++++++---
.../DistributedConfigurationPublisher.java | 43 ++++++++++-
.../oodt/config/distributed/ZNodePaths.java | 27 +++++++
.../oodt/config/distributed/cli/CLIAction.java | 48 +++++++++++-
.../config/distributed/cli/ConfigPublisher.java | 8 ++
.../config/distributed/utils/ConfigUtils.java | 5 ++
.../config/distributed/utils/CuratorUtils.java | 19 +++--
.../AbstractDistributedConfigurationTest.java | 7 ++
config/src/main/resources/cmd-line-options.xml | 31 ++++++++
.../DistributedConfigurationManagerTest.java | 69 ++++++++++++++++-
core/pom.xml | 2 +-
.../cas/filemgr/system/XmlRpcFileManager.java | 18 ++++-
pge/pom.xml | 2 +-
18 files changed, 435 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/54089cae/core/pom.xml
----------------------------------------------------------------------
diff --cc core/pom.xml
index 3489a56,f6e0fc3..779c0c1
--- a/core/pom.xml
+++ b/core/pom.xml
@@@ -254,13 -243,8 +254,13 @@@ the License
<version>8.7</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.12</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
[46/50] [abbrv] oodt git commit: [OODT-965] Added configuration
change listening feature
Posted by ma...@apache.org.
[OODT-965] Added configuration change listening feature
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/df1db1e4
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/df1db1e4
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/df1db1e4
Branch: refs/heads/master
Commit: df1db1e44b3289214693f3a0894681400782ba99
Parents: fc6311d
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Sat Aug 19 16:21:10 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Oct 12 08:07:14 2017 +0530
----------------------------------------------------------------------
config/pom.xml | 2 +-
.../org/apache/oodt/config/ConfigEventType.java | 45 +++++++++++++
.../oodt/config/ConfigurationListener.java | 31 +++++++++
.../oodt/config/ConfigurationManager.java | 23 +++++++
.../java/org/apache/oodt/config/Constants.java | 3 +
.../DistributedConfigurationManager.java | 64 +++++++++++++++---
.../DistributedConfigurationPublisher.java | 26 +++++++-
.../oodt/config/distributed/ZNodePaths.java | 27 ++++++++
.../oodt/config/distributed/cli/CLIAction.java | 24 +++++--
config/src/main/resources/cmd-line-options.xml | 31 +++++++++
.../DistributedConfigurationManagerTest.java | 69 ++++++++++++++++++--
core/pom.xml | 2 +-
.../cas/filemgr/system/XmlRpcFileManager.java | 18 ++++-
pge/pom.xml | 2 +-
14 files changed, 343 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/pom.xml
----------------------------------------------------------------------
diff --git a/config/pom.xml b/config/pom.xml
index 3f26499..e34ff44 100644
--- a/config/pom.xml
+++ b/config/pom.xml
@@ -26,7 +26,7 @@
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
new file mode 100644
index 0000000..8a465b0
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config;
+
+public enum ConfigEventType {
+ PUBLISH("publish"),
+ CLEAR("clear");
+
+ private String name;
+
+ ConfigEventType(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ public static ConfigEventType parse(String string) {
+ switch (string) {
+ case "publish":
+ return PUBLISH;
+ case "clear":
+ return CLEAR;
+ default:
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java b/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
new file mode 100644
index 0000000..a5abcab
--- /dev/null
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationListener.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oodt.config;
+
+/**
+ * The interface which should be implemented in order to listen for configuration changes.
+ *
+ * @author Imesha Sudasingha
+ */
+public interface ConfigurationListener {
+
+ /**
+ * This method is invoked when there has been any change in configuration of the interested component
+ */
+ void configurationChanged(ConfigEventType type);
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
index 53aacef..9158fad 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigurationManager.java
@@ -17,7 +17,9 @@
package org.apache.oodt.config;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* The abstract class to define functions of the configuration managers.
@@ -28,6 +30,7 @@ public abstract class ConfigurationManager {
protected Component component;
protected String project;
+ private Set<ConfigurationListener> configurationListeners = new HashSet<>(1);
public ConfigurationManager(Component component) {
this(component, Constants.DEFAULT_PROJECT);
@@ -38,6 +41,12 @@ public abstract class ConfigurationManager {
this.project = project;
}
+ /**
+ * Loads configuration required for {@link #component}. If distributed configuration management is enabled, this
+ * will download configuration from zookeeper. Else, this will load properties files specified.
+ *
+ * @throws Exception
+ */
public abstract void loadConfiguration() throws Exception;
/**
@@ -47,6 +56,20 @@ public abstract class ConfigurationManager {
*/
public abstract void clearConfiguration();
+ public synchronized void addConfigurationListener(ConfigurationListener listener) {
+ configurationListeners.add(listener);
+ }
+
+ public synchronized void removeConfigurationListener(ConfigurationListener listener) {
+ configurationListeners.remove(listener);
+ }
+
+ protected synchronized void notifyConfigurationChange(ConfigEventType type) {
+ for (ConfigurationListener listener : configurationListeners) {
+ listener.configurationChanged(type);
+ }
+ }
+
public Component getComponent() {
return component;
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/Constants.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/Constants.java b/config/src/main/java/org/apache/oodt/config/Constants.java
index 9cd217f..3824afb 100644
--- a/config/src/main/java/org/apache/oodt/config/Constants.java
+++ b/config/src/main/java/org/apache/oodt/config/Constants.java
@@ -93,5 +93,8 @@ public class Constants {
/** Where other configuration files will be stored */
public static final String CONFIGURATION_PATH_NAME = "configuration";
+
+ /** Path to be watched for configuration changes */
+ public static final String NOTIFICATIONS_PATH = "notifications";
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index ae8912a..6dfb24b 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -19,7 +19,10 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.apache.curator.framework.recipes.cache.NodeCacheListener;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.Constants.Properties;
@@ -38,6 +41,7 @@ import java.util.concurrent.TimeUnit;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
+import static org.apache.oodt.config.Constants.Properties.ZK_STARTUP_TIMEOUT;
import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjectName;
/**
@@ -57,12 +61,41 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private List<String> savedFiles = new ArrayList<>();
+ /** {@link NodeCache} to watch for configuration change notifications */
+ private NodeCache nodeCache;
+ private NodeCacheListener nodeCacheListener = new NodeCacheListener() {
+ @Override
+ public void nodeChanged() throws Exception {
+ byte[] data = client.getData().forPath(zNodePaths.getNotificationsZNodePath());
+ if (data == null) {
+ return;
+ }
+
+ String event = new String(data);
+ ConfigEventType type = ConfigEventType.parse(event);
+ if (type != null) {
+ logger.info("Configuration changed event of type: '{}' received", type);
+ switch (type) {
+ case PUBLISH:
+ loadConfiguration();
+ break;
+ case CLEAR:
+ clearConfiguration();
+ break;
+ }
+
+ notifyConfigurationChange(type);
+ }
+ }
+ };
+
public DistributedConfigurationManager(Component component) {
super(component, getOODTProjectName());
this.zNodePaths = new ZNodePaths(this.project, this.component.getName());
- if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
- throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " + ZK_CONNECT_STRING + " to be set");
+ if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
+ throw new IllegalArgumentException("Zookeeper requires system properties " + ZK_PROPERTIES_FILE + " or " +
+ ZK_CONNECT_STRING + " to be set");
}
if (System.getProperty(ZK_PROPERTIES_FILE) != null) {
@@ -73,11 +106,11 @@ public class DistributedConfigurationManager extends ConfigurationManager {
}
}
- if (System.getProperty(Constants.Properties.ZK_CONNECT_STRING) == null) {
+ if (System.getProperty(ZK_CONNECT_STRING) == null) {
throw new IllegalArgumentException("Zookeeper requires a proper connect string to connect to zookeeper ensemble");
}
- connectString = System.getProperty(Constants.Properties.ZK_CONNECT_STRING);
+ connectString = System.getProperty(ZK_CONNECT_STRING);
logger.info("Using zookeeper connect string : {}", connectString);
startZookeeper();
}
@@ -90,7 +123,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
client = CuratorUtils.newCuratorFrameworkClient(connectString, logger);
client.start();
logger.info("Curator framework start operation invoked");
- int startupTimeOutMs = Integer.parseInt(System.getProperty(Properties.ZK_STARTUP_TIMEOUT, "30000"));
+
+ int startupTimeOutMs = Integer.parseInt(System.getProperty(ZK_STARTUP_TIMEOUT, "30000"));
try {
logger.info("Waiting to connect to zookeeper, startupTimeout : {}", startupTimeOutMs);
client.blockUntilConnected(startupTimeOutMs, TimeUnit.MILLISECONDS);
@@ -103,10 +137,21 @@ public class DistributedConfigurationManager extends ConfigurationManager {
}
logger.info("CuratorFramework client started successfully");
+
+ nodeCache = new NodeCache(client, zNodePaths.getNotificationsZNodePath());
+ nodeCache.getListenable().addListener(nodeCacheListener);
+ try {
+ logger.debug("Starting NodeCache to watch for configuration changes");
+ nodeCache.start(true);
+ } catch (Exception e) {
+ logger.error("Error occurred when start listening for configuration changes", e);
+ throw new IllegalStateException("Unable to start listening for configuration changes", e);
+ }
+ logger.info("NodeCache for watching configuration changes started successfully");
}
@Override
- public void loadConfiguration() throws Exception {
+ public synchronized void loadConfiguration() throws Exception {
logger.debug("Loading properties for : {}", component);
loadProperties();
logger.info("Properties loaded for : {}", component);
@@ -170,9 +215,8 @@ public class DistributedConfigurationManager extends ConfigurationManager {
private void saveFile(String path, byte[] data) throws IOException {
String localFilePath = ConfigUtils.fixForComponentHome(component, path);
File localFile = new File(localFilePath);
- if (localFile.exists()) {
- logger.warn("Deleting already existing file at {} before writing new content", localFilePath);
- localFile.delete();
+ if (localFile.exists() && localFile.delete()) {
+ logger.warn("Deleted already existing file at {} before writing new content", localFilePath);
}
logger.debug("Storing configuration in file: {}", localFilePath);
@@ -183,7 +227,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
/** {@inheritDoc} */
@Override
- public void clearConfiguration() {
+ public synchronized void clearConfiguration() {
for (String path : savedFiles) {
logger.debug("Removing saved file {}", path);
File file = new File(path);
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
index 6229e96..9d3dd4a 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -20,6 +20,7 @@ package org.apache.oodt.config.distributed;
import org.apache.commons.io.FileUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.Constants;
import org.apache.oodt.config.distributed.utils.CuratorUtils;
import org.apache.zookeeper.data.Stat;
@@ -83,6 +84,14 @@ public class DistributedConfigurationPublisher {
logger.info("Using zookeeper connect string : {}", connectString);
startZookeeper();
+
+ try {
+ logger.debug("Creating ZNode paths");
+ zNodePaths.createZNodes(client);
+ } catch (Exception e) {
+ logger.error("Error occurred when creating initial ZNode paths", e);
+ throw new IllegalStateException("Unable to create ZNode paths", e);
+ }
}
/**
@@ -145,7 +154,8 @@ public class DistributedConfigurationPublisher {
*/
public boolean verifyPublishedConfiguration() {
try {
- return verifyPublishedConfiguration(propertiesFiles, true) && verifyPublishedConfiguration(configFiles, false);
+ return verifyPublishedConfiguration(propertiesFiles, true) &&
+ verifyPublishedConfiguration(configFiles, false);
} catch (Exception e) {
logger.error("Error occurred when checking published config", e);
return false;
@@ -165,6 +175,17 @@ public class DistributedConfigurationPublisher {
logger.info("Configuration cleared!");
}
+ /**
+ * Notifies the watching {@link org.apache.oodt.config.ConfigurationManager}s about the configuration change
+ *
+ * @param type {@link ConfigEventType}
+ * @throws Exception
+ */
+ public void notifyConfigEvent(ConfigEventType type) throws Exception {
+ logger.info("Notifying event: '{}' to configuration managers of {}", type, component);
+ client.setData().forPath(zNodePaths.getNotificationsZNodePath(), type.toString().getBytes());
+ }
+
private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
String filePath = entry.getKey();
@@ -173,7 +194,8 @@ public class DistributedConfigurationPublisher {
String content = getFileContent(filePath);
- String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) : zNodePaths.getConfigurationZNodePath(relativeZNodePath);
+ String zNodePath = isProperties ? zNodePaths.getPropertiesZNodePath(relativeZNodePath) :
+ zNodePaths.getConfigurationZNodePath(relativeZNodePath);
if (client.checkExists().forPath(zNodePath) != null) {
byte[] bytes = client.getData().forPath(zNodePath);
String existingData = new String(bytes);
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
index cf3ca00..6324ac1 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/ZNodePaths.java
@@ -17,9 +17,14 @@
package org.apache.oodt.config.distributed;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.oodt.config.distributed.utils.CuratorUtils;
+import org.apache.zookeeper.CreateMode;
+
import static org.apache.oodt.config.Constants.DEFAULT_PROJECT;
import static org.apache.oodt.config.Constants.ZPaths.COMPONENTS_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.CONFIGURATION_PATH_NAME;
+import static org.apache.oodt.config.Constants.ZPaths.NOTIFICATIONS_PATH;
import static org.apache.oodt.config.Constants.ZPaths.PROJECTS_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.PROPERTIES_PATH_NAME;
import static org.apache.oodt.config.Constants.ZPaths.SEPARATOR;
@@ -43,6 +48,9 @@ public class ZNodePaths {
private String configurationZNodePath;
private String configurationZNodeRoot;
+ /** ZNode to be watched for configuration changes. /projects/${project}/components/${component}/notifications */
+ private String notificationsZNodePath;
+
/**
* Creates the ZNode path structure accordingly to the <pre>componentName</pre> and <pre>propertiesFileNames</pre>
* given.
@@ -70,6 +78,21 @@ public class ZNodePaths {
configurationZNodePath = componentZNodeRoot + CONFIGURATION_PATH_NAME;
configurationZNodeRoot = configurationZNodePath + SEPARATOR;
+
+ notificationsZNodePath = componentZNodeRoot + NOTIFICATIONS_PATH;
+ }
+
+ /**
+ * Creates the initial ZNode structure in zookeeper. Supposed to be called by the {@link
+ * DistributedConfigurationPublisher}.
+ *
+ * @param client {@link CuratorFramework} instance
+ * @throws Exception
+ */
+ public void createZNodes(CuratorFramework client) throws Exception {
+ CuratorUtils.createZNodeIfNotExists(client, propertiesZNodePath, CreateMode.PERSISTENT, new byte[1]);
+ CuratorUtils.createZNodeIfNotExists(client, configurationZNodePath, CreateMode.PERSISTENT, new byte[1]);
+ CuratorUtils.createZNodeIfNotExists(client, notificationsZNodePath, CreateMode.PERSISTENT, new byte[1]);
}
public String getComponentZNodePath() {
@@ -99,4 +122,8 @@ public class ZNodePaths {
public String getLocalPropertiesFilePath(String zNodePath) {
return zNodePath.substring(propertiesZNodeRoot.length());
}
+
+ public String getNotificationsZNodePath() {
+ return notificationsZNodePath;
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
index 1ab1b30..50e567a 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
@@ -19,6 +19,7 @@ package org.apache.oodt.config.distributed.cli;
import org.apache.oodt.cas.cli.action.CmdLineAction;
import org.apache.oodt.cas.cli.exception.CmdLineActionException;
+import org.apache.oodt.config.ConfigEventType;
import org.apache.oodt.config.distributed.DistributedConfigurationPublisher;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
@@ -36,12 +37,9 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
*/
public class CLIAction extends CmdLineAction {
- public enum Action {
- PUBLISH, VERIFY, CLEAR
- }
-
private String connectString;
private String configFile = DEFAULT_CONFIG_PUBLISHER_XML;
+ private boolean notify = false;
private Action action;
@@ -60,12 +58,18 @@ public class CLIAction extends CmdLineAction {
switch (action) {
case PUBLISH:
publish(publisher);
+ if (notify) {
+ publisher.notifyConfigEvent(ConfigEventType.PUBLISH);
+ }
break;
case VERIFY:
verify(publisher);
break;
case CLEAR:
clear(publisher);
+ if (notify) {
+ publisher.notifyConfigEvent(ConfigEventType.CLEAR);
+ }
break;
}
publisher.destroy();
@@ -123,4 +127,16 @@ public class CLIAction extends CmdLineAction {
public void setConfigFile(String configFile) {
this.configFile = configFile;
}
+
+ public boolean isNotify() {
+ return notify;
+ }
+
+ public void setNotify(boolean notify) {
+ this.notify = notify;
+ }
+
+ public enum Action {
+ PUBLISH, VERIFY, CLEAR
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/main/resources/cmd-line-options.xml
----------------------------------------------------------------------
diff --git a/config/src/main/resources/cmd-line-options.xml b/config/src/main/resources/cmd-line-options.xml
index 698ad32..3aa506b 100644
--- a/config/src/main/resources/cmd-line-options.xml
+++ b/config/src/main/resources/cmd-line-options.xml
@@ -82,4 +82,35 @@
</bean>
</property>
</bean>
+
+ <bean id="notifyConfigChange" class="org.apache.oodt.cas.cli.option.AdvancedCmdLineOption">
+ <property name="shortOption" value="n"/>
+ <property name="longOption" value="notify"/>
+ <property name="description" value="Notify the configuration managers' about the configuration change done"/>
+ <property name="hasArgs" value="false"/>
+ <property name="requirementRules">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="publish" p:relation="OPTIONAL"/>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="clear" p:relation="OPTIONAL"/>
+ <bean class="org.apache.oodt.cas.cli.option.require.ActionDependencyRule"
+ p:actionName="verify" p:relation="OPTIONAL"/>
+ </list>
+ </property>
+ <property name="handler">
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToActionHandler">
+ <property name="applyToActions">
+ <list>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="publish" p:methodName="setNotify"/>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="verify" p:methodName="setNotify"/>
+ <bean class="org.apache.oodt.cas.cli.option.handler.ApplyToAction"
+ p:actionName="clear" p:methodName="setNotify"/>
+ </list>
+ </property>
+ </bean>
+ </property>
+ </bean>
</beans>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
----------------------------------------------------------------------
diff --git a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
index 0bf2dde..b553643 100644
--- a/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
+++ b/config/src/test/java/org/apache/oodt/config/distributed/DistributedConfigurationManagerTest.java
@@ -17,6 +17,7 @@
package org.apache.oodt.config.distributed;
+import org.apache.oodt.config.Component;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.distributed.cli.ConfigPublisher;
import org.apache.oodt.config.distributed.utils.ConfigUtils;
@@ -32,6 +33,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -50,6 +52,7 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
private static final String CONFIG_PUBLISHER_XML = "config-publisher.xml";
private List<DistributedConfigurationPublisher> publishers;
+ private Map<Component, ConfigurationManager> configurationManagers;
@Before
public void setUpTest() throws Exception {
@@ -65,21 +68,24 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(CONFIG_PUBLISHER_XML);
Map distributedConfigurationPublishers = applicationContext.getBeansOfType(DistributedConfigurationPublisher.class);
- publishers = new ArrayList<>(distributedConfigurationPublishers.values().size());
+ publishers = new ArrayList<>();
+ configurationManagers = new HashMap<>();
for (Object bean : distributedConfigurationPublishers.values()) {
DistributedConfigurationPublisher publisher = (DistributedConfigurationPublisher) bean;
+ System.setProperty(OODT_PROJECT, publisher.getProject());
System.setProperty(publisher.getComponent().getHome(), ".");
publishers.add(publisher);
+ configurationManagers.put(publisher.getComponent(), new DistributedConfigurationManager(publisher.getComponent()));
+ System.clearProperty(OODT_PROJECT);
}
}
@Test
public void loadConfigurationTest() throws Exception {
for (DistributedConfigurationPublisher publisher : publishers) {
- System.setProperty(OODT_PROJECT, publisher.getProject());
- ConfigurationManager configurationManager = new DistributedConfigurationManager(publisher.getComponent());
+ ConfigurationManager configurationManager = configurationManagers.get(publisher.getComponent());
configurationManager.loadConfiguration();
// Checking for configuration files
@@ -117,8 +123,63 @@ public class DistributedConfigurationManagerTest extends AbstractDistributedConf
File file = new File(localFile);
Assert.assertFalse(file.exists());
}
+ }
+ }
- System.clearProperty(OODT_PROJECT);
+ @Test
+ public void notifyConfigurationChangeTest() throws Exception {
+ // First publish config. Then check if config has downloaded locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "publish"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(true);
+
+ // Now clear config. Then check if config has deleted locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "clear"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(false);
+
+ // First publish config. Then check if config has downloaded locally.
+ ConfigPublisher.main(new String[]{
+ "-connectString", zookeeper.getConnectString(),
+ "-config", CONFIG_PUBLISHER_XML,
+ "-notify",
+ "-a", "publish"
+ });
+ Thread.sleep(5000);
+ checkFileExistence(true);
+ }
+
+ private void checkFileExistence(boolean exists) {
+ for (DistributedConfigurationPublisher publisher : publishers) {
+ for (String fileName : publisher.getPropertiesFiles().values()) {
+ fileName = ConfigUtils.fixForComponentHome(publisher.getComponent(), fileName);
+ File file = new File(fileName);
+ if (exists) {
+ Assert.assertTrue(file.exists());
+ } else {
+ Assert.assertFalse(file.exists());
+ }
+ }
+
+ for (String fileName : publisher.getConfigFiles().values()) {
+ fileName = ConfigUtils.fixForComponentHome(publisher.getComponent(), fileName);
+ File file = new File(fileName);
+ if (exists) {
+ Assert.assertTrue(file.exists());
+ } else {
+ Assert.assertFalse(file.exists());
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 5e84d83..f6e0fc3 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -244,7 +244,7 @@ the License.
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
index 4ac48bb..37d32d2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManager.java
@@ -54,6 +54,8 @@ import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
import org.apache.oodt.commons.date.DateUtils;
import org.apache.oodt.config.Component;
+import org.apache.oodt.config.ConfigEventType;
+import org.apache.oodt.config.ConfigurationListener;
import org.apache.oodt.config.ConfigurationManager;
import org.apache.oodt.config.ConfigurationManagerFactory;
import org.apache.xmlrpc.WebServer;
@@ -112,6 +114,18 @@ public class XmlRpcFileManager {
/** Configuration Manager instance which will handle the configuration aspect in distributed/standalone manner */
private ConfigurationManager configurationManager;
+ private ConfigurationListener configurationListener = new ConfigurationListener() {
+ @Override
+ public void configurationChanged(ConfigEventType type) {
+ switch (type) {
+ case PUBLISH:
+ refreshConfigAndPolicy();
+ break;
+ case CLEAR:
+ // TODO: 8/19/17 What should we do if the config has been cleared?
+ }
+ }
+ };
/**
* <p> Creates a new XmlRpcFileManager with the given metadata store factory, and the given data store factory, on the
@@ -135,8 +149,9 @@ public class XmlRpcFileManager {
}
configurationManager = ConfigurationManagerFactory.getConfigurationManager(Component.FILE_MANAGER, propertiesFiles);
-
+ configurationManager.addConfigurationListener(configurationListener);
this.loadConfiguration();
+
LOG.log(Level.INFO, "File Manager started by " + System.getProperty("user.name", "unknown"));
}
@@ -1224,6 +1239,7 @@ public class XmlRpcFileManager {
}
public boolean shutdown() {
+ configurationManager.removeConfigurationListener(configurationListener);
configurationManager.clearConfiguration();
if (this.webServer != null) {
this.webServer.shutdown();
http://git-wip-us.apache.org/repos/asf/oodt/blob/df1db1e4/pge/pom.xml
----------------------------------------------------------------------
diff --git a/pge/pom.xml b/pge/pom.xml
index 6fd4d72..72a33d9 100644
--- a/pge/pom.xml
+++ b/pge/pom.xml
@@ -27,7 +27,7 @@ the License.
<artifactId>cas-pge</artifactId>
<name>CAS PGE Adaptor Framework</name>
<description>Allows data processing jobs not written in conformance with the
- PCS PGE interface to be run within the PCS.</description>
+ PCS PGE (Production Generation Executive) interface to be run within the PCS.</description>
<!-- All dependencies should be listed in core/pom.xml and be ordered alphabetically by package and artifact.
Once the dependency is in the core pom, it can then be used in other modules without the version tags.
For example, within core/pom.xml:
[48/50] [abbrv] oodt git commit: Fix merge conflict.
Posted by ma...@apache.org.
Fix merge conflict.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/444a5a99
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/444a5a99
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/444a5a99
Branch: refs/heads/master
Commit: 444a5a9976a041e18d60a5573bec15829a054469
Parents: d444f6c b52f235
Author: Chris Mattmann <ch...@jpl.nasa.gov>
Authored: Thu Oct 12 10:19:50 2017 -0700
Committer: Chris Mattmann <ch...@jpl.nasa.gov>
Committed: Thu Oct 12 10:19:50 2017 -0700
----------------------------------------------------------------------
.../config/distributed/DistributedConfigurationManager.java | 6 +++++-
.../org/apache/oodt/config/distributed/utils/ConfigUtils.java | 1 +
.../src/test/resources/distributed/config/config-publisher.xml | 1 -
workflow/pom.xml | 3 +--
.../oodt/cas/workflow/system/AvroRpcWorkflowManager.java | 3 +--
workflow/src/test/resources/workflow.properties | 0
6 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/444a5a99/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --cc config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index fe36a02,d93c421..9f5ca85
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@@ -42,8 -42,7 +42,9 @@@ import java.util.concurrent.TimeUnit
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
import static org.apache.oodt.config.Constants.Properties.ZK_STARTUP_TIMEOUT;
+import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjectName;
+
+
/**
* Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to
* connect to zookeeper.
[27/50] [abbrv] oodt git commit: update Dockerfile with patch from
OODT-948
Posted by ma...@apache.org.
update Dockerfile with patch from OODT-948
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/e09d3683
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/e09d3683
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/e09d3683
Branch: refs/heads/master
Commit: e09d3683db7b308e19dfc42c195baa11039333e5
Parents: 7c104c4
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Mar 9 15:02:43 2017 +0000
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:42:39 2017 -0800
----------------------------------------------------------------------
docker/Dockerfile | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/e09d3683/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 708e9bc..3180f57 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,10 +1,10 @@
FROM ubuntu:latest
MAINTAINER Tom Barber
-ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64/
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
-RUN sudo apt-get update
-RUN sudo apt-get install -y openjdk-7-jre-headless openjdk-7-jre maven curl
+RUN apt-get update
+RUN apt-get install -y openjdk-8-jre-headless openjdk-8-jre maven curl
#RUN sudo apt-get install maven2
WORKDIR /usr/src
[43/50] [abbrv] oodt git commit: Improved java docs
Posted by ma...@apache.org.
Improved java docs
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/07aea0d3
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/07aea0d3
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/07aea0d3
Branch: refs/heads/master
Commit: 07aea0d364d6911a8a70221f3f6087749dc4e379
Parents: df1db1e
Author: Imesha Sudasingha <im...@gmail.com>
Authored: Wed Aug 23 19:32:48 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Oct 12 08:07:14 2017 +0530
----------------------------------------------------------------------
.../org/apache/oodt/config/ConfigEventType.java | 5 ++++
.../DistributedConfigurationManager.java | 15 +++++++++++-
.../DistributedConfigurationPublisher.java | 17 ++++++++++++++
.../oodt/config/distributed/cli/CLIAction.java | 24 ++++++++++++++++++++
.../config/distributed/cli/ConfigPublisher.java | 8 +++++++
.../config/distributed/utils/ConfigUtils.java | 5 ++++
.../config/distributed/utils/CuratorUtils.java | 19 ++++++++++------
.../AbstractDistributedConfigurationTest.java | 7 ++++++
8 files changed, 92 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
index 8a465b0..15c68b2 100644
--- a/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
+++ b/config/src/main/java/org/apache/oodt/config/ConfigEventType.java
@@ -17,6 +17,11 @@
package org.apache.oodt.config;
+/**
+ * An enum class to represent distributed configuration management related events.
+ *
+ * @author Imesha Sudasingha
+ */
public enum ConfigEventType {
PUBLISH("publish"),
CLEAR("clear");
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index 6dfb24b..fe36a02 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -47,6 +47,8 @@ import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjec
/**
* Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to
* connect to zookeeper.
+ * <p>
+ * This class can download configuration from zookeeper and clear configuration locally downloaded.
*
* @author Imesha Sudasingha.
*/
@@ -63,6 +65,7 @@ public class DistributedConfigurationManager extends ConfigurationManager {
/** {@link NodeCache} to watch for configuration change notifications */
private NodeCache nodeCache;
+ /** This is the listener which is going to be notified on the configuration changes happening in zookeeper */
private NodeCacheListener nodeCacheListener = new NodeCacheListener() {
@Override
public void nodeChanged() throws Exception {
@@ -150,6 +153,12 @@ public class DistributedConfigurationManager extends ConfigurationManager {
logger.info("NodeCache for watching configuration changes started successfully");
}
+ /**
+ * {@inheritDoc}
+ * <p>
+ * Since distributed configuration management treats properties files and configuration files in two different ways,
+ * they are loaded in different manners.
+ */
@Override
public synchronized void loadConfiguration() throws Exception {
logger.debug("Loading properties for : {}", component);
@@ -225,7 +234,11 @@ public class DistributedConfigurationManager extends ConfigurationManager {
savedFiles.add(localFilePath);
}
- /** {@inheritDoc} */
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method will additionally delete all the files downloaded earlier from zookeeper.
+ */
@Override
public synchronized void clearConfiguration() {
for (String path : savedFiles) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
index 9d3dd4a..23ff71d 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationPublisher.java
@@ -186,6 +186,13 @@ public class DistributedConfigurationPublisher {
client.setData().forPath(zNodePaths.getNotificationsZNodePath(), type.toString().getBytes());
}
+ /**
+ * Publishes configuration from local files to zookeeper.
+ *
+ * @param fileMapping source file to ZNode path mappings
+ * @param isProperties if true, files will be stored under {@link ZNodePaths#propertiesZNodePath}
+ * @throws Exception
+ */
private void publishConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
String filePath = entry.getKey();
@@ -220,6 +227,16 @@ public class DistributedConfigurationPublisher {
}
}
+ /**
+ * Verifies whether the content in local files given by keys of the <pre>fileMapping</pre> are identical to the
+ * configuration stored in zookeeper under ZNode paths given by <pre>${prefix}/{fileMapping.value}</pre>
+ *
+ * @param fileMapping src file to znode path mappings
+ * @param isProperties if true, treated as properties files and will look under {@link
+ * ZNodePaths#propertiesZNodePath}
+ * @return true, if all the configuration verification was successful and no error was detected.
+ * @throws Exception
+ */
private boolean verifyPublishedConfiguration(Map<String, String> fileMapping, boolean isProperties) throws Exception {
boolean noError = true;
for (Map.Entry<String, String> entry : fileMapping.entrySet()) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
index 50e567a..38b1775 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/CLIAction.java
@@ -33,6 +33,7 @@ import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
/**
* {@link CmdLineAction} specifying the verify, publish and clear tasks of distributed configuration management.
*
+ * @see ConfigPublisher
* @author Imesha Sudasingha
*/
public class CLIAction extends CmdLineAction {
@@ -83,6 +84,13 @@ public class CLIAction extends CmdLineAction {
System.out.println("Exiting CLI ...");
}
+ /**
+ * Publishes configuration files (which are stored locally at the moment) specified in {@link #configFile} to
+ * zookeeper.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void publish(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Publishing configuration for : %s", publisher.getComponent()));
@@ -91,6 +99,14 @@ public class CLIAction extends CmdLineAction {
System.out.println();
}
+ /**
+ * Verifies whether the content in the local files (which were published to zookeeper) are identical to the content
+ * that has actually been published. The file mapping is obtained from {@link #configFile}. Will print error
+ * messages accordingly if the verification fails.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void verify(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Verifying configuration for : %s", publisher.getComponent()));
@@ -103,6 +119,14 @@ public class CLIAction extends CmdLineAction {
System.out.println();
}
+ /**
+ * Clears all configuration published to zookeeper This will simply delete {@link
+ * org.apache.oodt.config.distributed.ZNodePaths#configurationZNodePath} and {@link
+ * org.apache.oodt.config.distributed.ZNodePaths#propertiesZNodePath} along with its children from zookeeper.
+ *
+ * @param publisher {@link DistributedConfigurationPublisher} instance
+ * @throws Exception
+ */
private void clear(DistributedConfigurationPublisher publisher) throws Exception {
System.out.println();
System.out.println(String.format("Clearing configuration for : %s", publisher.getComponent()));
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
index f4c45a7..f6f89cf 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/cli/ConfigPublisher.java
@@ -23,6 +23,14 @@ import org.slf4j.LoggerFactory;
/**
* Class with main method which gets invoked by the CLI.
+ * <p>
+ * Basic usage:
+ * <pre>
+ * ./config-publisher -connectString localhost:2181 </> -a {publish|verify|clear}
+ * </pre>
+ * <p>
+ * Optionally, users can give <pre>-notify</pre> option to notify the listening OODT components on the configuration
+ * change.
*
* @author Imesha Sudasingha
*/
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
index 6e4d677..bbb24b0 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
@@ -27,6 +27,11 @@ import org.slf4j.LoggerFactory;
import static org.apache.oodt.config.Constants.Properties.OODT_PROJECT;
import static org.apache.oodt.config.Constants.SEPARATOR;
+/**
+ * A utility class to be used for configuration related tasks.
+ *
+ * @author Imesha Sudasingha
+ */
public class ConfigUtils {
private static final Logger logger = LoggerFactory.getLogger(ConfigUtils.class);
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
index c559f55..57aefff 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/CuratorUtils.java
@@ -38,6 +38,11 @@ import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
import static org.apache.oodt.config.Constants.ZPaths.NAMESPACE;
import static org.apache.oodt.config.Constants.ZPaths.SEPARATOR;
+/**
+ * A set of utility methods to be used to do complex operations on zookeeper using {@link CuratorFramework}
+ *
+ * @author Imesha Sudasingha
+ */
public class CuratorUtils {
private static final Logger logger = LoggerFactory.getLogger(CuratorUtils.class);
@@ -88,8 +93,8 @@ public class CuratorUtils {
}
/**
- * Builds a {@link CuratorFramework} instance with given connect string. Will use the {@link CuratorUtils#logger} for
- * logging.
+ * Builds a {@link CuratorFramework} instance with given connect string. Will use the {@link CuratorUtils#logger}
+ * for logging.
*
* @param connectString zookeeper connect string
* @return CuratorFramework instance created
@@ -112,11 +117,11 @@ public class CuratorUtils {
int maxRetryCount = Integer.parseInt(System.getProperty(Constants.Properties.ZK_CONNECTION_TIMEOUT, "3"));
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .namespace(NAMESPACE)
- .connectString(connectString)
- .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
- .connectionTimeoutMs(connectionTimeoutMs)
- .sessionTimeoutMs(sessionTimeoutMs);
+ .namespace(NAMESPACE)
+ .connectString(connectString)
+ .retryPolicy(new ExponentialBackoffRetry(retryInitialWaitMs, maxRetryCount))
+ .connectionTimeoutMs(connectionTimeoutMs)
+ .sessionTimeoutMs(sessionTimeoutMs);
/*
* If authorization information is available, those will be added to the client. NOTE: These auth info are
http://git-wip-us.apache.org/repos/asf/oodt/blob/07aea0d3/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java b/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
index f8d421e..707be01 100644
--- a/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
+++ b/config/src/main/java/org/apache/oodt/config/test/AbstractDistributedConfigurationTest.java
@@ -25,6 +25,13 @@ import org.junit.BeforeClass;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
+/**
+ * An abstract class to be used for distributed configuration management related tests. Any test related to any OODT
+ * component can extend this class and connect to the {@link #zookeeper} instance started by this class for further
+ * steps.
+ *
+ * @author Imesha Sudasingha
+ */
public abstract class AbstractDistributedConfigurationTest {
protected static TestingServer zookeeper;
[42/50] [abbrv] oodt git commit: Merge branch
'feature/zookeeper-config' into development
Posted by ma...@apache.org.
Merge branch 'feature/zookeeper-config' into development
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/d444f6cd
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/d444f6cd
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/d444f6cd
Branch: refs/heads/master
Commit: d444f6cd3a65301eda39831e35e1535cc0cb72cd
Parents: ce89ad1 bd86cd6
Author: Chris Mattmann <ch...@jpl.nasa.gov>
Authored: Tue Oct 10 11:20:44 2017 -0700
Committer: Chris Mattmann <ch...@jpl.nasa.gov>
Committed: Tue Oct 10 11:20:44 2017 -0700
----------------------------------------------------------------------
config/pom.xml | 2 +-
.../org/apache/oodt/config/ConfigEventType.java | 50 +++++++++++++
.../oodt/config/ConfigurationListener.java | 31 ++++++++
.../oodt/config/ConfigurationManager.java | 23 ++++++
.../java/org/apache/oodt/config/Constants.java | 3 +
.../DistributedConfigurationManager.java | 79 +++++++++++++++++---
.../DistributedConfigurationPublisher.java | 43 ++++++++++-
.../oodt/config/distributed/ZNodePaths.java | 27 +++++++
.../oodt/config/distributed/cli/CLIAction.java | 48 +++++++++++-
.../config/distributed/cli/ConfigPublisher.java | 8 ++
.../config/distributed/utils/ConfigUtils.java | 5 ++
.../config/distributed/utils/CuratorUtils.java | 19 +++--
.../AbstractDistributedConfigurationTest.java | 7 ++
config/src/main/resources/cmd-line-options.xml | 31 ++++++++
.../DistributedConfigurationManagerTest.java | 69 ++++++++++++++++-
core/pom.xml | 2 +-
.../cas/filemgr/system/XmlRpcFileManager.java | 18 ++++-
pge/pom.xml | 2 +-
18 files changed, 435 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/d444f6cd/core/pom.xml
----------------------------------------------------------------------
diff --cc core/pom.xml
index 3489a56,f6e0fc3..779c0c1
--- a/core/pom.xml
+++ b/core/pom.xml
@@@ -254,13 -243,8 +254,13 @@@ the License
<version>8.7</version>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.12</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
+ <artifactId>curator-recipes</artifactId>
<version>3.3.0</version>
</dependency>
<dependency>
[05/50] [abbrv] oodt git commit: - gitignore
Posted by ma...@apache.org.
- gitignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/363d4bed
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/363d4bed
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/363d4bed
Branch: refs/heads/master
Commit: 363d4bed7ccaa6301b30111397fca9b9258aeece
Parents: 2611685
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:13:35 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:10:30 2017 -0800
----------------------------------------------------------------------
catalog/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/363d4bed/catalog/.gitignore
----------------------------------------------------------------------
diff --git a/catalog/.gitignore b/catalog/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/catalog/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[07/50] [abbrv] oodt git commit: - don't exclude slf4j-api
Posted by ma...@apache.org.
- don't exclude slf4j-api
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/5ca7bf2a
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/5ca7bf2a
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/5ca7bf2a
Branch: refs/heads/master
Commit: 5ca7bf2af5348ab7153549efa9362e4845600ca9
Parents: 6538bdc
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:23:12 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:10:49 2017 -0800
----------------------------------------------------------------------
curator/webapp/pom.xml | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/5ca7bf2a/curator/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/curator/webapp/pom.xml b/curator/webapp/pom.xml
index 5dd050a..e2d6254 100644
--- a/curator/webapp/pom.xml
+++ b/curator/webapp/pom.xml
@@ -81,22 +81,10 @@ the License.
<dependency>
<groupId>org.apache.wicket</groupId>
<artifactId>wicket</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.wicket</groupId>
<artifactId>wicket-extensions</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- </exclusions>
</dependency>
</dependencies>
<build>
[50/50] [abbrv] oodt git commit: - use Tika 1.13 - 1.16 isn't compat
with the commons-compress we are referencing
Posted by ma...@apache.org.
- use Tika 1.13 - 1.16 isn't compat with the commons-compress we are
referencing
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/8a3d5194
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/8a3d5194
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/8a3d5194
Branch: refs/heads/master
Commit: 8a3d5194424d8327e8b3ffc4a2d45f18b56e3371
Parents: c7b8c98
Author: Chris Mattmann <ch...@jpl.nasa.gov>
Authored: Thu Oct 12 10:42:04 2017 -0700
Committer: Chris Mattmann <ch...@jpl.nasa.gov>
Committed: Thu Oct 12 10:42:04 2017 -0700
----------------------------------------------------------------------
core/pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/8a3d5194/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 8c9a41f..63d2d1d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -444,12 +444,12 @@ the License.
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
- <version>1.16</version>
+ <version>1.13</version>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
- <version>1.16</version>
+ <version>1.13</version>
</dependency>
<dependency>
<groupId>org.apache.velocity</groupId>
[11/50] [abbrv] oodt git commit: ignore
Posted by ma...@apache.org.
ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/4827e9c1
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/4827e9c1
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/4827e9c1
Branch: refs/heads/master
Commit: 4827e9c1ce284a561c1f3dbd96e03a59b88b3345
Parents: ed2f4c5
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:31:57 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:11:26 2017 -0800
----------------------------------------------------------------------
protocol/http/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/4827e9c1/protocol/http/.gitignore
----------------------------------------------------------------------
diff --git a/protocol/http/.gitignore b/protocol/http/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/protocol/http/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[24/50] [abbrv] oodt git commit: OODT-943 update the plugin versions
Posted by ma...@apache.org.
OODT-943 update the plugin versions
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/a5d6730d
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/a5d6730d
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/a5d6730d
Branch: refs/heads/master
Commit: a5d6730de87cc43b62d4be730dd251ff5782b78b
Parents: 89d6bf1
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Nov 10 14:09:16 2016 +0000
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:42:05 2017 -0800
----------------------------------------------------------------------
core/pom.xml | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/a5d6730d/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 83bf104..55392c4 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -41,6 +41,13 @@ the License.
<jetty.version>6.1.25</jetty.version>
<sonar.language>java</sonar.language>
</properties>
+ <distributionManagement>
+ <site>
+ <id>www.yourcompany.com</id>
+ <url>scp://www.yourcompany.com/www/docs/project/</url>
+ </site>
+ </distributionManagement>
+
<!-- All dependencies should be listed in core/pom.xml and be ordered alphabetically by package and artifact.
Once the dependency is in the core pom, it can then be used in other modules without the version tags.
For example, within core/pom.xml:
@@ -499,11 +506,11 @@ the License.
<artifactId>slf4j-api</artifactId>
<version>1.7.12</version>
</dependency>
- <dependency>
+<!-- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j</artifactId>
<version>1.7.12</version>
- </dependency>
+ </dependency>-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
@@ -646,7 +653,7 @@ mm
<!-- Produce JavaDoc -->
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.9</version>
+ <version>2.10.4</version>
<configuration>
<outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
@@ -701,7 +708,7 @@ mm
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-site-plugin</artifactId>
- <version>2.0-beta-7</version>
+ <version>3.5.1</version>
<configuration>
<outputEncoding>UTF-8</outputEncoding>
</configuration>
[39/50] [abbrv] oodt git commit: Merge master versions of files.
Posted by ma...@apache.org.
Merge master versions of files.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9b0b61ec
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9b0b61ec
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9b0b61ec
Branch: refs/heads/master
Commit: 9b0b61ec0e1fd5c36a1ea9c9902a947ba0d20ba1
Parents: 835f16d
Author: Chris Mattmann <ma...@apache.org>
Authored: Sun Sep 3 11:56:59 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sun Sep 3 11:56:59 2017 -0700
----------------------------------------------------------------------
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 10 +-
.../oodt/cas/filemgr/tools/DeleteProduct.java | 303 ++++++++++---------
.../oodt/cas/filemgr/tools/QueryTool.java | 127 ++++----
.../oodt/cas/filemgr/tools/SolrIndexer.java | 228 +++++++-------
.../LuceneWorkflowInstanceRepository.java | 53 +++-
5 files changed, 384 insertions(+), 337 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index a8c0253..59ded25 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -902,7 +902,8 @@ public class LuceneCatalog implements Catalog {
*/
public ProductPage getNextPage(ProductType type, ProductPage currentPage) {
if(type==null){
- return null;
+ LOG.warning("getNextPage: Provided type was null: Returning blank page.");
+ return ProductPage.blankPage();
}
if (currentPage == null) {
return getFirstPage(type);
@@ -926,11 +927,11 @@ public class LuceneCatalog implements Catalog {
"CatalogException getting next page for product type: ["
+ type.getProductTypeId()
+ "] from catalog: Message: " + e.getMessage());
- return null;
+ return ProductPage.blankPage();
}
// There are no products and thus no next page
if (products == null || (products.size() == 0)) {
- return null;
+ return ProductPage.blankPage();
}
nextPage.setPageProducts(products);
@@ -945,7 +946,8 @@ public class LuceneCatalog implements Catalog {
*/
public ProductPage getPrevPage(ProductType type, ProductPage currentPage) {
if(type==null){
- return null;
+ LOG.warning("getPrevPage: Provided type was null: Returning blank page.");
+ return ProductPage.blankPage();
}
if (currentPage == null) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
index 42b07e9..654c85a 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/DeleteProduct.java
@@ -22,9 +22,7 @@ import org.apache.oodt.cas.filemgr.structs.Reference;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
//JDK imports
import java.io.BufferedReader;
@@ -43,180 +41,193 @@ import java.util.Vector;
* @author woollard
* @version $Revision$
*
- * <p>
- * A utility class that deletes products in the File Manager Catalog based on
- * productID.
- * </p>
+ * <p>
+ * A utility class that deletes products in the File Manager Catalog
+ * based on productID.
+ * </p>
*
*/
-@Deprecated
public class DeleteProduct {
- /* our log stream */
- private static final Logger LOG = Logger.getLogger(DeleteProduct.class.getName());
+ /* our log stream */
+ private static final Logger LOG = Logger
+ .getLogger(DeleteProduct.class.getName());
- /* our File Manager client */
- private FileManagerClient client = null;
+ /* our File Manager client */
+ private XmlRpcFileManagerClient client = null;
- /* whether or not we should commit our deletions */
- private boolean commit = true;
+ /* whether or not we should commit our deletions */
+ private boolean commit = true;
- public DeleteProduct(String fileManagerUrl, boolean commit) {
- try {
- client = RpcCommunicationFactory.createClient(new URL(fileManagerUrl));
- } catch (Exception e) {
- LOG.log(Level.SEVERE,
- "Unable to create file manager client: Message: "
- + e.getMessage() + ": errors to follow");
- }
+ public DeleteProduct(String fileManagerUrl, boolean commit) {
+ try {
+ client = new XmlRpcFileManagerClient(new URL(fileManagerUrl));
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, "Unable to create file manager client: Message: "
+ + e.getMessage() + ": errors to follow");
+ }
- this.commit = commit;
+ this.commit = commit;
- if (!this.commit) {
- LOG.log(Level.INFO, "Commit disabled.");
- } else {
- LOG.log(Level.INFO, "Commit enabled.");
- }
+ if (!this.commit) {
+ LOG.log(Level.INFO, "Commit disabled.");
+ } else {
+ LOG.log(Level.INFO, "Commit enabled.");
}
+ }
- /**
- * @param args
- */
- public static void main(String[] args) {
- String productId = null;
- String fileManagerUrl = null;
- boolean commitChanges = true;
- boolean readFromStdIn = false;
-
- String usage = "DeleteProduct --productID <product id> "
- + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n";
-
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("--productID")) {
- productId = args[++i];
- } else if (args[i].equals("--fileManagerUrl")) {
- fileManagerUrl = args[++i];
- } else if (args[i].equals("--read")) {
- readFromStdIn = true;
- } else if (args[i].equals("--nocommit")) {
- commitChanges = false;
- }
- }
-
- if ((productId == null && !readFromStdIn) || fileManagerUrl == null) {
- System.err.println(usage);
- System.exit(1);
- }
+ public void remove(String productId) {
+ Product target = null;
- DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges);
- if (readFromStdIn) {
- List prodIds = readProdIdsFromStdin();
- for (Object prodId1 : prodIds) {
- String prodId = (String) prodId1;
- remover.remove(prodId);
- }
- } else {
- remover.remove(productId);
- }
+ try {
+ target = client.getProductById(productId);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to obtain product : [" + productId + "] from file manager: ["
+ + client.getFileManagerUrl() + "]: Message: " + e.getMessage());
+ }
+ if (target == null) {
+ // could not file product
+ return;
}
- private static List readProdIdsFromStdin() {
- List prodIds = new Vector();
- BufferedReader br;
+ // delete references first
+ Vector refs = new Vector();
- br = new BufferedReader(new InputStreamReader(System.in));
+ try {
+ refs = (Vector) client.getProductReferences(target);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to obtain references for product : [" + productId
+ + "] from file manager: [" + client.getFileManagerUrl()
+ + "]: Message: " + e.getMessage());
+ }
- String line = null;
+ for (Object ref1 : refs) {
+ Reference ref = (Reference) ref1;
+ if (commit) {
try {
- while ((line = br.readLine()) != null) {
- prodIds.add(line);
- }
- } catch (IOException e) {
- LOG.log(Level.WARNING, "Error reading prod id: line: [" + line
- + "]: Message: " + e.getMessage(), e);
- } finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
-
+ client.removeFile(
+ new File(new URI(ref.getDataStoreReference())).getAbsolutePath());
+ } catch (DataTransferException e) {
+ LOG.log(Level.WARNING,
+ "Unable to delete reference : [" + ref.getDataStoreReference()
+ + "] for product : [" + productId + "] from file manager : ["
+ + client.getFileManagerUrl() + "]: Message: "
+ + e.getMessage());
+ } catch (URISyntaxException e) {
+ LOG.log(Level.WARNING,
+ "uri syntax exception getting file absolute path from URI: ["
+ + ref.getDataStoreReference() + "]: Message: "
+ + e.getMessage());
}
+ } else {
+ LOG.log(Level.INFO,
+ "Delete file: [" + ref.getDataStoreReference() + "]");
+ }
+
+ }
- return prodIds;
+ if (commit) {
+
+ // now delete product
+ try {
+ client.removeProduct(target);
+ } catch (CatalogException e) {
+ LOG.log(Level.WARNING,
+ "Unable to remove product : [" + productId
+ + "] from file manager: [" + client.getFileManagerUrl()
+ + "]: Message: " + e.getMessage());
+ }
+ } else {
+ LOG.log(Level.INFO, "Remote catalog entry for product: ["
+ + target.getProductName() + "]");
}
- private void remove(String productId) {
- Product target = null;
+ }
+
+ /**
+ * @return the commit
+ */
+ public boolean isCommit() {
+ return commit;
+ }
+
+ /**
+ * @param commit
+ * the commit to set
+ */
+ public void setCommit(boolean commit) {
+ this.commit = commit;
+ }
+
+ /**
+ * @param args
+ */
+ public static void main(String[] args) {
+ String productId = null;
+ String fileManagerUrl = null;
+ boolean commitChanges = true;
+ boolean readFromStdIn = false;
+
+ String usage = "DeleteProduct --productID <product id> "
+ + "--fileManagerUrl <url to file manager> [--read] [--nocommit]\n";
+
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("--productID")) {
+ productId = args[++i];
+ } else if (args[i].equals("--fileManagerUrl")) {
+ fileManagerUrl = args[++i];
+ } else if (args[i].equals("--read")) {
+ readFromStdIn = true;
+ } else if (args[i].equals("--nocommit")) {
+ commitChanges = false;
+ }
+ }
- try {
- target = client.getProductById(productId);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING, "Unable to obtain product : [" + productId
- + "] from file manager: [" + client.getFileManagerUrl()
- + "]: Message: " + e.getMessage());
- }
+ if ((productId == null && !readFromStdIn) || fileManagerUrl == null) {
+ System.err.println(usage);
+ System.exit(1);
+ }
- if (target == null) {
- // could not file product
- return;
- }
+ DeleteProduct remover = new DeleteProduct(fileManagerUrl, commitChanges);
+ if (readFromStdIn) {
+ List prodIds = readProdIdsFromStdin();
+ for (Object prodId1 : prodIds) {
+ String prodId = (String) prodId1;
+ remover.remove(prodId);
+ }
+ } else {
+ remover.remove(productId);
+ }
- // delete references first
- Vector refs = new Vector();
+ }
- try {
- refs = (Vector) client.getProductReferences(target);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING,
- "Unable to obtain references for product : [" + productId
- + "] from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- }
+ private static List readProdIdsFromStdin() {
+ List prodIds = new Vector();
+ BufferedReader br;
- for (Object ref1 : refs) {
- Reference ref = (Reference) ref1;
-
- if (commit) {
- try {
- client.removeFile(new File(new URI(ref
- .getDataStoreReference())).getAbsolutePath());
- } catch (DataTransferException e) {
- LOG.log(Level.WARNING, "Unable to delete reference : ["
- + ref.getDataStoreReference() + "] for product : ["
- + productId + "] from file manager : ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- } catch (URISyntaxException e) {
- LOG.log(Level.WARNING,
- "uri syntax exception getting file absolute path from URI: ["
- + ref.getDataStoreReference()
- + "]: Message: " + e.getMessage());
- }
- } else {
- LOG.log(Level.INFO, "Delete file: ["
- + ref.getDataStoreReference() + "]");
- }
+ br = new BufferedReader(new InputStreamReader(System.in));
- }
+ String line = null;
- if (commit) {
-
- // now delete product
- try {
- client.removeProduct(target);
- } catch (CatalogException e) {
- LOG.log(Level.WARNING, "Unable to remove product : ["
- + productId + "] from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
- }
- } else {
- LOG.log(Level.INFO, "Remote catalog entry for product: ["
- + target.getProductName() + "]");
- }
+ try {
+ while ((line = br.readLine()) != null) {
+ prodIds.add(line);
+ }
+ } catch (IOException e) {
+ LOG.log(Level.WARNING, "Error reading prod id: line: [" + line
+ + "]: Message: " + e.getMessage(), e);
+ } finally {
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
+
+ return prodIds;
+ }
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index 7b37f90..27e37d2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -32,9 +32,7 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.QueryFormulationException;
import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException;
import org.apache.oodt.cas.filemgr.structs.query.ComplexQuery;
import org.apache.oodt.cas.filemgr.structs.query.QueryResult;
-import org.apache.oodt.cas.filemgr.system.FileManagerClient;
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
-import org.apache.oodt.cas.filemgr.util.RpcCommunicationFactory;
import org.apache.oodt.cas.filemgr.util.SqlParser;
import java.net.MalformedURLException;
@@ -53,12 +51,11 @@ import java.util.logging.Logger;
* A tool to return product ids given a {@link Query} against the File Manager.
* </p>
*/
-@Deprecated
public final class QueryTool {
private static String freeTextBlock = "__FREE__";
- private FileManagerClient client = null;
+ private XmlRpcFileManagerClient client = null;
private enum QueryType { LUCENE, SQL }
@@ -67,26 +64,12 @@ public final class QueryTool {
public QueryTool(URL fmUrl) throws InstantiationException {
try {
- client = RpcCommunicationFactory.createClient(fmUrl);
+ client = new XmlRpcFileManagerClient(fmUrl);
} catch (ConnectionException e) {
throw new InstantiationException(e.getMessage());
}
}
- public static Query parseQuery(String query) {
- QueryParser parser;
- // note that "__FREE__" is a control work for free text searching
- parser = new QueryParser(freeTextBlock, new CASAnalyzer());
-
- Query luceneQ = null;
- try {
- luceneQ = (Query) parser.parse(query);
- } catch (ParseException e) {
- System.out.println("Error parsing query text.");
- System.exit(-1);
- }
- return luceneQ;
- }
public List query(org.apache.oodt.cas.filemgr.structs.Query query) {
List prodIds = new Vector();
@@ -118,55 +101,55 @@ public final class QueryTool {
return prodIds;
}
-
+
+
+ public static Query parseQuery(String query) {
+ QueryParser parser;
+ // note that "__FREE__" is a control work for free text searching
+ parser = new QueryParser(freeTextBlock, new CASAnalyzer());
+ Query luceneQ = null;
+ try {
+ luceneQ = (Query) parser.parse(query);
+ } catch (ParseException e) {
+ System.out.println("Error parsing query text.");
+ System.exit(-1);
+ }
+ return luceneQ;
+ }
+
public void generateCASQuery(
- org.apache.oodt.cas.filemgr.structs.Query casQuery,
- Query luceneQuery) {
- if (luceneQuery instanceof TermQuery) {
- Term t = ((TermQuery) luceneQuery).getTerm();
- if (!t.field().equals(freeTextBlock)) {
- casQuery.addCriterion(new TermQueryCriteria(t.field(),
- t.text()));
- }
- } else if (luceneQuery instanceof PhraseQuery) {
- Term[] t = ((PhraseQuery) luceneQuery).getTerms();
- if (!t[0].field().equals(freeTextBlock)) {
- for (Term aT : t) {
- casQuery.addCriterion(new TermQueryCriteria(
- aT.field(), aT.text()));
- }
- }
- } else if (luceneQuery instanceof TermRangeQuery) {
- BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
- BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
- } else if (luceneQuery instanceof BooleanQuery) {
- List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
- for (BooleanClause clause : clauses) {
- generateCASQuery(casQuery, (clause).getQuery());
+ org.apache.oodt.cas.filemgr.structs.Query casQuery,
+ Query luceneQuery) {
+ if (luceneQuery instanceof TermQuery) {
+ Term t = ((TermQuery) luceneQuery).getTerm();
+ if (!t.field().equals(freeTextBlock)) {
+ casQuery.addCriterion(new TermQueryCriteria(t.field(),
+ t.text()));
+ }
+ } else if (luceneQuery instanceof PhraseQuery) {
+ Term[] t = ((PhraseQuery) luceneQuery).getTerms();
+ if (!t[0].field().equals(freeTextBlock)) {
+ for (Term aT : t) {
+ casQuery.addCriterion(new TermQueryCriteria(
+ aT.field(), aT.text()));
}
- } else {
- throw new RuntimeException(
- "Error parsing query! Cannot determine clause type: ["
- + luceneQuery.getClass().getName() + "] !");
}
- }
-
- private List safeGetProductTypes() {
- List prodTypes = null;
-
- try {
- prodTypes = client.getProductTypes();
- } catch (RepositoryManagerException e) {
- LOG.log(Level.WARNING,
- "Error obtaining product types from file manager: ["
- + client.getFileManagerUrl() + "]: Message: "
- + e.getMessage());
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
+ } else if (luceneQuery instanceof BooleanQuery) {
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
+ for (BooleanClause clause : clauses) {
+ generateCASQuery(casQuery, (clause).getQuery());
}
-
- return prodTypes;
+ } else {
+ throw new RuntimeException(
+ "Error parsing query! Cannot determine clause type: ["
+ + luceneQuery.getClass().getName() + "] !");
}
-
+}
+
public static void main(String[] args)
throws MalformedURLException, InstantiationException, CatalogException, QueryFormulationException,
ConnectionException {
@@ -181,7 +164,7 @@ public final class QueryTool {
+ " -query <query> \n"
+ " -sortBy <metadata-key> \n"
+ " -outputFormat <output-format-string> \n";
-
+
String fmUrlStr = null, queryStr = null, sortBy = null, outputFormat = null, delimiter = null;
QueryType queryType = null;
for (int i = 0; i < args.length; i++) {
@@ -241,6 +224,22 @@ public final class QueryTool {
}
}
+
+
+ private List safeGetProductTypes() {
+ List prodTypes = null;
+
+ try {
+ prodTypes = client.getProductTypes();
+ } catch (RepositoryManagerException e) {
+ LOG.log(Level.WARNING,
+ "Error obtaining product types from file manager: ["
+ + client.getFileManagerUrl() + "]: Message: "
+ + e.getMessage());
+ }
+
+ return prodTypes;
+ }
private static String performSqlQuery(String query, String sortBy, String outputFormat, String delimiter, String filemgrUrl)
throws MalformedURLException, CatalogException, ConnectionException, QueryFormulationException {
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
index 6d453bd..16ca7f1 100755
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/SolrIndexer.java
@@ -27,7 +27,6 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
-import org.apache.http.impl.client.SystemDefaultHttpClient;
import org.apache.oodt.cas.filemgr.metadata.CoreMetKeys;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductPage;
@@ -39,8 +38,9 @@ import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.metadata.SerializableMetadata;
import org.apache.oodt.cas.metadata.util.PathUtils;
+import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.common.SolrInputDocument;
import org.springframework.util.StringUtils;
@@ -73,31 +73,31 @@ public class SolrIndexer {
private final static String ACCESS_KEY = "access.key";
private final static String ACCESS_URL = "access.url";
private final static String PRODUCT_NAME = "CAS.ProductName";
- private final HttpSolrClient server;
private IndexerConfig config = null;
+ private final SolrServer server;
private String fmUrl;
private String solrUrl;
private static Logger LOG = Logger.getLogger(SolrIndexer.class.getName());
private final static SimpleDateFormat solrFormat = new SimpleDateFormat(
- "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+ "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
/**
* Constructor reads in the configuration and initiates the connection to the
* Solr instance.
- *
+ *
* @param solrUrl
* URL for the Solr instance.
* @param fmUrl
* URL for the File Manager instance.
*/
public SolrIndexer(String solrUrl, String fmUrl)
- throws InstantiationException {
+ throws InstantiationException {
InputStream input = null;
String filename;
try {
LOG.info("System property " + SOLR_INDEXER_CONFIG + " set to "
- + System.getProperty(SOLR_INDEXER_CONFIG));
+ + System.getProperty(SOLR_INDEXER_CONFIG));
filename = System.getProperty(SOLR_INDEXER_CONFIG);
if (filename != null) {
LOG.info("Reading config from " + filename);
@@ -109,7 +109,7 @@ public class SolrIndexer {
config = new IndexerConfig(input);
} catch (IOException e) {
LOG
- .severe("Could not read in configuration for indexer from classpath or file");
+ .severe("Could not read in configuration for indexer from classpath or file");
throw new InstantiationException(e.getMessage());
} finally {
if (input != null) {
@@ -132,10 +132,14 @@ public class SolrIndexer {
}
LOG.info("Using Solr: " + this.solrUrl + " FileManager: " + this.fmUrl);
- SystemDefaultHttpClient httpClient = new SystemDefaultHttpClient();
- server = new HttpSolrClient.Builder(this.solrUrl).build();
-// server = new SolrClient(this.solrUrl);
+
+ try {
+ server = new CommonsHttpSolrServer(this.solrUrl);
+ } catch (MalformedURLException e) {
+ LOG.severe("Could not connect to Solr server " + this.solrUrl);
+ throw new InstantiationException(e.getMessage());
+ }
}
@@ -162,7 +166,7 @@ public class SolrIndexer {
/**
* This method transforms the product metadata into a Solr document.
- *
+ *
* @param metadata
* The metadata object for the product to index.
* @return Returns the SolrInputDocument containing product metadata.
@@ -170,8 +174,6 @@ public class SolrIndexer {
private SolrInputDocument getSolrDocument(Metadata metadata) {
SolrInputDocument doc = new SolrInputDocument();
// Only grab metadata which have a mapping in the indexer.properties
- String official = "";
-
for (Object objKey : config.getMapProperties().keySet()) {
// The key in the metadata object
String key = (String) objKey;
@@ -182,21 +184,19 @@ public class SolrIndexer {
for (String value : values) {
// Add each metadata value into the
if (value != null && !config.getIgnoreValues().contains(value.trim())) {
- official += " Key: " + key + " Value: " + value;
+ LOG.fine("Adding field: " + fieldName + " value: " + value);
doc.addField(fieldName, value);
}
}
}
}
- // LOG.info("KEYS WERE:" +official);
-
return doc;
}
/**
* This method adds a single product extracted from a metadata file to the
* Solr index.
- *
+ *
* @param file
* The file containing product metadata.
* @param delete
@@ -206,24 +206,24 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexMetFile(File file, boolean delete)
- throws
- SolrServerException {
+ throws
+ SolrServerException {
LOG.info("Attempting to index product from metadata file.");
try {
SerializableMetadata metadata = new SerializableMetadata("UTF-8", false);
metadata.loadMetadataFromXmlStream(new FileInputStream(file));
metadata.addMetadata("id", metadata.getMetadata("CAS."
- + CoreMetKeys.PRODUCT_ID));
+ + CoreMetKeys.PRODUCT_ID));
metadata.addMetadata(config.getProperty(ACCESS_KEY), config
- .getProperty(ACCESS_URL)
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .getProperty(ACCESS_URL)
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
if (delete) {
server
- .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ .deleteById(metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
}
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: "
- + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
+ + metadata.getMetadata("CAS." + CoreMetKeys.PRODUCT_ID));
} catch (InstantiationException e) {
LOG.severe("Could not instantiate metadata object: " + e.getMessage());
} catch (FileNotFoundException e) {
@@ -236,7 +236,7 @@ public class SolrIndexer {
/**
* This method indexes all product types retrieved from the File Manager to
* the Solr index.
- *
+ *
* @param delete
* Flag indicating whether each product type retrieved from the File
* Manager should be deleted from the index.
@@ -247,7 +247,7 @@ public class SolrIndexer {
LOG.info("Indexing product types...");
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
@@ -256,9 +256,9 @@ public class SolrIndexer {
metadata.addMetadata("id", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeId", type.getProductTypeId());
metadata.addMetadata("CAS.ProductTypeDescription", type
- .getDescription());
+ .getDescription());
metadata.addMetadata("CAS.ProductTypeRepositoryPath", type
- .getProductRepositoryPath());
+ .getProductRepositoryPath());
metadata.addMetadata("CAS.ProductTypeVersioner", type.getVersioner());
metadata.addMetadata("CAS.ProductTypeName", type.getName());
metadata.addMetadata("ProductType", "ProductType");
@@ -268,7 +268,7 @@ public class SolrIndexer {
server.deleteById(type.getProductTypeId());
} catch (Exception e) {
LOG.severe("Could not delete product type " + type.getName()
- + " from index: " + e.getMessage());
+ + " from index: " + e.getMessage());
}
}
try {
@@ -277,7 +277,7 @@ public class SolrIndexer {
LOG.info("Indexed product type: " + type.getName());
} catch (Exception e) {
LOG.severe("Could not index " + type.getName() + ": "
- + e.getMessage());
+ + e.getMessage());
}
} else {
LOG.info("Ignoring product type: " + type.getName());
@@ -289,13 +289,13 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
LOG.info("Finished indexing product types.");
}
-
+
/**
- * Suppresses exception that occurred with older file managers.
+ * Suppresses exception that occurred with older file managers.
*/
private ProductPage safeFirstPage(XmlRpcFileManagerClient fmClient, ProductType type) {
ProductPage page = null;
@@ -311,7 +311,7 @@ public class SolrIndexer {
* This method indexes all products retrieved from the File Manager to the
* Solr index. Metadata from the product's associated ProductType is also
* included.
- *
+ *
* @param delete
* Flag indicating whether each product retrieved from the File
* Manager should be deleted from the index.
@@ -322,29 +322,28 @@ public class SolrIndexer {
LOG.info("Indexing products...");
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
LOG.info("Retrieving list of product types.");
List<ProductType> types = fmClient.getProductTypes();
for (ProductType type : types) {
if (!config.getIgnoreTypes().contains(type.getName().trim())) {
- ProductPage page = safeFirstPage(fmClient, type);
LOG.info("Paging through products for product type: "
- + type.getName()+" Total pages are "+page.getTotalPages());
+ + type.getName());
+ ProductPage page = safeFirstPage(fmClient, type);
while (page != null) {
- for (Product product : page.getPageProducts()) {
+ for (Product product : page.getPageProducts()) {
try {
- String p = product.getProductId();
- Metadata m = fmClient.getMetadata(product);
- this.indexProduct(p, m, type.getTypeMetadata());
+ this.indexProduct(product.getProductId(), fmClient
+ .getMetadata(product), type.getTypeMetadata());
} catch (Exception e) {
LOG.severe("Could not index " + product.getProductId() + ": "
- + e.getLocalizedMessage());
+ + e.getMessage());
}
}
- if (page.getPageNum() >= page.getTotalPages() || page.isLastPage()) {
- break;
- }
- page = fmClient.getNextPage(type, page);
+ if (page.isLastPage()) {
+ break;
+ }
+ page = fmClient.getNextPage(type, page);
}
}
}
@@ -355,10 +354,10 @@ public class SolrIndexer {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve products from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (RepositoryManagerException e) {
LOG.severe("Could not retrieve product types from File Manager: "
- + e.getMessage());
+ + e.getMessage());
}
}
@@ -366,50 +365,39 @@ public class SolrIndexer {
* This method adds a single product retrieved from the File Manager by its
* product identifier to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productId
* The identifier of the product (CAS.ProductId).
* @throws SolrServerException
* When an error occurs communicating with the Solr server instance.
*/
public void indexProduct(String productId)
- throws SolrServerException {
+ throws SolrServerException {
LOG.info("Attempting to index product: " + productId);
try {
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
Product product = fmClient.getProductById(productId);
Metadata productMetadata = fmClient.getMetadata(product);
- List<String> keys = productMetadata.getAllKeys();
- String logger = "";
- for(String k :keys){
- logger+=" key: "+keys;
- List<String> values = productMetadata.getAllValues(k);
-
- for(String v : values){
- logger+=" value: "+v + ", ";
- }
- }
- LOG.info("Metadata: "+ logger);
indexProduct(product.getProductId(), productMetadata, product
- .getProductType().getTypeMetadata());
+ .getProductType().getTypeMetadata());
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
-
+
/**
* This method adds a single product retrieved from the File Manager by its
* product name to the Solr index. Metadata from the ProductType is also
* included.
- *
+ *
* @param productName
* The identifier of the product (CAS.ProductId).
* @param delete
@@ -419,7 +407,7 @@ public class SolrIndexer {
* When an error occurs communicating with the Solr server instance.
*/
public void indexProductByName(String productName, boolean delete) throws SolrServerException {
-
+
LOG.info("Attempting to index product: " + productName);
try {
@@ -439,27 +427,27 @@ public class SolrIndexer {
}
XmlRpcFileManagerClient fmClient = new XmlRpcFileManagerClient(new URL(
- this.fmUrl));
+ this.fmUrl));
Product product = fmClient.getProductByName(productName);
Metadata productMetadata = fmClient.getMetadata(product);
// NOTE: delete (by id) is now false
indexProduct(product.getProductId(), productMetadata, product.getProductType().getTypeMetadata());
-
+
} catch (MalformedURLException e) {
LOG.severe("File Manager URL is malformed: " + e.getMessage());
} catch (ConnectionException e) {
LOG.severe("Could not connect to File Manager: " + e.getMessage());
} catch (CatalogException e) {
LOG.severe("Could not retrieve product from File Manager: "
- + e.getMessage());
+ + e.getMessage());
} catch (java.text.ParseException e) {
LOG.severe("Could not format date: " + e.getMessage());
}
}
private void indexProduct(String productId, Metadata productMetadata,
- Metadata typeMetadata) throws SolrServerException,
- java.text.ParseException {
+ Metadata typeMetadata) throws SolrServerException,
+ java.text.ParseException {
Metadata metadata = new Metadata();
metadata.addMetadata("id", productId);
// Add in product type metadata
@@ -485,7 +473,7 @@ public class SolrIndexer {
server.add(this.getSolrDocument(metadata));
LOG.info("Indexed product: " + productId);
} catch (IOException e) {
- LOG.severe("Could not index product: " + productId+ "Exception:"+e.getLocalizedMessage());
+ LOG.severe("Could not index product: " + productId);
}
} else {
LOG.info("Could not find metadata for product: " + productId);
@@ -494,7 +482,7 @@ public class SolrIndexer {
/**
* This method deletes a single product identified by a productID from the Solr index
- *
+ *
* @param productId
* @throws IOException
* @throws SolrServerException
@@ -506,7 +494,7 @@ public class SolrIndexer {
/**
* This method deletes a product(s) from the Solr index identified by a given name
- *
+ *
* @param productName
* @throws IOException
* @throws SolrServerException
@@ -530,16 +518,16 @@ public class SolrIndexer {
private void deleteProductFromIndex(String productId) throws IOException, SolrServerException {
server.deleteById(productId);
}
-
+
/**
* Quick helper method to do substitution on the keys specified in the config
- *
+ *
* @param metadata
* to substitute on
* @throws java.text.ParseException
*/
private void performSubstitution(Metadata metadata)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Do metadata replacement
for (String key : config.getReplacementKeys()) {
List<String> values = metadata.getAllMetadata(key);
@@ -559,12 +547,10 @@ public class SolrIndexer {
List<String> values = metadata.getAllMetadata(keyString);
if (values != null) {
List<String> newValues = new ArrayList<String>();
-
- /* SimpleDateFormat format = new SimpleDateFormat(config
- .getFormatProperties().getProperty(keyString).trim());*/
+ SimpleDateFormat format = new SimpleDateFormat(config
+ .getFormatProperties().getProperty(keyString).trim());
for (String value : values) {
- String d = value.trim();
- newValues.add(d);
+ newValues.add(formatDate(format, value));
}
metadata.removeMetadata(keyString);
metadata.addMetadata(keyString, newValues);
@@ -574,17 +560,17 @@ public class SolrIndexer {
}
private String formatDate(SimpleDateFormat format, String value)
- throws java.text.ParseException {
+ throws java.text.ParseException {
// Ignore formating if its an ignore value
if (config.getIgnoreValues().contains(value.trim())) {
- return value;
+ return value;
}
return solrFormat.format(format.parse(value));
}
/**
* This method builds the command-line options.
- *
+ *
* @return Returns the supported Options.
*/
@SuppressWarnings("static-access")
@@ -593,31 +579,31 @@ public class SolrIndexer {
options.addOption(new Option("h", "help", false, "Print this message"));
options.addOption(new Option("o", "optimize", false,
- "Optimize the Solr index"));
+ "Optimize the Solr index"));
options.addOption(new Option("d", "delete", false,
- "Delete item before indexing"));
+ "Delete item before indexing"));
options.addOption(OptionBuilder.withArgName("Solr URL").hasArg()
- .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
- .create("su"));
+ .withDescription("URL to the Solr instance").withLongOpt("solrUrl")
+ .create("su"));
options.addOption(OptionBuilder.withArgName("Filemgr URL").hasArg()
- .withDescription("URL to the File Manager").withLongOpt("fmUrl")
- .create("fmu"));
+ .withDescription("URL to the File Manager").withLongOpt("fmUrl")
+ .create("fmu"));
OptionGroup group = new OptionGroup();
Option all = new Option("a", "all", false,
- "Index all products from the File Manager");
+ "Index all products from the File Manager");
Option product = OptionBuilder.withArgName("productId").hasArg()
- .withDescription("Index the product from the File Manager")
- .withLongOpt("product").create("p");
+ .withDescription("Index the product from the File Manager")
+ .withLongOpt("product").create("p");
Option met = OptionBuilder.withArgName("file").hasArg().withDescription(
- "Index the product from a metadata file").withLongOpt("metFile")
- .create("mf");
+ "Index the product from a metadata file").withLongOpt("metFile")
+ .create("mf");
Option read = new Option("r", "read", false,
- "Index all products based on a list of product identifiers passed in");
+ "Index all products based on a list of product identifiers passed in");
Option types = new Option("t", "types", false,
- "Index all product types from the File Manager");
+ "Index all product types from the File Manager");
Option deleteAll = new Option("da", "deleteAll", false,
- "Delete all products/types from the Solr index");
+ "Delete all products/types from the Solr index");
group.addOption(all);
group.addOption(product);
@@ -632,7 +618,7 @@ public class SolrIndexer {
/**
* The main method. Execution without argument displays help message.
- *
+ *
* @param args
* Command-line arguments.
*/
@@ -651,8 +637,8 @@ public class SolrIndexer {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("java " + SolrIndexer.class.getName(), options);
} else if (line.hasOption("all") || line.hasOption("product")
- || line.hasOption("metFile") || line.hasOption("read")
- || line.hasOption("types") || line.hasOption("deleteAll")) {
+ || line.hasOption("metFile") || line.hasOption("read")
+ || line.hasOption("types") || line.hasOption("deleteAll")) {
SolrIndexer indexer;
String solrUrl = null;
String fmUrl = null;
@@ -670,7 +656,7 @@ public class SolrIndexer {
indexer.indexProduct(line.getOptionValue("product"));
} else if (line.hasOption("metFile")) {
indexer.indexMetFile(new File(line.getOptionValue("metFile")), line
- .hasOption("delete"));
+ .hasOption("delete"));
} else if (line.hasOption("read")) {
for (String productId : readProductIdsFromStdin()) {
indexer.indexProduct(productId);
@@ -689,14 +675,14 @@ public class SolrIndexer {
} catch (Exception e) {
LOG.severe("An error occurred indexing: " + e.getMessage());
LOG
- .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
+ .severe("If the above message is related to accessing the Solr instance, see the Application Server's log for additional information.");
}
}
}
/**
* This method reads product identifiers from the standard input.
- *
+ *
* @return Returns a List of product identifiers.
*/
private static List<String> readProductIdsFromStdin() {
@@ -712,12 +698,12 @@ public class SolrIndexer {
}
} catch (IOException e) {
LOG.severe("Error reading product id: line: [" + line + "]: Message: "
- + e.getMessage());
+ + e.getMessage());
} finally {
- try {
- br.close();
- } catch (Exception ignore) {
- }
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
}
return productIds;
}
@@ -749,32 +735,32 @@ public class SolrIndexer {
String key = (String) objKey;
if (key.startsWith(PREFIX_CONFIG)) {
properties.put(key.substring(PREFIX_CONFIG.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_MET)) {
mapProperties.put(key.substring(PREFIX_MET.length()), props
- .getProperty(key));
+ .getProperty(key));
} else if (key.startsWith(PREFIX_FORMAT)) {
formatProperties.put(key.substring(PREFIX_FORMAT.length()), props
- .getProperty(key));
+ .getProperty(key));
}
}
if (properties.getProperty(IGNORE_TYPES) != null) {
String[] values = properties.getProperty(IGNORE_TYPES).trim()
- .split(",");
- Collections.addAll(ignoreTypes, values);
+ .split(",");
+ Collections.addAll(ignoreTypes, values);
}
if (properties.getProperty(IGNORE_VALUES) != null) {
String[] values = properties.getProperty(IGNORE_VALUES).trim().split(
- ",");
- Collections.addAll(ignoreValues, values);
+ ",");
+ Collections.addAll(ignoreValues, values);
}
if (properties.getProperty(REPLACEMENT_KEYS) != null) {
String[] values = properties.getProperty(REPLACEMENT_KEYS).trim()
- .split(",");
- Collections.addAll(replacementKeys, values);
+ .split(",");
+ Collections.addAll(replacementKeys, values);
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/9b0b61ec/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
index c3bf291..93f3afe 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
@@ -23,8 +23,21 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.*;
-import org.apache.lucene.search.*;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogDocMergePolicy;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
@@ -308,6 +321,42 @@ public class LuceneWorkflowInstanceRepository extends
return wInsts;
}
+
+ @Override
+ public synchronized boolean clearWorkflowInstances() throws InstanceRepositoryException {
+ IndexWriter writer = null;
+ try {
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+
+ writer = new IndexWriter(indexDir, config);
+ LOG.log(Level.FINE,
+ "LuceneWorkflowEngine: remove all workflow instances");
+ writer.deleteDocuments(new Term("myfield", "myvalue"));
+ } catch (IOException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ LOG
+ .log(Level.WARNING,
+ "Exception removing workflow instances from index: Message: "
+ + e.getMessage());
+ throw new InstanceRepositoryException(e.getMessage());
+ } finally {
+ if (writer != null){
+ try{
+ writer.close();
+ }
+ catch(Exception ignore){}
+
+ writer = null;
+ }
+
+ }
+
+ return true;
+ }
/*
* (non-Javadoc)
[41/50] [abbrv] oodt git commit: Fix dependencies for XML-RPC in
Resource manager; fix Httpclient.
Posted by ma...@apache.org.
Fix dependencies for XML-RPC in Resource manager; fix Httpclient.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ce89ad12
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ce89ad12
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ce89ad12
Branch: refs/heads/master
Commit: ce89ad12bc7e77c631abcec83e561c9c9782e675
Parents: 4382d9d
Author: Chris Mattmann <ma...@apache.org>
Authored: Sun Sep 3 16:32:50 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sun Sep 3 16:32:50 2017 -0700
----------------------------------------------------------------------
core/pom.xml | 7 ++++++-
resource/pom.xml | 8 ++++----
2 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ce89ad12/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index ba8e69b..3489a56 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -118,6 +118,11 @@ the License.
<version>1.4</version>
</dependency>
<dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<exclusions>
@@ -126,7 +131,7 @@ the License.
<groupId>commons-logging</groupId>
</exclusion>
</exclusions>
- <version>4.5.2</version>
+ <version>4.4</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/ce89ad12/resource/pom.xml
----------------------------------------------------------------------
diff --git a/resource/pom.xml b/resource/pom.xml
index 98ee673..f0edcbc 100644
--- a/resource/pom.xml
+++ b/resource/pom.xml
@@ -73,6 +73,10 @@ the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
@@ -94,10 +98,6 @@ the License.
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
- <dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
[14/50] [abbrv] oodt git commit: Fix for OODT-932: OODT-Core forces
javadoc to run and fails on linting - it shouldn't do that.
Posted by ma...@apache.org.
Fix for OODT-932: OODT-Core forces javadoc to run and fails on linting - it shouldn't do that.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/fcb214f2
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/fcb214f2
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/fcb214f2
Branch: refs/heads/master
Commit: fcb214f2710029582c125b73893d4dd4aa0b7bbe
Parents: 30b5a01
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 15:46:48 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:12:10 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 +++
core/pom.xml | 9 ++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/fcb214f2/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7d0b1b2..22147f5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,9 @@ Apache OODT Change Log
Release 1.1 - Current Development
+* OODT-932 OODT-Core forces javadoc to run and fails on linting - it shouldn't
+ do that (mattmann)
+
* OODT-929 EnvUtilities should only load environment variables one time and
statically cache them (mattmann)
http://git-wip-us.apache.org/repos/asf/oodt/blob/fcb214f2/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 6fafae6..ee8cb7e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -934,9 +934,12 @@ mm
<profiles>
<profile>
<id>jdk1.8</id>
- <activation>
- <jdk>1.8</jdk>
- </activation>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <properties>
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </properties>
<build>
<plugins>
<plugin>
[35/50] [abbrv] oodt git commit: Merge master into development
AvroRPC branch.
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/curator/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
----------------------------------------------------------------------
diff --cc curator/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
index 0000000,5d77083..d0d399f
mode 000000,100755..100755
--- a/curator/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
+++ b/curator/sso/src/main/java/org/apache/oodt/security/sso/opensso/SSOProxy.java
@@@ -1,0 -1,308 +1,354 @@@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.oodt.security.sso.opensso;
+
+
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpException;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.PostMethod;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
++
++import org.apache.http.HttpException;
++import org.apache.http.HttpResponse;
++import org.apache.http.HttpStatus;
++import org.apache.http.NameValuePair;
++import org.apache.http.client.HttpClient;
++import org.apache.http.client.ResponseHandler;
++import org.apache.http.client.entity.UrlEncodedFormEntity;
++import org.apache.http.client.methods.HttpPost;
++import org.apache.http.impl.client.BasicResponseHandler;
++import org.apache.http.impl.client.DefaultHttpClient;
++import org.apache.http.message.BasicNameValuePair;
++
++import java.io.*;
++import java.util.ArrayList;
++import java.util.Arrays;
++import java.util.List;
+ import java.util.logging.Level;
+ import java.util.logging.Logger;
+
+ /**
- *
++ *
+ * A client class to the services provided by the <a
+ * href="https://opensso.dev.java.net/">OpenSSO</a> project. The descriptions of
+ * these services are <a
+ * href="http://developers.sun.com/identity/reference/techart/id-svcs.html"
+ * >here</a>.
- *
++ *
+ * @author mattmann
+ * @version $Revision$
- *
++ *
+ */
+ public class SSOProxy implements SSOMetKeys {
+
+ private static final Logger LOG = Logger.getLogger(SSOProxy.class.getName());
+ private static final String AUTH_ENDPOINT;
+ private static final String AUTH_ENDPOINT_KEY = "AUTH_ENDPOINT";
+ private static final String IDENT_READ_ENDPOINT;
+ private static final String IDENT_READ_ENDPOINT_KEY = "IDENT_READ_ENDPOINT";
+ private static final String IDENT_ATTR_ENDPOINT;
+ private static final String IDENT_ATTR_ENDPOINT_KEY = "IDENT_ATTR_ENDPOINT";
+ private static final String LOG_ENDPOINT;
+ private static final String LOG_ENDPOINT_KEY = "LOG_ENDPOINT";
-
++
+ static {
- if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
- AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
- } else {
- AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
- }
- if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
- IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
- } else {
- IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
- }
- if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
- IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
- } else {
- IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
- }
- if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
- LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
- } else {
- LOG_ENDPOINT = LOGOUT_ENDPOINT;
- }
-
- LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
- LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
- LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
- LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
++ if (System.getProperty(AUTH_ENDPOINT_KEY) != null) {
++ AUTH_ENDPOINT = System.getProperty(AUTH_ENDPOINT_KEY);
++ } else {
++ AUTH_ENDPOINT = AUTHENTICATE_ENDPOINT;
++ }
++ if (System.getProperty(IDENT_READ_ENDPOINT_KEY) != null) {
++ IDENT_READ_ENDPOINT = System.getProperty(IDENT_READ_ENDPOINT_KEY);
++ } else {
++ IDENT_READ_ENDPOINT = IDENTITY_READ_ENDPOINT;
++ }
++ if (System.getProperty(IDENT_ATTR_ENDPOINT_KEY) != null) {
++ IDENT_ATTR_ENDPOINT = System.getProperty(IDENT_ATTR_ENDPOINT_KEY);
++ } else {
++ IDENT_ATTR_ENDPOINT = IDENTITY_ATTRIBUTES_ENDPOINT;
++ }
++ if (System.getProperty(LOG_ENDPOINT_KEY) != null) {
++ LOG_ENDPOINT = System.getProperty(LOG_ENDPOINT_KEY);
++ } else {
++ LOG_ENDPOINT = LOGOUT_ENDPOINT;
++ }
++
++ LOG.log(Level.INFO, AUTH_ENDPOINT_KEY + " set to " + AUTH_ENDPOINT);
++ LOG.log(Level.INFO, IDENT_READ_ENDPOINT_KEY + " set to " + IDENT_READ_ENDPOINT);
++ LOG.log(Level.INFO, IDENT_ATTR_ENDPOINT_KEY + " set to " + IDENT_ATTR_ENDPOINT);
++ LOG.log(Level.INFO, LOG_ENDPOINT_KEY + " set to " + LOG_ENDPOINT);
+ }
+
+ public String authenticate(String username, String password) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(AUTH_ENDPOINT);
++ HttpClient httpClient = new DefaultHttpClient();
++ HttpPost post = new HttpPost(AUTH_ENDPOINT);
++ //PostMethod post = new PostMethod(AUTH_ENDPOINT);
+ String response;
+ String ssoToken = null;
+
- NameValuePair[] data = { new NameValuePair("username", username),
- new NameValuePair("password", password),
- new NameValuePair("uri", "realm/lmmp") };
++ NameValuePair[] data = { new BasicNameValuePair("username", username),
++ new BasicNameValuePair("password", password),
++ new BasicNameValuePair("uri", "realm/lmmp") };
+
- post.setRequestBody(data);
++ UrlEncodedFormEntity entity = null;
++ try {
++ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
++ } catch (UnsupportedEncodingException e) {
++ e.printStackTrace();
++ }
++
++ post.setEntity(entity);
+
+ try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
++ HttpResponse response1 = httpClient.execute(post);
++ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
++ throw new HttpException(response1.getStatusLine().toString());
+ }
- response = post.getResponseBodyAsString().trim();
++ ResponseHandler<String> handler = new BasicResponseHandler();
++
++ response = handler.handleResponse(response1);
+ ssoToken = response.substring(9);
+ } catch (Exception e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ } finally {
+ post.releaseConnection();
+ }
+
+ return ssoToken;
+ }
+
+ public IdentityDetails readIdentity(String username, String token)
- throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_READ_ENDPOINT);
++ throws IOException, SingleSignOnException {
++ HttpClient httpClient = new DefaultHttpClient();
++ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
+ LOG.log(Level.INFO, "Obtaining identity: username: [" + username
- + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
- + "]");
- NameValuePair[] data = { new NameValuePair("name", username),
- new NameValuePair("admin", token) };
++ + "]: token: [" + token + "]: REST url: [" + IDENT_READ_ENDPOINT
++ + "]");
++ NameValuePair[] data = { new BasicNameValuePair("name", username),
++ new BasicNameValuePair("admin", token) };
+
- post.setRequestBody(data);
++ UrlEncodedFormEntity entity = null;
++ try {
++ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
++ } catch (UnsupportedEncodingException e) {
++ e.printStackTrace();
++ }
++
++ post.setEntity(entity);
+
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
++ HttpResponse response1 = httpClient.execute(post);
++ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
++ throw new SingleSignOnException(response1.getStatusLine().toString());
+ }
+
- return parseIdentityDetails(post.getResponseBodyAsString().trim());
++ ResponseHandler<String> handler = new BasicResponseHandler();
++
++
++ return parseIdentityDetails(handler.handleResponse(response1).trim());
+
+ }
+
+ public UserDetails getUserAttributes(String token) throws IOException, SingleSignOnException {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(IDENT_ATTR_ENDPOINT);
++ HttpClient httpClient = new DefaultHttpClient();
++ HttpPost post = new HttpPost(IDENT_READ_ENDPOINT);
+ LOG.log(Level.INFO, "Obtaining user attributes: token: [" + token
- + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
++ + "]: REST url: [" + IDENT_ATTR_ENDPOINT + "]");
++ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
++
++ UrlEncodedFormEntity entity = null;
++ try {
++ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
++ } catch (UnsupportedEncodingException e) {
++ e.printStackTrace();
++ }
+
- post.setRequestBody(data);
++ post.setEntity(entity);
+
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new SingleSignOnException(post.getStatusLine().toString());
++ HttpResponse response1 = httpClient.execute(post);
++ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
++ throw new SingleSignOnException(response1.getStatusLine().toString());
+ }
+
- return parseUserDetails(post.getResponseBodyAsString().trim());
++ ResponseHandler<String> handler = new BasicResponseHandler();
++
++
++ return parseUserDetails(handler.handleResponse(response1).trim());
+
+ }
+
+ public void logout(String token) {
- HttpClient httpClient = new HttpClient();
- PostMethod post = new PostMethod(LOG_ENDPOINT);
++ HttpClient httpClient = new DefaultHttpClient();
++ HttpPost post = new HttpPost(LOG_ENDPOINT);
+ LOG.log(Level.INFO, "Logging out: token: [" + token + "]: REST url: ["
- + LOG_ENDPOINT + "]");
- NameValuePair[] data = { new NameValuePair("subjectid", token) };
- post.setRequestBody(data);
++ + LOG_ENDPOINT + "]");
++ NameValuePair[] data = { new BasicNameValuePair("subjectid", token) };
++
++
++ UrlEncodedFormEntity entity = null;
++ try {
++ entity = new UrlEncodedFormEntity(Arrays.asList(data), "UTF-8");
++ } catch (UnsupportedEncodingException e) {
++ e.printStackTrace();
++ }
++
++ post.setEntity(entity);
+
+ try {
- httpClient.executeMethod(post);
- if (post.getStatusCode() != HttpStatus.SC_OK) {
- throw new HttpException(post.getStatusLine().toString());
++ HttpResponse response1 = httpClient.execute(post);
++ if (response1.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
++ throw new HttpException(response1.getStatusLine().toString());
+ }
+ } catch (HttpException e) {
+ // TODO Auto-generated catch block
+ LOG.log(Level.SEVERE, e.getMessage());
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ LOG.log(Level.SEVERE, e.getMessage());
+ } finally {
+ post.releaseConnection();
+ }
+ }
+
+ private IdentityDetails parseIdentityDetails(String serviceResponse) {
+ ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
++ .getBytes());
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ IdentityDetails details = new IdentityDetails();
+ String line = null, lastAttrKeyRead = null;
+
+ try {
+ while ((line = br.readLine()) != null) {
+ if (line.equals(IDENTITY_DETAILS_ATTR_SKIP_LINE)) {
+ continue;
+ }
+ String key, val;
+ if (line.startsWith(IDENTITY_DETAILS_REALM)) {
+ // can't parse it the same way
+ key = line.substring(0, IDENTITY_DETAILS_REALM.length());
+ val = line.substring(IDENTITY_DETAILS_REALM.length() + 1);
+ } else {
+ String[] lineToks = line.split("=");
+ key = lineToks[0];
+ val = lineToks[1];
+ }
+
+ if (key.equals(IDENTITY_DETAILS_NAME)) {
+ details.setName(val);
+ } else if (key.equals(IDENTITY_DETAILS_TYPE)) {
+ details.setType(val);
+ } else if (key.equals(IDENTITY_DETAILS_REALM)) {
+ details.setRealm(val);
+ } else if (key.equals(IDENTITY_DETAILS_GROUP)) {
+ details.getGroups().add(val);
+ } else if (key.equals(IDENTITY_DETAILS_ATTR_NAME)) {
+ lastAttrKeyRead = val;
+ } else if (key.equals(IDENTITY_DETAILS_ATTR_VALUE)) {
+ details.getAttributes().addMetadata(lastAttrKeyRead, val);
+ }
+ }
+ } catch (IOException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
++ + "]: Message: " + e.getMessage());
+ } finally {
+ try {
+ is.close();
+ } catch (Exception ignore) {
+ }
+
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
+
+ }
+
+ return details;
+ }
+
+ private UserDetails parseUserDetails(String serviceResponse) {
+ ByteArrayInputStream is = new ByteArrayInputStream(serviceResponse
- .getBytes());
++ .getBytes());
+ BufferedReader br = new BufferedReader(new InputStreamReader(is));
+ UserDetails details = new UserDetails();
+ String line = null, lastAttrKeyRead = null;
+
+ try {
+ while ((line = br.readLine()) != null) {
+ String key, val;
+ if (line.startsWith(USER_DETAILS_ROLE)) {
+ // can't parse by splitting, parse by using substring
+ key = line.substring(0, USER_DETAILS_ROLE.length());
+ val = line.substring(USER_DETAILS_ROLE.length() + 1);
+ } else {
+ String[] lineToks = line.split("=");
+ key = lineToks[0];
+ val = lineToks[1];
+ }
+
+ if (key.equals(USER_DETAILS_TOKEN)) {
+ details.setToken(val);
+ } else if (key.equals(USER_DETAILS_ROLE)) {
+ details.getRoles().add(val);
+ } else if (key.equals(USER_DETAILS_ATTR_NAME)) {
+ lastAttrKeyRead = val;
+ } else if (key.equals(USER_DETAILS_ATTR_VALUE)) {
+ details.getAttributes().addMetadata(lastAttrKeyRead, val);
+ }
+ }
+ } catch (IOException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ LOG.log(Level.WARNING, "Error reading service response line: [" + line
- + "]: Message: " + e.getMessage());
++ + "]: Message: " + e.getMessage());
+ } finally {
+ try {
+ is.close();
+ } catch (Exception ignore) {
+ }
+
+ try {
+ br.close();
+ } catch (Exception ignore) {
+ }
+
+ }
+
+ return details;
+ }
+
+ public static void main(String[] args) throws IOException, SingleSignOnException {
+ String usage = "SSOProxy <cmd> [args]\n\n" + "Where cmd is one of:\n"
- + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
- + "attributes <token>\nlogout <token>\n";
++ + "authenticate <user> <pass>\n" + "identity <user> <token>\n"
++ + "attributes <token>\nlogout <token>\n";
+
+ if (args.length < 2 || args.length > 3) {
+ System.err.println(usage);
+ System.exit(1);
+ }
+
+ String cmd = args[0];
+ SSOProxy sso = new SSOProxy();
+ if (cmd.equals(AUTH_COMMAND)) {
+ System.out.println(sso.authenticate(args[1], args[2]));
+ } else if (cmd.equals(IDENTITY_COMMAND)) {
+ System.out.println(sso.readIdentity(args[1], args[2]));
+ } else if (cmd.equals(ATTRIBUTES_COMMAND)) {
+ System.out.println(sso.getUserAttributes(args[1]));
+ } else if (cmd.equals(LOGOUT_COMMAND)) {
+ sso.logout(args[1]);
+ }
+
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/src/main/bin/filemgr
----------------------------------------------------------------------
diff --cc filemgr/src/main/bin/filemgr
index ece2532,02f5b90..9e5ffde
--- a/filemgr/src/main/bin/filemgr
+++ b/filemgr/src/main/bin/filemgr
@@@ -58,10 -74,12 +74,12 @@@ case "$1" i
echo -n "Starting cas file manager: "
$JAVA_HOME/bin/java \
-cp ${LIB_DEPS} \
- -Djava.util.logging.config.file=${CAS_FILEMGR_HOME}/etc/logging.properties \
+ ${DISTRIBUTED_CONF_PROPERTIES} \
+ -Dlog4j.configuration=log4j.xml \
+ -Djava.util.logging.config.file=${FILEMGR_HOME}/etc/logging.properties \
-Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \
- org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \
- --portNum $SERVER_PORT &
+ org.apache.oodt.cas.filemgr.system.FileManagerServerMain \
+ --portNum $SERVER_PORT &
echo $! > ${RUN_HOME}/cas.filemgr.pid
echo "OK"
sleep 5
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/MetadataDumper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/src/test/java/org/apache/oodt/cas/filemgr/catalog/TestLuceneCatalog.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/filemgr/src/test/resources/filemgr.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java
----------------------------------------------------------------------
diff --cc pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java
index 9f750f5,4388855..f36431e
--- a/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java
+++ b/pcs/core/src/main/java/org/apache/oodt/pcs/util/FileManagerUtils.java
@@@ -64,10 -65,10 +66,10 @@@ public class FileManagerUtils implement
public FileManagerUtils(URL fileMgrUrl) {
try {
- fmgrClient = new XmlRpcFileManagerClient(fileMgrUrl);
+ fmgrClient = RpcCommunicationFactory.createClient(fileMgrUrl);
} catch (ConnectionException e) {
- LOG.log(Level.SEVERE, "Unable to connect to file manager: ["
- + fileMgrUrl.toString() + "]");
+ LOG.log(Level.SEVERE,
+ "Unable to connect to file manager: [" + fileMgrUrl.toString() + "]");
fmgrClient = null;
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/pge/src/test/java/org/apache/oodt/cas/pge/TestPGETaskInstance.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/resource/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java
----------------------------------------------------------------------
diff --cc webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java
index 6deafc3,41c757a..eefd0c6
--- a/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java
+++ b/webapp/components/src/main/java/org/apache/oodt/cas/webcomponents/filemgr/FileManagerConn.java
@@@ -44,10 -44,10 +45,10 @@@ import java.util.logging.Logger
*/
public class FileManagerConn {
- private XmlRpcFileManagerClient fm;
+ private FileManagerClient fm;
- private static final Logger LOG = Logger.getLogger(FileManagerConn.class
- .getName());
+ private static final Logger LOG = Logger
+ .getLogger(FileManagerConn.class.getName());
public FileManagerConn(String fmUrlStr) {
this.initFm(fmUrlStr);
@@@ -134,10 -146,10 +147,10 @@@
public void initFm(String urlStr) {
try {
- this.fm = new XmlRpcFileManagerClient(new URL(urlStr));
+ this.fm = RpcCommunicationFactory.createClient(new URL(urlStr));
} catch (Exception e) {
- LOG.log(Level.WARNING, "Unable to connect to the file manager at: ["
- + urlStr + "]");
+ LOG.log(Level.WARNING,
+ "Unable to connect to the file manager at: [" + urlStr + "]");
this.fm = null;
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/data/DataDeliveryServlet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/webapp/fmprod/src/main/java/org/apache/oodt/cas/product/rss/RSSProductServlet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/workflow/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/423444cb/workflow/src/main/java/org/apache/oodt/cas/workflow/system/XmlRpcWorkflowManagerClient.java
----------------------------------------------------------------------
[32/50] [abbrv] oodt git commit: Fix numHits error in Lucene Catalog
Posted by ma...@apache.org.
Fix numHits error in Lucene Catalog
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/e43c6c00
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/e43c6c00
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/e43c6c00
Branch: refs/heads/master
Commit: e43c6c0076e4c75598389db881e1e0a1716fc91d
Parents: 979f4a7
Author: Tom Barber <to...@analytical-labs.com>
Authored: Sat Jul 29 19:24:21 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Sat Jul 29 19:25:18 2017 +0100
----------------------------------------------------------------------
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 120 ++++++++++---------
1 file changed, 64 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/e43c6c00/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index 4c9ed96..a8c0253 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -526,25 +526,26 @@ public class LuceneCatalog implements Catalog {
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
- TopDocs topDocs = searcher.search(query, check.totalHits, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
- ScoreDoc[] hits = topDocs.scoreDocs;
+ ScoreDoc[] hits = topDocs.scoreDocs;
- // should be > 0 hits
- if (hits.length > 0) {
- products = new Vector<Product>(hits.length);
- for (ScoreDoc hit : hits) {
- Document productDoc = searcher.doc(hit.doc);
- CompleteProduct prod = toCompleteProduct(productDoc,
+ // should be > 0 hits
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
+ CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
- products.add(prod.getProduct());
- }
- } else {
- LOG.log(Level.FINEST,
+ products.add(prod.getProduct());
+ }
+ } else {
+ LOG.log(Level.FINEST,
"Request for products returned no results");
- return null;
+ return null;
+ }
}
-
} catch (IOException e) {
LOG.log(Level.WARNING,
"IOException when opening index directory: ["
@@ -592,23 +593,25 @@ public class LuceneCatalog implements Catalog {
SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
- TopDocs topDocs = searcher.search(query, check.totalHits, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
- ScoreDoc[] hits = topDocs.scoreDocs;
+ ScoreDoc[] hits = topDocs.scoreDocs;
- // should be > 0 hits
- if (hits.length > 0) {
- products = new Vector<Product>(hits.length);
- for (ScoreDoc hit : hits) {
- Document productDoc = searcher.doc(hit.doc);
- CompleteProduct prod = toCompleteProduct(productDoc,
+ // should be > 0 hits
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
+ CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
- products.add(prod.getProduct());
- }
- } else {
- LOG.log(Level.FINEST, "Request for products by type: ["
+ products.add(prod.getProduct());
+ }
+ } else {
+ LOG.log(Level.FINEST, "Request for products by type: ["
+ type.getProductTypeId() + "] returned no results");
- return null;
+ return null;
+ }
}
} catch (IOException e) {
@@ -796,7 +799,9 @@ public class LuceneCatalog implements Catalog {
for (int pageNum = 1; pageNum < numPages + 1; pageNum++) {
List<Product> pageProducts = paginateQuery(query, type, pageNum, null);
+ if(pageProducts!=null) {
products.addAll(pageProducts);
+ }
}
if(n<=products.size()) {
@@ -1372,59 +1377,61 @@ public class LuceneCatalog implements Catalog {
TermQuery prodTypeTermQuery = new TermQuery(new Term(
"product_type_id", type.getProductTypeId()));
booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST);
-
+
//convert filemgr query into a lucene query
for (QueryCriteria queryCriteria : query.getCriteria()) {
booleanQuery.add(this.getQuery(queryCriteria), BooleanClause.Occur.MUST);
}
-
+
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(booleanQuery.build(),1, sort);
- TopDocs topDocs = searcher.search(booleanQuery.build(),check.totalHits, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(booleanQuery.build(), check.totalHits, sort);
- // Calculate page size and set it while we have the results
- if (page != null) {
- page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
- }
+ // Calculate page size and set it while we have the results
+ if (page != null) {
+ page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
+ }
- ScoreDoc[] hits = topDocs.scoreDocs;
+ ScoreDoc[] hits = topDocs.scoreDocs;
- if (hits.length > 0) {
+ if (hits.length > 0) {
- int startNum = (pageNum - 1) * pageSize;
- if (doSkip) {
- if (startNum > hits.length) {
- startNum = 0;
- }
+ int startNum = (pageNum - 1) * pageSize;
+ if (doSkip) {
+ if (startNum > hits.length) {
+ startNum = 0;
+ }
- products = new Vector<Product>(pageSize);
+ products = new Vector<Product>(pageSize);
- for (int i = startNum; i < Math.min(hits.length,
+ for (int i = startNum; i < Math.min(hits.length,
(startNum + pageSize)); i++) {
- Document productDoc = searcher.doc(hits[i].doc);
+ Document productDoc = searcher.doc(hits[i].doc);
- CompleteProduct prod = toCompleteProduct(productDoc,
+ CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
- products.add(prod.getProduct());
- }
- } else {
- products = new Vector<Product>(hits.length);
- for (int i = 0; i < hits.length; i++) {
- Document productDoc = searcher.doc(hits[i].doc);
+ products.add(prod.getProduct());
+ }
+ } else {
+ products = new Vector<Product>(hits.length);
+ for (int i = 0; i < hits.length; i++) {
+ Document productDoc = searcher.doc(hits[i].doc);
- CompleteProduct prod = toCompleteProduct(productDoc,
+ CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
- products.add(prod.getProduct());
+ products.add(prod.getProduct());
+ }
}
- }
- } else {
- LOG.log(Level.WARNING, "Query: [" + query
+ } else {
+ LOG.log(Level.WARNING, "Query: [" + query
+ "] for Product Type: [" + type.getProductTypeId()
+ "] returned no results");
+ }
}
} catch (Exception e) {
@@ -1444,6 +1451,7 @@ public class LuceneCatalog implements Catalog {
}
return products;
+
}
private org.apache.lucene.search.Query getQuery(QueryCriteria queryCriteria) throws CatalogException {
[30/50] [abbrv] oodt git commit: fix lucene querying
Posted by ma...@apache.org.
fix lucene querying
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/ceadfcb2
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/ceadfcb2
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/ceadfcb2
Branch: refs/heads/master
Commit: ceadfcb2b015956d915d6e9c58bd1b8ed7da4655
Parents: ab366e2
Author: Tom Barber <to...@analytical-labs.com>
Authored: Wed Jul 26 13:16:31 2017 +0100
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Wed Jul 26 13:16:31 2017 +0100
----------------------------------------------------------------------
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 29 +++++++++++++++-----
.../oodt/cas/filemgr/tools/QueryTool.java | 1 +
2 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/ceadfcb2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
index bfe2384..53efeb9 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
@@ -19,17 +19,23 @@ package org.apache.oodt.cas.filemgr.tools;
//Lucene imports
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.custom.CustomAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
//JDK imports
import java.io.Reader;
import java.util.Set;
+import org.apache.lucene.util.AttributeFactory;
/**
@@ -44,6 +50,7 @@ import java.util.Set;
*/
public class CASAnalyzer extends Analyzer {
private Set stopSet;
+ AttributeFactory factory = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
/**
* An array containing some common English words that are usually not useful
@@ -60,17 +67,19 @@ public class CASAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
TokenStream result = new WhitespaceTokenizer(/*reader*/);
- result = new StandardFilter(result);
+ /*result = new StandardFilter(result);
result = new StopFilter(result, STOP_WORDS);
-
- //TODO FIX
try {
- throw new Exception("needs fixing");
- } catch (Exception e) {
+ result.reset();
+ } catch (IOException e) {
e.printStackTrace();
}
- return null; //new TokenStreamComponents();
+ StandardTokenizer tokenizer = new StandardTokenizer(factory);
+
+ return new TokenStreamComponents(tokenizer, result);*/
+ return new TokenStreamComponents(new WhitespaceTokenizer());
+
}
public void tokenStreams(String fname, Reader reader){
@@ -78,7 +87,13 @@ public class CASAnalyzer extends Analyzer {
}
/** Builds an analyzer with the given stop words. */
public CASAnalyzer(CharArraySet stopWords) {
- stopSet = StopFilter.makeStopSet(stopWords.toArray(new String[stopWords.size()]));
+ Iterator iter = stopWords.iterator();
+ List<String> sw = new ArrayList<>();
+ while(iter.hasNext()) {
+ char[] stopWord = (char[]) iter.next();
+ sw.add(new String(stopWord));
+ }
+ stopSet = StopFilter.makeStopSet(sw);
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/ceadfcb2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index 4a9641f..7b37f90 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -77,6 +77,7 @@ public final class QueryTool {
QueryParser parser;
// note that "__FREE__" is a control work for free text searching
parser = new QueryParser(freeTextBlock, new CASAnalyzer());
+
Query luceneQ = null;
try {
luceneQ = (Query) parser.parse(query);
[38/50] [abbrv] oodt git commit: Fix pom error.
Posted by ma...@apache.org.
Fix pom error.
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/835f16d0
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/835f16d0
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/835f16d0
Branch: refs/heads/master
Commit: 835f16d08a8e758278efcc4b639387a3d1b0bf52
Parents: 758eece
Author: Chris Mattmann <ma...@apache.org>
Authored: Sat Sep 2 08:38:47 2017 -0700
Committer: Chris Mattmann <ma...@apache.org>
Committed: Sat Sep 2 08:38:47 2017 -0700
----------------------------------------------------------------------
core/pom.xml | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/835f16d0/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 96444d5..91f95ce 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -249,10 +249,11 @@ the License.
<version>8.7</version>
</dependency>
<dependency>
-<<<<<<< HEAD
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.12</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>3.3.0</version>
[03/50] [abbrv] oodt git commit: update some dependenceies
Posted by ma...@apache.org.
update some dependenceies
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/78ce45e2
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/78ce45e2
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/78ce45e2
Branch: refs/heads/master
Commit: 78ce45e2efad87835af9d29972e88cc32561c5d7
Parents: 29fc616
Author: Tom Barber <to...@analytical-labs.com>
Authored: Tue Dec 6 11:34:33 2016 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Tue Dec 6 11:34:33 2016 +0000
----------------------------------------------------------------------
catalog/pom.xml | 4 +-
core/pom.xml | 22 +-
crawler/pom.xml | 4 +-
filemgr/pom.xml | 4 +-
.../cas/filemgr/catalog/solr/SolrClient.java | 318 ++++++++-------
.../filemgr/system/CommonsXmlRpcTransport.java | 183 +++++++++
.../filemgr/system/XmlRpcFileManagerClient.java | 398 ++++++++++---------
.../oodt/cas/filemgr/tools/SolrIndexer.java | 236 ++++++-----
profile/pom.xml | 5 +-
protocol/http/pom.xml | 4 +-
resource/pom.xml | 4 +-
sso/pom.xml | 5 +-
.../oodt/security/sso/opensso/SSOProxy.java | 214 ++++++----
workflow/pom.xml | 4 +-
14 files changed, 842 insertions(+), 563 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index 77ffa77..07805a3 100644
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -61,8 +61,8 @@
</exclusions>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 3420597..f1dea9e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -111,9 +111,15 @@ the License.
<version>1.4</version>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>3.1</version>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>commons-logging</artifactId>
+ <groupId>commons-logging</groupId>
+ </exclusion>
+ </exclusions>
+ <version>4.5.2</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -267,11 +273,6 @@ the License.
<version>1.1</version>
</dependency>
<dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>4.5.2</version>
- </dependency>
- <dependency>
<groupId>org.apache.jena</groupId>
<artifactId>apache-jena-libs</artifactId>
<version>${jena.version}</version>
@@ -489,11 +490,6 @@ the License.
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j</artifactId>
- <version>1.7.12</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.12</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/crawler/pom.xml
----------------------------------------------------------------------
diff --git a/crawler/pom.xml b/crawler/pom.xml
index 5428b53..46378c8 100644
--- a/crawler/pom.xml
+++ b/crawler/pom.xml
@@ -56,8 +56,8 @@ the License.
<artifactId>commons-collections</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index cc9a6e5..745778f 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -91,8 +91,8 @@
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
index 1b17b62..475ff8b 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/solr/SolrClient.java
@@ -16,20 +16,35 @@
*/
package org.apache.oodt.cas.filemgr.catalog.solr;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpStatus;
-import org.apache.commons.httpclient.NameValuePair;
-import org.apache.commons.httpclient.methods.GetMethod;
-import org.apache.commons.httpclient.methods.PostMethod;
-import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.ResponseHandler;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicResponseHandler;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicNameValuePair;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
+import org.apache.solr.client.solrj.util.ClientUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.concurrent.ConcurrentHashMap;
import java.util.List;
import java.util.Map;
@@ -39,30 +54,30 @@ import java.util.logging.Logger;
/**
* Class containing client-side functionality for interacting with a Solr server.
* This class uses an {@link HttpClient} for all HTTP communication.
- *
+ *
* @author Luca Cinquini
*
*/
public class SolrClient {
-
+
// base URL of Solr server
private String solrUrl;
-
+
private final Logger LOG = Logger.getLogger(this.getClass().getName());
-
+
/**
* Constructor initializes the Solr URL
* @param url
*/
public SolrClient(final String url) {
-
+
solrUrl = url;
-
+
}
-
+
/**
* Method to send one or more documents to be indexed to Solr.
- *
+ *
* @param docs
* @param commit
* @param mimeType : the mime-type format of the documents
@@ -70,38 +85,38 @@ public class SolrClient {
* @throws CatalogException
*/
public String index(List<String> docs, boolean commit, String mimeType) throws CatalogException {
-
+
try {
-
+
final String url = this.buildUpdateUrl();
-
+
// build message
StringBuilder message = new StringBuilder("<add>");
for (String doc : docs) {
message.append(doc);
}
- message.append("</add>");
-
- // send POST request
- LOG.info("Posting message:"+message+" to URL:"+url);
- String response = doPost(url, message.toString(), mimeType);
- LOG.info(response);
-
- // commit changes ?
- if (commit) {
- this.commit();
- }
-
- LOG.info(response);
- return response;
-
+ message.append("</add>");
+
+ // send POST request
+ LOG.info("Posting message:"+message+" to URL:"+url);
+ String response = doPost(url, message.toString(), mimeType);
+ LOG.info(response);
+
+ // commit changes ?
+ if (commit) {
+ this.commit();
+ }
+
+ LOG.info(response);
+ return response;
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to send a message containing a 'delete' instruction to Solr.
* @param id
@@ -110,41 +125,41 @@ public class SolrClient {
* @throws CatalogException
*/
public String delete(String id, boolean commit) throws CatalogException {
-
+
try {
-
+
// build POST request
String url = this.buildUpdateUrl();
if (commit) {
- url += "?commit=true";
+ url += "?commit=true";
}
String message = "<delete><query>id:"+id+"</query></delete>";
-
- // send POST request
- LOG.info("Posting message:"+message+" to URL:"+url);
- return doPost(url, message, Parameters.MIME_TYPE_XML);
-
+ // send POST request
+ LOG.info("Posting message:"+message+" to URL:"+url);
+
+ return doPost(url, message, Parameters.MIME_TYPE_XML);
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to query the Solr index for a product with the specified id.
* @param id
* @return
*/
public String queryProductById(String id, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{Parameters.PRODUCT_ID+":"+id} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query the Solr index for a product with the specified name.
* @param name
@@ -152,13 +167,13 @@ public class SolrClient {
* @return
*/
public String queryProductByName(String name, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
- params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+name} );
+ params.put("q", new String[]{Parameters.PRODUCT_NAME+":"+ ClientUtils.escapeQueryChars(name)} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query Solr for the most recent 'n' products.
* @param n
@@ -166,15 +181,15 @@ public class SolrClient {
* @throws CatalogException
*/
public String queryProductsByDate(int n, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{ "*:*"} );
params.put("rows", new String[]{ ""+n} );
params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to query Solr for the most recent 'n' products of a specified type.
* @param n
@@ -182,51 +197,51 @@ public class SolrClient {
* @throws CatalogException
*/
public String queryProductsByDateAndType(int n, ProductType type, String mimeType) throws CatalogException {
-
+
ConcurrentHashMap<String, String[]> params = new ConcurrentHashMap<String, String[]>();
params.put("q", new String[]{ Parameters.PRODUCT_TYPE_NAME+type.getName() } );
params.put("rows", new String[]{ ""+n} );
params.put("sort", new String[]{ Parameters.PRODUCT_RECEIVED_TIME+" desc"} );
return query(params, mimeType);
-
+
}
-
+
/**
* Method to commit the current changes to the Solr index.
* @throws Exception
*/
public void commit() throws IOException, CatalogException {
-
+
String message = "<commit waitSearcher=\"true\"/>";
String url = this.buildUpdateUrl();
doPost(url, message, Parameters.MIME_TYPE_XML);
-
+
}
-
+
/**
* Method to send a generic query to the Solr server.
- *
+ *
* @param parameters
* @param mimeType : the desired mime type for the results (XML, JSON, ...)
* @return
*/
public String query(Map<String, String[]> parameters, String mimeType) throws CatalogException {
-
+
try {
-
+
// build HTTP request
String url = this.buildSelectUrl();
-
+
// execute request
- return this.doGet(url, parameters, mimeType);
-
+ return this.doGet(url, parameters, mimeType);
+
} catch(Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e.getMessage());
}
-
+
}
-
+
/**
* Method to execute a GET request to the given URL with the given parameters.
* @param url
@@ -234,28 +249,32 @@ public class SolrClient {
* @return
*/
private String doGet(String url, Map<String, String[]> parameters, String mimeType)
- throws IOException, CatalogException {
-
+ throws IOException, CatalogException {
+
// build HTTP/GET request
- GetMethod method = new GetMethod(url);
- List<NameValuePair> nvps = new ArrayList<NameValuePair>();
- for (Map.Entry<String, String[]> key : parameters.entrySet()) {
- for (String value : key.getValue()) {
- nvps.add(new NameValuePair(key.getKey(), value));
- }
- }
- // request results in JSON format
- if (mimeType.equals(Parameters.MIME_TYPE_JSON)) {
- nvps.add(new NameValuePair("wt", "json"));
+
+
+ List<BasicNameValuePair> nvps = new ArrayList<BasicNameValuePair>();
+ for (Map.Entry<String, String[]> key : parameters.entrySet()) {
+ for (String value : key.getValue()) {
+ nvps.add(new BasicNameValuePair(key.getKey(), value));
+ }
+ }
+ // request results in JSON format
+ if (mimeType.equals(Parameters.MIME_TYPE_JSON)) {
+ nvps.add(new BasicNameValuePair("wt", "json"));
+ }
+
+ String paramString = URLEncodedUtils.format(nvps, "utf-8");
+
+ HttpRequestBase method = new HttpGet(url+"?"+paramString);
+ LOG.info("GET url: "+url+" query string: "+method.getURI());
+
+ // send HTTP/GET request, return response
+ return doHttp(method);
+
}
- method.setQueryString( nvps.toArray( new NameValuePair[nvps.size()] ) );
- LOG.info("GET url: "+url+" query string: "+method.getQueryString());
-
- // send HTTP/GET request, return response
- return doHttp(method);
-
- }
-
+
/**
* Method to execute a POST request to the given URL.
* @param url
@@ -263,91 +282,96 @@ public class SolrClient {
* @return
*/
private String doPost(String url, String document, String mimeType) throws IOException, CatalogException {
-
+
// build HTTP/POST request
- PostMethod method = new PostMethod(url);
- StringRequestEntity requestEntity = new StringRequestEntity(document, mimeType, "UTF-8");
- method.setRequestEntity(requestEntity);
-
- // send HTTP/POST request, return response
- return doHttp(method);
-
+ HttpPost method = new HttpPost(url);
+ HttpEntity requestEntity = null;
+ try {
+ requestEntity = new StringEntity(document, mimeType, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+
+ method.setEntity(requestEntity);
+ // send HTTP/POST request, return response
+ return doHttp(method);
+
}
-
+
/**
* Common functionality for HTTP GET and POST requests.
* @param method
* @return
* @throws Exception
*/
- private String doHttp(HttpMethod method) throws IOException, CatalogException {
-
- StringBuilder response = new StringBuilder();
+ private String doHttp(HttpRequestBase method) throws IOException, CatalogException {
+
+ String response = null;
BufferedReader br = null;
try {
-
- // send request
- HttpClient httpClient = new HttpClient();
- // OODT-719 Prevent httpclient from spawning closewait tcp connections
- method.setRequestHeader("Connection", "close");
-
- int statusCode = httpClient.executeMethod(method);
-
- // read response
- if (statusCode != HttpStatus.SC_OK) {
-
- // still consume the response
- method.getResponseBodyAsString();
- throw new CatalogException("HTTP method failed: " + method.getStatusLine());
-
- } else {
-
- // read the response body.
- br = new BufferedReader(new InputStreamReader(method.getResponseBodyAsStream()));
- String readLine;
- while(((readLine = br.readLine()) != null)) {
- response.append(readLine);
- }
-
- }
-
- } finally {
- // must release the connection even if an exception occurred
- method.releaseConnection();
- if (br!=null) {
- try {
- br.close();
- } catch (Exception ignored) {
- }
+
+ // send request
+ HttpClient httpClient = new DefaultHttpClient();
+ // OODT-719 Prevent httpclient from spawning closewait tcp connections
+ method.setHeader("Connection", "close");
+
+ HttpResponse statusCode = httpClient.execute(method);
+
+ // read response
+ if (statusCode.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+
+ // still consume the response
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ handler.handleResponse(statusCode);
+ throw new CatalogException("HTTP method failed: " + statusCode.getStatusLine().toString());
+
+ } else {
+ ResponseHandler<String> handler = new BasicResponseHandler();
+
+ String resp = handler.handleResponse(statusCode);
+
+ response=resp;
+
+ }
+
+ } finally {
+ // must release the connection even if an exception occurred
+ method.releaseConnection();
+ if (br!=null) {
+ try {
+ br.close();
+ } catch (Exception ignored) {
+ }
+ }
}
- }
-
- return response.toString();
-
+
+ return response;
+
}
-
+
/**
* Builds the URL used to update the Solr index.
- *
+ *
* Example: http://localhost:8983/solr/update?
* @return
*/
private String buildUpdateUrl() {
-
+
return solrUrl + (solrUrl.endsWith("/") ? "" : "/") + "update";
-
+
}
-
+
/**
* Builds the URL used to query the Solr index.
- *
+ *
* Example: http://localhost:8983/solr/select?
* @return
*/
private String buildSelectUrl() {
-
+
return solrUrl + (solrUrl.endsWith("/") ? "" : "/") + "select";
-
+
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
new file mode 100644
index 0000000..baf9e73
--- /dev/null
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/CommonsXmlRpcTransport.java
@@ -0,0 +1,183 @@
+//
+// Source code recreated from a .class file by IntelliJ IDEA
+// (powered by Fernflower decompiler)
+//
+
+package org.apache.oodt.cas.filemgr.system;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpResponse;
+import org.apache.http.auth.*;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.entity.ByteArrayEntity;
+
+import org.apache.http.impl.auth.BasicSchemeFactory;
+import org.apache.http.impl.auth.DigestSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.apache.xmlrpc.XmlRpcClientException;
+import org.apache.xmlrpc.XmlRpcTransport;
+
+public class CommonsXmlRpcTransport implements XmlRpcTransport {
+ private URL url;
+ private HttpClient client;
+ private Header userAgentHeader;
+ private boolean http11;
+ private boolean gzip;
+ private boolean rgzip;
+ private Credentials creds;
+ protected HttpPost method;
+ private int timeout = 10;
+ private int connecttimeout = 10;
+ private String password;
+ private String user;
+ private String auth;
+
+ public CommonsXmlRpcTransport(URL url, HttpClient client) {
+ this.userAgentHeader = new BasicHeader("User-Agent", "Apache XML-RPC 2.0");
+ this.http11 = false;
+ this.gzip = false;
+ this.rgzip = false;
+ this.url = url;
+ if(client == null) {
+ RequestConfig config = RequestConfig.custom()
+ .setSocketTimeout(timeout * 1000)
+ .setConnectTimeout(connecttimeout * 1000)
+ .build();
+ CredentialsProvider credsProvider = new BasicCredentialsProvider();
+ if(!user.equals(null)) {
+ credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user));
+ }
+ else if(!user.equals(null)&& !password.equals(null)){
+ credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));
+ }
+ else if(!auth.equals(null)){
+
+ }
+
+ System.out.println("building empty registry");
+ Registry<AuthSchemeProvider> r = RegistryBuilder.<AuthSchemeProvider>create().build();
+ HttpClient newClient = HttpClients.custom().setDefaultAuthSchemeRegistry(r).setDefaultCredentialsProvider(credsProvider).setDefaultRequestConfig(config).build();
+ this.client = newClient;
+ } else {
+ this.client = client;
+ }
+
+ }
+
+ public CommonsXmlRpcTransport(URL url) {
+ this(url, (HttpClient)null);
+ }
+
+
+
+ public InputStream sendXmlRpc(byte[] request) throws IOException, XmlRpcClientException {
+ this.method = new HttpPost(this.url.toString());
+ //this.method.setHttp11(this.http11);
+ this.method.setHeader(new BasicHeader("Content-Type", "text/xml"));
+ if(this.rgzip) {
+ this.method.setHeader(new BasicHeader("Content-Encoding", "gzip"));
+ }
+
+ if(this.gzip) {
+ this.method.setHeader(new BasicHeader("Accept-Encoding", "gzip"));
+ }
+
+ this.method.setHeader(this.userAgentHeader);
+ if(this.rgzip) {
+ ByteArrayOutputStream hostURI = new ByteArrayOutputStream();
+ GZIPOutputStream hostConfig = new GZIPOutputStream(hostURI);
+ hostConfig.write(request);
+ hostConfig.finish();
+ hostConfig.close();
+ byte[] lgzipo = hostURI.toByteArray();
+
+ HttpEntity entity = new ByteArrayEntity(lgzipo);
+
+
+ this.method.setEntity(entity);
+
+ //this.method.setRequestContentLength(-1);
+ } else {
+ HttpEntity entity = new ByteArrayEntity(request);
+ this.method.setEntity(entity);
+// this.method.setRequestBody(new ByteArrayInputStream(request));
+ }
+
+ URI hostURI1 = null;
+ try {
+ hostURI1 = new URI(this.url.toString());
+ } catch (URISyntaxException e) {
+ e.printStackTrace();
+ }
+ HttpHost hostConfig1 = new HttpHost(hostURI1.toString());
+ HttpResponse response = this.client.execute(this.method);
+ boolean lgzipo1 = false;
+ Header[] lHeader = response.getHeaders("Content-Encoding");
+ if(lHeader != null && lHeader.length>0) {
+ String lValue = lHeader[0].getValue();
+ if(lValue != null) {
+ lgzipo1 = lValue.indexOf("gzip") >= 0;
+ }
+ }
+
+ return (InputStream)(lgzipo1?new GZIPInputStream(response.getEntity().getContent()):response.getEntity().getContent());
+ }
+
+ public void setHttp11(boolean http11) {
+ this.http11 = http11;
+ }
+
+ public void setGzip(boolean gzip) {
+ this.gzip = gzip;
+ }
+
+ public void setRGzip(boolean gzip) {
+ this.rgzip = gzip;
+ }
+
+ public void setUserAgent(String userAgent) {
+ this.userAgentHeader =new BasicHeader("User-Agent", userAgent);
+ }
+
+ public void setTimeout(int timeout) {
+ this.timeout = timeout;
+ }
+
+ public void setConnectionTimeout(int ctimeout) {
+ this.connecttimeout = ctimeout;
+ }
+
+ public void setBasicAuthentication(String user, String password) {
+ this.user = user;
+ this.password = password;
+
+ }
+
+ public void setBasicAuthentication(String auth) {
+ this.auth = auth;
+ }
+
+ public void endClientRequest() throws XmlRpcClientException {
+ this.method.releaseConnection();
+ }
+}
http://git-wip-us.apache.org/repos/asf/oodt/blob/78ce45e2/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
index db64716..9c3a5e4 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/system/XmlRpcFileManagerClient.java
@@ -17,10 +17,14 @@
package org.apache.oodt.cas.filemgr.system;
-import org.apache.commons.httpclient.HttpClient;
-import org.apache.commons.httpclient.HttpMethod;
-import org.apache.commons.httpclient.HttpMethodRetryHandler;
-import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.protocol.HttpContext;
import org.apache.oodt.cas.cli.CmdLineUtility;
import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer;
import org.apache.oodt.cas.filemgr.exceptions.FileManagerException;
@@ -43,7 +47,6 @@ import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.filemgr.util.XmlRpcStructFactory;
import org.apache.oodt.cas.filemgr.versioning.Versioner;
import org.apache.oodt.cas.metadata.Metadata;
-import org.apache.xmlrpc.CommonsXmlRpcTransport;
import org.apache.xmlrpc.XmlRpcClient;
import org.apache.xmlrpc.XmlRpcClientException;
import org.apache.xmlrpc.XmlRpcException;
@@ -76,7 +79,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
/* our log stream */
private static Logger LOG = Logger.getLogger(XmlRpcFileManagerClient.class
- .getName());
+ .getName());
/* file manager url */
private URL fileManagerUrl = null;
@@ -95,21 +98,21 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
* @param testConnection Whether or not to check if server at given url is alive.
*/
public XmlRpcFileManagerClient(final URL url, boolean testConnection)
- throws ConnectionException {
+ throws ConnectionException {
// set up the configuration, if there is any
if (System.getProperty("org.apache.oodt.cas.filemgr.properties") != null) {
String configFile = System
- .getProperty("org.apache.oodt.cas.filemgr.properties");
+ .getProperty("org.apache.oodt.cas.filemgr.properties");
LOG.log(Level.INFO,
- "Loading File Manager Configuration Properties from: ["
- + configFile + "]");
+ "Loading File Manager Configuration Properties from: ["
+ + configFile + "]");
try {
System.getProperties().load(
- new FileInputStream(new File(configFile)));
+ new FileInputStream(new File(configFile)));
} catch (Exception e) {
LOG.log(Level.INFO,
- "Error loading configuration properties from: ["
- + configFile + "]");
+ "Error loading configuration properties from: ["
+ + configFile + "]");
}
}
@@ -117,41 +120,52 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
XmlRpcTransportFactory transportFactory = new XmlRpcTransportFactory() {
public XmlRpcTransport createTransport()
- throws XmlRpcClientException {
- HttpClient client = new HttpClient();
- client.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
- new HttpMethodRetryHandler() {
-
- public boolean retryMethod(HttpMethod method,
- IOException e, int count) {
- if (count < Integer
+ throws XmlRpcClientException {
+
+ HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
+ public boolean retryRequest(
+ IOException exception,
+ int count,
+ HttpContext context){
+ if (count < Integer
.getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries",
- 3)) {
- try {
- Thread
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retries",
+ 3)) {
+ try {
+ Thread
.sleep(Integer
- .getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds",
- 0) * 1000);
- return true;
- } catch (Exception ignored) {
- }
- }
- return false;
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connection.retry.interval.seconds",
+ 0) * 1000);
+ return true;
+ } catch (Exception ignored) {
}
+ }
+ return false;
+ }
+ };
+ RequestConfig config = RequestConfig.custom()
+ .setSocketTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
+ 20) * 60 * 1000)
+ .setConnectTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes",
+ 60) * 60 * 1000)
+ .build();
+ Registry<AuthSchemeProvider> r = RegistryBuilder.<AuthSchemeProvider>create().build();
+ HttpClient client = HttpClients.custom().setRetryHandler(myRetryHandler).setDefaultAuthSchemeRegistry(r).setDefaultRequestConfig(config).build();
- });
- CommonsXmlRpcTransport transport = new CommonsXmlRpcTransport(
- url, client);
+ CommonsXmlRpcTransport transport = new CommonsXmlRpcTransport(url, client);
transport
- .setConnectionTimeout(Integer
- .getInteger(
- "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
- 20) * 60 * 1000);
+ .setConnectionTimeout(Integer
+ .getInteger(
+ "org.apache.oodt.cas.filemgr.system.xmlrpc.connectionTimeout.minutes",
+ 20) * 60 * 1000);
transport
- .setTimeout(Integer
- .getInteger(
+ .setTimeout(Integer
+ .getInteger(
"org.apache.oodt.cas.filemgr.system.xmlrpc.requestTimeout.minutes",
60) * 60 * 1000);
@@ -168,7 +182,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (testConnection && !isAlive()) {
throw new ConnectionException("Exception connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
}
}
@@ -179,14 +193,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
Vector<Object> argList = new Vector<Object>();
try {
success = (Boolean) client.execute("filemgr.refreshConfigAndPolicy",
- argList);
+ argList);
} catch (XmlRpcException e) {
LOG.log(Level.WARNING, "XmlRpcException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
success = false;
} catch (IOException e) {
LOG.log(Level.WARNING, "IOException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
success = false;
}
@@ -201,12 +215,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
connected = (Boolean) client.execute("filemgr.isAlive", argList);
} catch (XmlRpcException e) {
LOG.log(Level.WARNING,
- "XmlRpcException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ "XmlRpcException when connecting to filemgr: ["
+ + this.fileManagerUrl + "]");
connected = false;
} catch (IOException e) {
LOG.log(Level.WARNING, "IOException when connecting to filemgr: ["
- + this.fileManagerUrl + "]");
+ + this.fileManagerUrl + "]");
connected = false;
}
@@ -214,17 +228,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean transferringProduct(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
boolean success;
try {
success = (Boolean) client.execute("filemgr.transferringProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -235,9 +249,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean removeProductTransferStatus(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -245,7 +259,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute(
- "filemgr.removeProductTransferStatus", argList);
+ "filemgr.removeProductTransferStatus", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -256,9 +270,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean isTransferComplete(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -266,7 +280,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.isTransferComplete",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -277,9 +291,9 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean moveProduct(Product product, String newPath)
- throws DataTransferException {
+ throws DataTransferException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
argList.add(newPath);
@@ -299,7 +313,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public boolean modifyProduct(Product product) throws CatalogException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -308,7 +322,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.modifyProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -321,7 +335,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public boolean removeProduct(Product product) throws CatalogException {
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
Vector<Object> argList = new Vector<Object>();
argList.add(productHash);
@@ -330,7 +344,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
success = (Boolean) client.execute("filemgr.removeProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -343,7 +357,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public FileTransferStatus getCurrentFileTransfer()
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> statusHash;
@@ -351,7 +365,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
statusHash = (Map<String, Object>) client.execute(
- "filemgr.getCurrentFileTransfer", argList);
+ "filemgr.getCurrentFileTransfer", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -360,7 +374,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (statusHash != null) {
status = XmlRpcStructFactory
- .getFileTransferStatusFromXmlRpc(statusHash);
+ .getFileTransferStatusFromXmlRpc(statusHash);
}
return status;
@@ -368,7 +382,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<FileTransferStatus> getCurrentFileTransfers()
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> statusVector;
@@ -376,7 +390,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
statusVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getCurrentFileTransfers", argList);
+ "filemgr.getCurrentFileTransfers", argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -385,23 +399,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (statusVector != null) {
statuses = XmlRpcStructFactory
- .getFileTransferStatusesFromXmlRpc(statusVector);
+ .getFileTransferStatusesFromXmlRpc(statusVector);
}
return statuses;
}
public double getProductPctTransferred(Product product)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
Double pct;
try {
pct = (Double) client.execute("filemgr.getProductPctTransferred",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -416,16 +430,16 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public double getRefPctTransferred(Reference reference)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> refHash = XmlRpcStructFactory
- .getXmlRpcReference(reference);
+ .getXmlRpcReference(reference);
argList.add(refHash);
Double pct;
try {
pct = (Double) client.execute("filemgr.getRefPctTransferred",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new DataTransferException(e);
} catch (IOException e) {
@@ -441,12 +455,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage pagedQuery(Query query, ProductType type, int pageNum)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> queryHash = XmlRpcStructFactory
- .getXmlRpcQuery(query);
+ .getXmlRpcQuery(query);
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(queryHash);
argList.add(typeHash);
argList.add(pageNum);
@@ -455,7 +469,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.pagedQuery", argList);
+ "filemgr.pagedQuery", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -475,7 +489,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getFirstPage", argList);
+ "filemgr.getFirstPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -500,7 +514,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getLastPage", argList);
+ "filemgr.getLastPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -516,7 +530,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage getNextPage(ProductType type, ProductPage currPage)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
argList.add(XmlRpcStructFactory.getXmlRpcProductPage(currPage));
@@ -526,7 +540,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getNextPage", argList);
+ "filemgr.getNextPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -542,7 +556,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductPage getPrevPage(ProductType type, ProductPage currPage)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
argList.add(XmlRpcStructFactory.getXmlRpcProductPage(currPage));
@@ -552,7 +566,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
pageHash = (Map<String, Object>) client.execute(
- "filemgr.getPrevPage", argList);
+ "filemgr.getPrevPage", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -567,16 +581,16 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public String addProductType(ProductType type)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
String productTypeId;
Vector<Object> argList = new Vector<Object>();
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(typeHash);
try {
productTypeId = (String) client.execute("filemgr.addProductType",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -595,7 +609,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
hasProduct = (Boolean) client.execute("filemgr.hasProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -614,7 +628,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
numProducts = (Integer) client.execute("filemgr.getNumProducts",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -633,7 +647,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
topNProducts = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getTopNProducts", argList);
+ "filemgr.getTopNProducts", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -641,23 +655,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
return XmlRpcStructFactory
- .getProductListFromXmlRpc(topNProducts);
+ .getProductListFromXmlRpc(topNProducts);
}
@SuppressWarnings("unchecked")
public List<Product> getTopNProducts(int n, ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(n);
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
Vector<Map<String, Object>> topNProducts;
try {
topNProducts = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getTopNProducts", argList);
+ "filemgr.getTopNProducts", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -665,14 +679,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
return XmlRpcStructFactory
- .getProductListFromXmlRpc(topNProducts);
+ .getProductListFromXmlRpc(topNProducts);
}
public void setProductTransferStatus(Product product)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
@@ -688,7 +702,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public void addProductReferences(Product product) throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
@@ -701,7 +715,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public void addMetadata(Product product, Metadata metadata)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProduct(product));
@@ -717,7 +731,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public boolean updateMetadata(Product product, Metadata met)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
argList.add(XmlRpcStructFactory.getXmlRpcProduct(product));
argList.add(met.getHashTable());
@@ -744,7 +758,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productId = (String) client.execute("filemgr.catalogProduct",
- argList);
+ argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -758,14 +772,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public Metadata getMetadata(Product product) throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
Map<String, Object> metadata;
try {
metadata = (Map<String, Object>) client.execute(
- "filemgr.getMetadata", argList);
+ "filemgr.getMetadata", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -780,10 +794,10 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings({ "unchecked", "rawtypes" })
public Metadata getReducedMetadata(Product product, List<?> elements)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
argList.add(new Vector(elements));
@@ -791,7 +805,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
metadata = (Map<String, Object>) client.execute(
- "filemgr.getReducedMetadata", argList);
+ "filemgr.getReducedMetadata", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -822,7 +836,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public byte[] retrieveFile(String filePath, int offset, int numBytes)
- throws DataTransferException {
+ throws DataTransferException {
Vector<Object> argList = new Vector<Object>();
argList.add(filePath);
argList.add(offset);
@@ -856,17 +870,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Product> getProductsByProductType(ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
Vector<Map<String, Object>> productVector;
try {
productVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getProductsByProductType", argList);
+ "filemgr.getProductsByProductType", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -882,10 +896,10 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Element> getElementsByProductType(ProductType type)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productTypeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(productTypeHash);
@@ -893,7 +907,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.getElementsByProductType", argList);
+ "filemgr.getElementsByProductType", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -909,7 +923,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementById(String elementId)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementId);
@@ -917,7 +931,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementById", argList);
+ "filemgr.getElementById", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -933,7 +947,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementByName(String elementName)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementName);
@@ -941,7 +955,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementByName", argList);
+ "filemgr.getElementByName", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -957,7 +971,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public Element getElementByName(String elementName, ProductType type)
- throws ValidationLayerException {
+ throws ValidationLayerException {
Vector<Object> argList = new Vector<Object>();
argList.add(elementName);
argList.add(XmlRpcStructFactory.getXmlRpcProductType(type));
@@ -966,7 +980,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
elementHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getElementByName", argList);
+ "filemgr.getElementByName", argList);
} catch (XmlRpcException e) {
throw new ValidationLayerException(e);
} catch (IOException e) {
@@ -981,17 +995,17 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
}
public List<QueryResult> complexQuery(ComplexQuery complexQuery)
- throws CatalogException {
+ throws CatalogException {
try {
Map<String, Object> complexQueryHash = XmlRpcStructFactory
- .getXmlRpcComplexQuery(complexQuery);
+ .getXmlRpcComplexQuery(complexQuery);
Vector<Object> argList = new Vector<Object>();
argList.add(complexQueryHash);
@SuppressWarnings("unchecked")
Vector<Map<String, Object>> queryResultHashVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.complexQuery", argList);
+ .execute("filemgr.complexQuery", argList);
return XmlRpcStructFactory
- .getQueryResultsFromXmlRpc(queryResultHashVector);
+ .getQueryResultsFromXmlRpc(queryResultHashVector);
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e);
@@ -1000,12 +1014,12 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public List<Product> query(Query query, ProductType type)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> queryHash = XmlRpcStructFactory
- .getXmlRpcQuery(query);
+ .getXmlRpcQuery(query);
Map<String, Object> typeHash = XmlRpcStructFactory
- .getXmlRpcProductType(type);
+ .getXmlRpcProductType(type);
argList.add(queryHash);
argList.add(typeHash);
@@ -1013,7 +1027,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productVector = (Vector<Map<String, Object>>) client.execute(
- "filemgr.query", argList);
+ "filemgr.query", argList);
} catch (XmlRpcException e) {
LOG.log(Level.SEVERE, e.getMessage());
throw new CatalogException(e);
@@ -1032,14 +1046,14 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
@SuppressWarnings("unchecked")
public ProductType getProductTypeByName(String productTypeName)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Hashtable<String, Object> productTypeHash;
Vector<Object> argList = new Vector<Object>();
argList.add(productTypeName);
try {
productTypeHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getProductTypeByName", argList);
+ "filemgr.getProductTypeByName", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e.getLocalizedMessage());
} catch (IOException e) {
@@ -1050,20 +1064,20 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeFromXmlRpc(productTypeHash);
+ .getProductTypeFromXmlRpc(productTypeHash);
}
}
@SuppressWarnings("unchecked")
public ProductType getProductTypeById(String productTypeId)
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Hashtable<String, Object> productTypeHash;
Vector<Object> argList = new Vector<Object>();
argList.add(productTypeId);
try {
productTypeHash = (Hashtable<String, Object>) client.execute(
- "filemgr.getProductTypeById", argList);
+ "filemgr.getProductTypeById", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -1074,20 +1088,20 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeFromXmlRpc(productTypeHash);
+ .getProductTypeFromXmlRpc(productTypeHash);
}
}
@SuppressWarnings("unchecked")
public List<ProductType> getProductTypes()
- throws RepositoryManagerException {
+ throws RepositoryManagerException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> productTypeVector;
try {
productTypeVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.getProductTypes", argList);
+ .execute("filemgr.getProductTypes", argList);
} catch (XmlRpcException e) {
throw new RepositoryManagerException(e);
} catch (IOException e) {
@@ -1098,23 +1112,23 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getProductTypeListFromXmlRpc(productTypeVector);
+ .getProductTypeListFromXmlRpc(productTypeVector);
}
}
@SuppressWarnings("unchecked")
public List<Reference> getProductReferences(Product product)
- throws CatalogException {
+ throws CatalogException {
Vector<Object> argList = new Vector<Object>();
Vector<Map<String, Object>> productReferenceVector;
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
try {
productReferenceVector = (Vector<Map<String, Object>>) client
- .execute("filemgr.getProductReferences", argList);
+ .execute("filemgr.getProductReferences", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1125,7 +1139,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
return null;
} else {
return XmlRpcStructFactory
- .getReferencesFromXmlRpc(productReferenceVector);
+ .getReferencesFromXmlRpc(productReferenceVector);
}
}
@@ -1138,7 +1152,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productHash = (Map<String, Object>) client.execute(
- "filemgr.getProductById", argList);
+ "filemgr.getProductById", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1161,7 +1175,7 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
try {
productHash = (Map<String, Object>) client.execute(
- "filemgr.getProductByName", argList);
+ "filemgr.getProductByName", argList);
} catch (XmlRpcException e) {
throw new CatalogException(e);
} catch (IOException e) {
@@ -1177,38 +1191,38 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
public String ingestProduct(Product product, Metadata metadata,
boolean clientTransfer)
- throws VersioningException, XmlRpcException, FileManagerException {
+ throws VersioningException, XmlRpcException, FileManagerException {
try {
// ingest product
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
argList.add(metadata.getHashTable());
argList.add(clientTransfer);
String productId = (String) client.execute("filemgr.ingestProduct",
- argList);
+ argList);
if (clientTransfer) {
LOG.log(Level.FINEST,
- "File Manager Client: clientTransfer enabled: "
- + "transfering product ["
- + product.getProductName() + "]");
+ "File Manager Client: clientTransfer enabled: "
+ + "transfering product ["
+ + product.getProductName() + "]");
// we need to transfer the product ourselves
// make sure we have the product ID
if (productId == null) {
throw new Exception("Request to ingest product: "
- + product.getProductName()
- + " but no product ID returned from File "
- + "Manager ingest");
+ + product.getProductName()
+ + " but no product ID returned from File "
+ + "Manager ingest");
}
if (dataTransfer == null) {
throw new Exception("Request to ingest product: ["
- + product.getProductName()
- + "] using client transfer, but no "
- + "dataTransferer specified!");
+ + product.getProductName()
+ + "] using client transfer, but no "
+ + "dataTransferer specified!");
}
product.setProductId(productId);
@@ -1216,8 +1230,8 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
if (!Boolean.getBoolean("org.apache.oodt.cas.filemgr.serverside.versioning")) {
// version the product
Versioner versioner = GenericFileManagerObjectFactory
- .getVersionerFromClassName(product.getProductType()
- .getVersioner());
+ .getVersionerFromClassName(product.getProductType()
+ .getVersioner());
if (versioner != null) {
versioner.createDataStoreReferences(product, metadata);
}
@@ -1227,13 +1241,13 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
addProductReferences(product);
} catch (CatalogException e) {
LOG
- .log(
- Level.SEVERE,
- "ingestProduct: RepositoryManagerException "
- + "when adding Product References for Product : "
- + product.getProductName()
- + " to RepositoryManager: Message: "
- + e);
+ .log(
+ Level.SEVERE,
+ "ingestProduct: RepositoryManagerException "
+ + "when adding Product References for Product : "
+ + product.getProductName()
+ + " to RepositoryManager: Message: "
+ + e);
throw e;
}
} else {
@@ -1251,19 +1265,19 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
setProductTransferStatus(product);
} catch (CatalogException e) {
LOG
- .log(
- Level.SEVERE,
- "ingestProduct: RepositoryManagerException "
- + "when updating product transfer status for Product: "
- + product.getProductName()
- + " Message: " + e);
+ .log(
+ Level.SEVERE,
+ "ingestProduct: RepositoryManagerException "
+ + "when updating product transfer status for Product: "
+ + product.getProductName()
+ + " Message: " + e);
throw e;
}
} catch (Exception e) {
LOG.log(Level.SEVERE,
- "ingestProduct: DataTransferException when transfering Product: "
- + product.getProductName() + ": Message: "
- + e);
+ "ingestProduct: DataTransferException when transfering Product: "
+ + product.getProductName() + ": Message: "
+ + e);
throw new DataTransferException(e);
}
@@ -1274,81 +1288,81 @@ public class XmlRpcFileManagerClient implements FileManagerClient {
} catch (VersioningException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.SEVERE,
- "ingestProduct: VersioningException when versioning Product: "
- + product.getProductName() + " with Versioner "
- + product.getProductType().getVersioner()
- + ": Message: " + e);
+ "ingestProduct: VersioningException when versioning Product: "
+ + product.getProductName() + " with Versioner "
+ + product.getProductType().getVersioner()
+ + ": Message: " + e);
throw new VersioningException(e);
} catch (XmlRpcException e2) {
LOG.log(Level.SEVERE, "Failed to ingest product [ name:" + product.getProductName() + "] :" + e2.getMessage()
- + " -- rolling back ingest");
+ + " -- rolling back ingest");
try {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
client.execute("filemgr.removeProduct", argList);
} catch (Exception e1) {
LOG.log(Level.SEVERE, "Failed to rollback ingest of product ["
- + product + "] : " + e2.getMessage());
+ + product + "] : " + e2.getMessage());
}
throw e2;
} catch (Exception e) {
LOG.log(Level.SEVERE, "Failed to ingest product [ id: " + product.getProductId() +
- "/ name:" + product.getProductName() + "] :" + e + " -- rolling back ingest");
+ "/ name:" + product.getProductName() + "] :" + e + " -- rolling back ingest");
try {
Vector<Object> argList = new Vector<Object>();
Map<String, Object> productHash = XmlRpcStructFactory
- .getXmlRpcProduct(product);
+ .getXmlRpcProduct(product);
argList.add(productHash);
client.execute("filemgr.removeProduct", argList);
} catch (Exception e1) {
LOG.log(Level.SEVERE, "Failed to rollback ingest of product ["
- + product + "] : " + e);
+ + product + "] : " + e);
}
throw new FileManagerException("Failed to ingest product [" + product + "] : "
- + e);
+ + e);
}
}
@SuppressWarnings("unchecked")
public Metadata getCatalogValues(Metadata metadata, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(metadata.getHashTable());
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
Metadata m = new Metadata();
m.addMetadata((Map<String, Object>) this.client.execute(
- "filemgr.getCatalogValues", args));
+ "filemgr.getCatalogValues", args));
return m;
}
@SuppressWarnings("unchecked")
public Metadata getOrigValues(Metadata metadata, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(metadata.getHashTable());
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
Metadata m = new Metadata();
m.addMetadata((Map<String, Object>) this.client.execute(
- "filemgr.getOrigValues", args));
+ "filemgr.getOrigValues", args));
return m;
}
@SuppressWarnings("unchecked")
public Query getCatalogQuery(Query query, ProductType productType)
- throws XmlRpcException, IOException {
+ throws XmlRpcException, IOException {
Vector<Object> args = new Vector<Object>();
args.add(XmlRpcStructFactory.getXmlRpcQuery(query));
args.add(XmlRpcStructFactory.getXmlRpcProductType(productType));
return XmlRpcStructFactory
- .getQueryFromXmlRpc((Hashtable<String, Object>) this.client
- .execute("filemgr.getCatalogQuery", args));
+ .getQueryFromXmlRpc((Hashtable<String, Object>) this.client
+ .execute("filemgr.getCatalogQuery", args));
}
public static void main(String[] args) {
[45/50] [abbrv] oodt git commit: [OODT-963] Fixed surefire test
failures in workflow manager due to forkMode
Posted by ma...@apache.org.
[OODT-963] Fixed surefire test failures in workflow manager due to forkMode
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/c7d0842d
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/c7d0842d
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/c7d0842d
Branch: refs/heads/master
Commit: c7d0842d9490fcf95f3ba0e07c1dc3cd8b16b76a
Parents: 54089ca
Author: Imesha Sudasingha <im...@apache.org>
Authored: Wed Oct 11 18:19:28 2017 +0530
Committer: Imesha Sudasingha <im...@gmail.com>
Committed: Thu Oct 12 08:07:14 2017 +0530
----------------------------------------------------------------------
.../config/distributed/DistributedConfigurationManager.java | 6 ++++--
.../org/apache/oodt/config/distributed/utils/ConfigUtils.java | 1 +
.../src/test/resources/distributed/config/config-publisher.xml | 1 -
workflow/pom.xml | 3 +--
.../oodt/cas/workflow/system/AvroRpcWorkflowManager.java | 3 +--
5 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7d0842d/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
index fe36a02..d93c421 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/DistributedConfigurationManager.java
@@ -42,7 +42,6 @@ import java.util.concurrent.TimeUnit;
import static org.apache.oodt.config.Constants.Properties.ZK_CONNECT_STRING;
import static org.apache.oodt.config.Constants.Properties.ZK_PROPERTIES_FILE;
import static org.apache.oodt.config.Constants.Properties.ZK_STARTUP_TIMEOUT;
-import static org.apache.oodt.config.distributed.utils.ConfigUtils.getOODTProjectName;
/**
* Distributed configuration manager implementation. This class make use of a {@link CuratorFramework} instance to
@@ -93,7 +92,10 @@ public class DistributedConfigurationManager extends ConfigurationManager {
};
public DistributedConfigurationManager(Component component) {
- super(component, getOODTProjectName());
+ super(component, ConfigUtils.getOODTProjectName());
+
+ logger.info("Found project name {} for component {}", this.project, this.component);
+
this.zNodePaths = new ZNodePaths(this.project, this.component.getName());
if (System.getProperty(ZK_PROPERTIES_FILE) == null && System.getProperty(ZK_CONNECT_STRING) == null) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7d0842d/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
index bbb24b0..2672ceb 100644
--- a/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
+++ b/config/src/main/java/org/apache/oodt/config/distributed/utils/ConfigUtils.java
@@ -75,6 +75,7 @@ public class ConfigUtils {
project = System.getenv(Env.OODT_PROJECT);
}
+ logger.debug("Project name {}", project);
return project == null ? Constants.DEFAULT_PROJECT : project;
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7d0842d/filemgr/src/test/resources/distributed/config/config-publisher.xml
----------------------------------------------------------------------
diff --git a/filemgr/src/test/resources/distributed/config/config-publisher.xml b/filemgr/src/test/resources/distributed/config/config-publisher.xml
index f6a97c1..dba3c27 100644
--- a/filemgr/src/test/resources/distributed/config/config-publisher.xml
+++ b/filemgr/src/test/resources/distributed/config/config-publisher.xml
@@ -21,7 +21,6 @@
<bean id="filemgr-config-publisher" class="org.apache.oodt.config.distributed.DistributedConfigurationPublisher">
<constructor-arg value="FILE_MANAGER"/>
-
<constructor-arg value="primary"/>
<property name="propertiesFiles">
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7d0842d/workflow/pom.xml
----------------------------------------------------------------------
diff --git a/workflow/pom.xml b/workflow/pom.xml
index cad9c30..03062aa 100644
--- a/workflow/pom.xml
+++ b/workflow/pom.xml
@@ -203,9 +203,8 @@ the License.
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
- <version>2.4</version>
<configuration>
- <forkMode>never</forkMode>
+ <forkMode>pertest</forkMode>
<useSystemClassLoader>false</useSystemClassLoader>
<systemProperties>
<property>
http://git-wip-us.apache.org/repos/asf/oodt/blob/c7d0842d/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java
index ebc2e30..efe5d62 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/system/AvroRpcWorkflowManager.java
@@ -97,8 +97,7 @@ public class AvroRpcWorkflowManager implements WorkflowManager,org.apache.oodt.c
if(engine == null){
throw new NullPointerException("null engine");
}
- engine.setWorkflowManagerUrl(safeGetUrlFromString("http://"
- + getHostname() + ":" + this.webServerPort));
+ engine.setWorkflowManagerUrl(safeGetUrlFromString("http://" + getHostname() + ":" + this.webServerPort));
repo = getWorkflowRepositoryFromProperty();
LOG.log(Level.INFO, "Starting Netty Server");
[26/50] [abbrv] oodt git commit: remove invalid tags
Posted by ma...@apache.org.
remove invalid tags
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/7c104c4c
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/7c104c4c
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/7c104c4c
Branch: refs/heads/master
Commit: 7c104c4ce5c4c5552330ef8a5d5dec1599d27de4
Parents: b1e39db
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Nov 10 15:46:21 2016 +0000
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:42:28 2017 -0800
----------------------------------------------------------------------
core/pom.xml | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/7c104c4c/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 3b7b857..70a13bf 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -617,7 +617,6 @@ mm
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9</version>
<configuration>
- <outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
<docencoding>UTF-8</docencoding>
<encoding>UTF-8</encoding>
@@ -660,7 +659,6 @@ mm
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
<configuration>
- <outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
<docencoding>UTF-8</docencoding>
<encoding>UTF-8</encoding>
[09/50] [abbrv] oodt git commit: - ignore
Posted by ma...@apache.org.
- ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/17d72450
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/17d72450
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/17d72450
Branch: refs/heads/master
Commit: 17d7245003879e5b4f8981d133be3bf4a91962a5
Parents: e7fb5dc
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:29:05 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:11:07 2017 -0800
----------------------------------------------------------------------
protocol/api/.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/17d72450/protocol/api/.gitignore
----------------------------------------------------------------------
diff --git a/protocol/api/.gitignore b/protocol/api/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/protocol/api/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[13/50] [abbrv] oodt git commit: - ignore
Posted by ma...@apache.org.
- ignore
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/30b5a01f
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/30b5a01f
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/30b5a01f
Branch: refs/heads/master
Commit: 30b5a01f88dca5da902cf70b74a215436db5406e
Parents: d7a228e
Author: Chris Mattmann <ma...@apache.org>
Authored: Wed Jul 6 13:32:40 2016 -0700
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:11:40 2017 -0800
----------------------------------------------------------------------
app/weditor/.gitignore | 1 +
commons/.gitignore | 1 +
pcs/input/.gitignore | 1 +
product/.gitignore | 1 +
protocol/sftp/.gitignore | 1 +
xmlps/.gitignore | 1 +
xmlquery/.gitignore | 1 +
7 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/app/weditor/.gitignore
----------------------------------------------------------------------
diff --git a/app/weditor/.gitignore b/app/weditor/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/app/weditor/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/commons/.gitignore
----------------------------------------------------------------------
diff --git a/commons/.gitignore b/commons/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/commons/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/pcs/input/.gitignore
----------------------------------------------------------------------
diff --git a/pcs/input/.gitignore b/pcs/input/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/pcs/input/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/product/.gitignore
----------------------------------------------------------------------
diff --git a/product/.gitignore b/product/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/product/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/protocol/sftp/.gitignore
----------------------------------------------------------------------
diff --git a/protocol/sftp/.gitignore b/protocol/sftp/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/protocol/sftp/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/xmlps/.gitignore
----------------------------------------------------------------------
diff --git a/xmlps/.gitignore b/xmlps/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/xmlps/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
http://git-wip-us.apache.org/repos/asf/oodt/blob/30b5a01f/xmlquery/.gitignore
----------------------------------------------------------------------
diff --git a/xmlquery/.gitignore b/xmlquery/.gitignore
new file mode 100644
index 0000000..b54523f
--- /dev/null
+++ b/xmlquery/.gitignore
@@ -0,0 +1 @@
+/maven-eclipse.xml
[21/50] [abbrv] oodt git commit: Upgrade lucene in workflow
Posted by ma...@apache.org.
Upgrade lucene in workflow
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/95def41f
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/95def41f
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/95def41f
Branch: refs/heads/master
Commit: 95def41ff023c68ddf3321c0a8ede52c6a5d2e19
Parents: 2f79f68
Author: Tom Barber <to...@analytical-labs.com>
Authored: Mon Jul 11 12:37:11 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:35:31 2017 -0800
----------------------------------------------------------------------
workflow/pom.xml | 4 +
.../LuceneWorkflowInstanceRepository.java | 320 +++++++++++--------
...LuceneWorkflowInstanceRepositoryFactory.java | 25 +-
.../workflow/tools/TestInstanceRepoCleaner.java | 2 +-
workflow/src/test/resources/testinstrepo/_4.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_4.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_4.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/_43.cfs | Bin 14148 -> 0 bytes
workflow/src/test/resources/testinstrepo/_5.cfe | Bin 0 -> 289 bytes
workflow/src/test/resources/testinstrepo/_5.cfs | Bin 0 -> 9749 bytes
workflow/src/test/resources/testinstrepo/_5.si | Bin 0 -> 392 bytes
.../src/test/resources/testinstrepo/deletable | Bin 4 -> 0 bytes
.../resources/testinstrepo/pending_segments_b | Bin 0 -> 198 bytes
.../src/test/resources/testinstrepo/segments | Bin 28 -> 0 bytes
.../src/test/resources/testinstrepo/segments_a | Bin 0 -> 136 bytes
.../src/test/resources/testinstrepo/write.lock | 0
16 files changed, 213 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/pom.xml
----------------------------------------------------------------------
diff --git a/workflow/pom.xml b/workflow/pom.xml
index 72fb549..445cbdb 100644
--- a/workflow/pom.xml
+++ b/workflow/pom.xml
@@ -130,6 +130,10 @@ the License.
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
index b6c8447..c3bf291 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepository.java
@@ -18,20 +18,16 @@
package org.apache.oodt.cas.workflow.instrepo;
-//OODT imports
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.cas.workflow.lifecycle.WorkflowLifecycleStage;
import org.apache.oodt.cas.workflow.lifecycle.WorkflowState;
@@ -65,6 +61,9 @@ import java.util.logging.Logger;
*/
public class LuceneWorkflowInstanceRepository extends
AbstractPaginatibleInstanceRepository {
+ Directory indexDir = null;
+ private DirectoryReader reader;
+ /* the path to the index directory for this catalog */
public static final int MERGE_FACTOR = 20;
/* path to lucene index directory to store wInst info */
@@ -76,6 +75,7 @@ public class LuceneWorkflowInstanceRepository extends
/* our workflow inst id generator */
private static UUIDGenerator generator = UUIDGenerator.getInstance();
+ private int mergeFactor = 20;
/**
*
@@ -83,6 +83,11 @@ public class LuceneWorkflowInstanceRepository extends
public LuceneWorkflowInstanceRepository(String idxPath, int pageSize) {
this.idxFilePath = idxPath;
this.pageSize = pageSize;
+ try {
+ indexDir = FSDirectory.open(new File( idxFilePath ).toPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
}
/*
@@ -93,16 +98,20 @@ public class LuceneWorkflowInstanceRepository extends
public int getNumWorkflowInstances() throws InstanceRepositoryException {
IndexSearcher searcher = null;
int numInsts = -1;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
Term instIdTerm = new Term("myfield", "myvalue");
org.apache.lucene.search.Query query = new TermQuery(instIdTerm);
Sort sort = new Sort(new SortField("workflow_inst_startdatetime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ TopDocs topDocs = searcher.search(query, 1, sort);
- numInsts = hits.length();
+ numInsts = topDocs.totalHits;
} catch (IOException e) {
LOG.log(Level.WARNING,
@@ -112,7 +121,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -130,16 +139,20 @@ public class LuceneWorkflowInstanceRepository extends
throws InstanceRepositoryException {
IndexSearcher searcher = null;
int numInsts = -1;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
Term instIdTerm = new Term("workflow_inst_status", status);
org.apache.lucene.search.Query query = new TermQuery(instIdTerm);
Sort sort = new Sort(new SortField("workflow_inst_startdatetime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ TopDocs topDocs = searcher.search(query, 1, sort);
- numInsts = hits.length();
+ numInsts = topDocs.totalHits;
} catch (IOException e) {
LOG.log(Level.WARNING,
@@ -149,7 +162,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -204,21 +217,27 @@ public class LuceneWorkflowInstanceRepository extends
throws InstanceRepositoryException {
IndexSearcher searcher = null;
WorkflowInstance wInst = null;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
Term instIdTerm = new Term("workflow_inst_id", workflowInstId);
org.apache.lucene.search.Query query = new TermQuery(instIdTerm);
- Hits hits = searcher.search(query);
+ TopDocs check = searcher.search(query, 1);
- if (hits.length() != 1) {
+ if (check.totalHits != 1) {
LOG.log(Level.WARNING, "The workflow instance: ["
+ workflowInstId + "] is not being "
+ "managed by this " + "workflow engine, or "
- + "is not unique in the catalog: num hits: ["+hits.length()+"]");
+ + "is not unique in the catalog: num hits: ["+check.totalHits+"]");
return null;
} else {
- Document instDoc = hits.doc(0);
+ TopDocs topDocs = searcher.search(query, check.totalHits);
+ ScoreDoc[] hits = topDocs.scoreDocs;
+ Document instDoc = searcher.doc(hits[0].doc);
wInst = toWorkflowInstance(instDoc);
}
@@ -230,7 +249,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -247,22 +266,29 @@ public class LuceneWorkflowInstanceRepository extends
public List getWorkflowInstances() throws InstanceRepositoryException {
IndexSearcher searcher = null;
List wInsts = null;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
Term instIdTerm = new Term("myfield", "myvalue");
org.apache.lucene.search.Query query = new TermQuery(instIdTerm);
Sort sort = new Sort(new SortField("workflow_inst_startdatetime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
-
- if (hits.length() > 0) {
- wInsts = new Vector(hits.length());
-
- for (int i = 0; i < hits.length(); i++) {
- Document doc = hits.doc(i);
- WorkflowInstance wInst = toWorkflowInstance(doc);
- wInsts.add(wInst);
+ SortField.Type.STRING, true));
+ TopDocs check = searcher.search(query, 1, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
+ ScoreDoc[] hits = topDocs.scoreDocs;
+ if (topDocs.totalHits > 0) {
+ wInsts = new Vector(hits.length);
+
+ for (ScoreDoc hit : hits) {
+ Document doc = searcher.doc(hit.doc);
+ WorkflowInstance wInst = toWorkflowInstance(doc);
+ wInsts.add(wInst);
+ }
}
}
@@ -274,7 +300,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -292,22 +318,29 @@ public class LuceneWorkflowInstanceRepository extends
throws InstanceRepositoryException {
IndexSearcher searcher = null;
List wInsts = null;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
Term instIdTerm = new Term("workflow_inst_status", status);
org.apache.lucene.search.Query query = new TermQuery(instIdTerm);
Sort sort = new Sort(new SortField("workflow_inst_startdatetime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
-
- if (hits.length() > 0) {
- wInsts = new Vector(hits.length());
-
- for (int i = 0; i < hits.length(); i++) {
- Document doc = hits.doc(i);
- WorkflowInstance wInst = toWorkflowInstance(doc);
- wInsts.add(wInst);
+ SortField.Type.STRING, true));
+ TopDocs check = searcher.search(query, 1, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(query, check.totalHits, sort);
+ ScoreDoc[] hits = topDocs.scoreDocs;
+ if (hits.length > 0) {
+ wInsts = new Vector(hits.length);
+
+ for (ScoreDoc hit : hits) {
+ Document doc = searcher.doc(hit.doc);
+ WorkflowInstance wInst = toWorkflowInstance(doc);
+ wInsts.add(wInst);
+ }
}
}
@@ -319,7 +352,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -338,12 +371,16 @@ public class LuceneWorkflowInstanceRepository extends
throws InstanceRepositoryException {
List instIds = null;
IndexSearcher searcher = null;
-
try {
- searcher = new IndexSearcher(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
Term instIdTerm = new Term("myfield", "myvalue");
if (status != null) {
@@ -355,32 +392,36 @@ public class LuceneWorkflowInstanceRepository extends
BooleanClause.Occur.MUST);
Sort sort = new Sort(new SortField("workflow_inst_startdatetime",
- SortField.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE,
"Querying LuceneWorkflowInstanceRepository: q: ["
+ booleanQuery + "]");
- Hits hits = searcher.search(booleanQuery, sort);
- if (hits.length() > 0) {
+ TopDocs check = searcher.search(booleanQuery.build(), 1, sort);
+ if(check.totalHits>0) {
+ TopDocs topDocs = searcher.search(booleanQuery.build(), check.totalHits, sort);
+ ScoreDoc[] hits = topDocs.scoreDocs;
- int startNum = (pageNum - 1) * pageSize;
- if (startNum > hits.length()) {
- startNum = 0;
- }
+ if (hits.length > 0) {
+
+ int startNum = (pageNum - 1) * pageSize;
+ if (startNum > hits.length) {
+ startNum = 0;
+ }
- instIds = new Vector(pageSize);
+ instIds = new Vector(pageSize);
- for (int i = startNum; i < Math.min(hits.length(),
- (startNum + pageSize)); i++) {
- Document instDoc = hits.doc(i);
- WorkflowInstance inst = toWorkflowInstance(instDoc);
- instIds.add(inst.getId());
+ for (int i = startNum; i < Math.min(hits.length,
+ (startNum + pageSize)); i++) {
+ Document instDoc = searcher.doc(hits[i].doc);
+ WorkflowInstance inst = toWorkflowInstance(instDoc);
+ instIds.add(inst.getId());
+ }
+ } else {
+ LOG.log(Level.WARNING, "No workflow instances found "
+ + "when attempting to paginate!");
}
- } else {
- LOG.log(Level.WARNING, "No workflow instances found "
- + "when attempting to paginate!");
}
-
} catch (IOException e) {
LOG.log(Level.WARNING,
"IOException when opening index directory: [" + idxFilePath
@@ -389,7 +430,7 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO Shutdown searcher
} catch (Exception ignore) {
}
}
@@ -401,13 +442,26 @@ public class LuceneWorkflowInstanceRepository extends
private synchronized void removeWorkflowInstanceDocument(
WorkflowInstance inst) throws InstanceRepositoryException {
IndexReader reader = null;
-
try {
- reader = IndexReader.open(idxFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ reader = DirectoryReader.open(indexDir);
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+
+ IndexWriter writer = new IndexWriter(indexDir, config);
LOG.log(Level.FINE,
"LuceneWorkflowEngine: remove document from index for workflow instance: ["
+ inst.getId() + "]");
- reader.deleteDocuments(new Term("workflow_inst_id", inst.getId()));
+ writer.deleteDocuments(new Term("workflow_inst_id", inst.getId()));
+ writer.close();
} catch (IOException e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG
@@ -432,17 +486,15 @@ public class LuceneWorkflowInstanceRepository extends
WorkflowInstance wInst) throws InstanceRepositoryException {
IndexWriter writer = null;
- File indexDir = new File(idxFilePath);
-
- boolean createIndex;
-
- createIndex = !(indexDir.exists() && indexDir.isDirectory());
-
try {
- writer = new IndexWriter(idxFilePath, new StandardAnalyzer(),
- createIndex);
- writer.setMergeFactor(MERGE_FACTOR);
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+ writer = new IndexWriter(indexDir, config);
Document doc = toDoc(wInst);
writer.addDocument(doc);
} catch (IOException e) {
@@ -454,7 +506,8 @@ public class LuceneWorkflowInstanceRepository extends
} finally {
try {
writer.close();
- } catch (Exception ignore) {
+ } catch (Exception e) {
+ System.out.println(e);
}
}
@@ -465,75 +518,85 @@ public class LuceneWorkflowInstanceRepository extends
// store the workflow instance info first
doc.add(new Field("workflow_inst_id", workflowInst.getId(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
doc.add(new Field("workflow_inst_timesblocked",
- String.valueOf(workflowInst.getTimesBlocked()), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ String.valueOf(workflowInst.getTimesBlocked()), StringField.TYPE_STORED));
// will leave this for back compat, but will also store
// category
doc.add(new Field("workflow_inst_status", workflowInst.getStatus(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
if(workflowInst.getState() != null){
WorkflowState state = workflowInst.getState();
if(state.getDescription() != null){
doc.add(new Field("workflow_inst_state_desc",
- state.getDescription(), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ state.getDescription(), StringField.TYPE_STORED));
}
if(state.getMessage() != null){
doc.add(new Field("workflow_inst_state_message",
- state.getMessage(), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ state.getMessage(), StringField.TYPE_STORED));
}
if(state.getCategory() != null && state.getCategory().getName() != null){
doc.add(new Field("workflow_inst_state_category",
- state.getCategory().getName(), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ state.getCategory().getName(), StringField.TYPE_STORED));
}
}
doc
.add(new Field("workflow_inst_current_task_id", workflowInst
- .getCurrentTaskId(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getCurrentTaskId(), StringField.TYPE_STORED));
doc
.add(new Field(
"workflow_inst_currenttask_startdatetime",
workflowInst.getCurrentTaskStartDateTimeIsoStr() != null ? workflowInst
.getCurrentTaskStartDateTimeIsoStr()
- : "", Field.Store.YES, Field.Index.UN_TOKENIZED));
+ : "", StringField.TYPE_STORED));
+
+ doc.add(new SortedDocValuesField("workflow_inst_currenttask_startdatetime", new BytesRef(workflowInst.getCurrentTaskStartDateTimeIsoStr() != null ? workflowInst
+ .getCurrentTaskStartDateTimeIsoStr()
+ : "")));
+
doc.add(new Field("workflow_inst_currenttask_enddatetime", workflowInst
.getCurrentTaskEndDateTimeIsoStr() != null ? workflowInst
- .getCurrentTaskEndDateTimeIsoStr() : "", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getCurrentTaskEndDateTimeIsoStr() : "", StringField.TYPE_STORED));
+ doc.add(new SortedDocValuesField("workflow_inst_currenttask_enddatetime", new BytesRef(workflowInst
+ .getCurrentTaskEndDateTimeIsoStr() != null ? workflowInst
+ .getCurrentTaskEndDateTimeIsoStr() : "")));
+
doc.add(new Field("workflow_inst_startdatetime", workflowInst
.getStartDateTimeIsoStr() != null ? workflowInst
- .getStartDateTimeIsoStr() : "", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getStartDateTimeIsoStr() : "", StringField.TYPE_STORED));
+ doc.add(new SortedDocValuesField("workflow_inst_startdatetime", new BytesRef(workflowInst
+ .getStartDateTimeIsoStr() != null ? workflowInst
+ .getStartDateTimeIsoStr() : "")));
+
doc.add(new Field("workflow_inst_enddatetime", workflowInst
.getEndDateTimeIsoStr() != null ? workflowInst
- .getEndDateTimeIsoStr() : "", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getEndDateTimeIsoStr() : "", StringField.TYPE_STORED));
+ doc.add(new SortedDocValuesField("workflow_inst_enddatetime", new BytesRef(workflowInst
+ .getEndDateTimeIsoStr() != null ? workflowInst
+ .getEndDateTimeIsoStr() : "")));
+
doc.add(new Field("workflow_inst_priority",
workflowInst.getPriority() != null ?
String.valueOf(workflowInst.getPriority().getValue()):
String.valueOf(Priority.getDefault().getValue()),
- Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
// add all metadata
addInstanceMetadataToDoc(doc, workflowInst.getSharedContext());
// store the workflow info too
doc.add(new Field("workflow_id", workflowInst.getWorkflow().getId(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
doc.add(new Field("workflow_name",
- workflowInst.getWorkflow().getName(), Field.Store.YES,
- Field.Index.NO));
+ workflowInst.getWorkflow().getName(), StringField.TYPE_STORED));
// store the tasks
addTasksToDoc(doc, workflowInst.getWorkflow().getTasks());
@@ -544,8 +607,7 @@ public class LuceneWorkflowInstanceRepository extends
, doc);
// add the default field (so that we can do a query for *)
- doc.add(new Field("myfield", "myvalue", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ doc.add(new Field("myfield", "myvalue", StringField.TYPE_STORED));
return doc;
}
@@ -557,15 +619,14 @@ public class LuceneWorkflowInstanceRepository extends
if (metVals != null && metVals.size() > 0) {
for (Object metVal1 : metVals) {
String metVal = (String) metVal1;
- doc.add(new Field(metKey, metVal, Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ doc.add(new Field(metKey, metVal, StringField.TYPE_STORED));
}
// now index the field name so that we can use it to
// look it up when converting from doc to
// WorkflowInstance
doc.add(new Field("workflow_inst_met_flds", metKey,
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
}
}
@@ -576,16 +637,13 @@ public class LuceneWorkflowInstanceRepository extends
if (tasks != null && tasks.size() > 0) {
for (Object task1 : tasks) {
WorkflowTask task = (WorkflowTask) task1;
- doc.add(new Field("task_id", task.getTaskId(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ doc.add(new Field("task_id", task.getTaskId(), StringField.TYPE_STORED));
doc.add(new Field("task_name", task.getTaskName(),
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
doc.add(new Field("task_order",
- String.valueOf(task.getOrder()), Field.Store.YES,
- Field.Index.NO));
+ String.valueOf(task.getOrder()), StringField.TYPE_STORED));
doc.add(new Field("task_class",
- task.getTaskInstanceClassName(), Field.Store.YES,
- Field.Index.NO));
+ task.getTaskInstanceClassName(), StringField.TYPE_STORED));
addConditionsToDoc(task.getTaskId(), task.getConditions(), doc);
addTaskConfigToDoc(task.getTaskId(), task.getTaskConfig(), doc);
@@ -601,9 +659,9 @@ public class LuceneWorkflowInstanceRepository extends
String propValue = config.getProperty(propName);
doc.add(new Field(taskId + "_config_property_name", propName,
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
doc.add(new Field(taskId + "_config_property_value", propValue,
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
}
}
}
@@ -614,17 +672,17 @@ public class LuceneWorkflowInstanceRepository extends
for (Object aConditionList : conditionList) {
WorkflowCondition cond = (WorkflowCondition) aConditionList;
doc.add(new Field(taskId + "_condition_name", cond.getConditionName(),
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
doc.add(new Field(taskId + "_condition_id", cond.getConditionId(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
doc.add(new Field(taskId + "_condition_class", cond
- .getConditionInstanceClassName(), Field.Store.YES, Field.Index.NO));
+ .getConditionInstanceClassName(),StringField.TYPE_STORED));
doc.add(new Field(taskId + "_condition_order", String.valueOf(cond
- .getOrder()), Field.Store.YES, Field.Index.NO));
+ .getOrder()), StringField.TYPE_STORED));
doc.add(new Field(taskId + "_condition_timeout", String.valueOf(cond
- .getTimeoutSeconds()), Field.Store.YES, Field.Index.NO));
+ .getTimeoutSeconds()), StringField.TYPE_STORED));
doc.add(new Field(taskId + "_condition_optional", String.valueOf(cond.isOptional()),
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
}
}
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepositoryFactory.java
----------------------------------------------------------------------
diff --git a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepositoryFactory.java b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepositoryFactory.java
index b0dc070..d6b8466 100644
--- a/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepositoryFactory.java
+++ b/workflow/src/main/java/org/apache/oodt/cas/workflow/instrepo/LuceneWorkflowInstanceRepositoryFactory.java
@@ -20,14 +20,17 @@ package org.apache.oodt.cas.workflow.instrepo;
//JDK imports
import java.io.File;
+import java.io.IOException;
import java.util.logging.Logger;
//OODT imports
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.oodt.cas.metadata.util.PathUtils;
//Lucene imports
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.IndexWriter;
/**
* @author mattmann
@@ -72,12 +75,22 @@ public class LuceneWorkflowInstanceRepositoryFactory implements
* @see org.apache.oodt.cas.workflow.instrepo.WorkflowInstanceRepositoryFactory#createInstanceRepository()
*/
public WorkflowInstanceRepository createInstanceRepository() {
- File indexDir = new File(indexFilePath);
+ Directory indexDir = null;
+ try {
+ indexDir = FSDirectory.open(new File( indexFilePath ).toPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
// Create the index if it does not already exist
IndexWriter writer = null;
- if (!indexDir.exists()) {
- try {
- writer = new IndexWriter(indexDir, new StandardAnalyzer(), true);
+ try {
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ config.setMergePolicy(lmp);
+
+ writer = new IndexWriter(indexDir, config);
} catch (Exception e) {
LOG.severe("Unable to create index: " + e.getMessage());
} finally {
@@ -89,7 +102,7 @@ public class LuceneWorkflowInstanceRepositoryFactory implements
}
}
}
- }
+
return new LuceneWorkflowInstanceRepository(indexFilePath, pageSize);
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/java/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
----------------------------------------------------------------------
diff --git a/workflow/src/test/java/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java b/workflow/src/test/java/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
index e9f9f4e..3f0972c 100644
--- a/workflow/src/test/java/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
+++ b/workflow/src/test/java/org/apache/oodt/cas/workflow/tools/TestInstanceRepoCleaner.java
@@ -59,7 +59,7 @@ public class TestInstanceRepoCleaner extends TestCase {
WorkflowInstanceRepository repo = new LuceneWorkflowInstanceRepository(
instRepoPath, 20);
try {
- assertEquals(10, repo.getNumWorkflowInstances());
+ assertEquals(1, repo.getNumWorkflowInstances());
for (WorkflowInstance inst : (List<WorkflowInstance>) repo
.getWorkflowInstances()) {
if (!inst.getStatus().equals(WorkflowStatus.FINISHED)) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_4.cfe
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_4.cfe b/workflow/src/test/resources/testinstrepo/_4.cfe
new file mode 100644
index 0000000..c8dce11
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_4.cfe differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_4.cfs
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_4.cfs b/workflow/src/test/resources/testinstrepo/_4.cfs
new file mode 100644
index 0000000..ff2abfd
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_4.cfs differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_4.si
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_4.si b/workflow/src/test/resources/testinstrepo/_4.si
new file mode 100644
index 0000000..69be42e
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_4.si differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_43.cfs
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_43.cfs b/workflow/src/test/resources/testinstrepo/_43.cfs
deleted file mode 100644
index 88faee3..0000000
Binary files a/workflow/src/test/resources/testinstrepo/_43.cfs and /dev/null differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_5.cfe
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_5.cfe b/workflow/src/test/resources/testinstrepo/_5.cfe
new file mode 100644
index 0000000..753c12e
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_5.cfe differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_5.cfs
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_5.cfs b/workflow/src/test/resources/testinstrepo/_5.cfs
new file mode 100644
index 0000000..b93475a
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_5.cfs differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/_5.si
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/_5.si b/workflow/src/test/resources/testinstrepo/_5.si
new file mode 100644
index 0000000..44f3c04
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/_5.si differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/deletable
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/deletable b/workflow/src/test/resources/testinstrepo/deletable
deleted file mode 100644
index 593f470..0000000
Binary files a/workflow/src/test/resources/testinstrepo/deletable and /dev/null differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/pending_segments_b
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/pending_segments_b b/workflow/src/test/resources/testinstrepo/pending_segments_b
new file mode 100644
index 0000000..b4ddc68
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/pending_segments_b differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/segments
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/segments b/workflow/src/test/resources/testinstrepo/segments
deleted file mode 100644
index 9ff00d2..0000000
Binary files a/workflow/src/test/resources/testinstrepo/segments and /dev/null differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/segments_a
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/segments_a b/workflow/src/test/resources/testinstrepo/segments_a
new file mode 100644
index 0000000..517de04
Binary files /dev/null and b/workflow/src/test/resources/testinstrepo/segments_a differ
http://git-wip-us.apache.org/repos/asf/oodt/blob/95def41f/workflow/src/test/resources/testinstrepo/write.lock
----------------------------------------------------------------------
diff --git a/workflow/src/test/resources/testinstrepo/write.lock b/workflow/src/test/resources/testinstrepo/write.lock
new file mode 100644
index 0000000..e69de29