You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/03/09 12:35:34 UTC
oodt git commit: slight tweaks to fix build
Repository: oodt
Updated Branches:
refs/heads/curator [created] 9897fcff9
slight tweaks to fix build
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/9897fcff
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/9897fcff
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/9897fcff
Branch: refs/heads/curator
Commit: 9897fcff9f4e9efb7b25eb17b1ba9093e4f2b8c5
Parents: a47b088
Author: Tom Barber <to...@analytical-labs.com>
Authored: Thu Mar 9 12:35:19 2017 +0000
Committer: Tom Barber <to...@analytical-labs.com>
Committed: Thu Mar 9 12:35:19 2017 +0000
----------------------------------------------------------------------
core/pom.xml | 9 +-
curator2/pom.xml | 2 +-
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 540 ++++++++++---------
3 files changed, 281 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 35f3a70..514e51c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -70,6 +70,11 @@ the License.
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>6.4.2</version>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.4</version>
@@ -416,6 +421,7 @@ the License.
<version>6.2.1</version>
</dependency>
<dependency>
+
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>1.10</version>
@@ -619,7 +625,6 @@ mm
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9</version>
<configuration>
- <outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
<docencoding>UTF-8</docencoding>
<encoding>UTF-8</encoding>
@@ -662,7 +667,6 @@ mm
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version>
<configuration>
- <outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
<docencoding>UTF-8</docencoding>
<encoding>UTF-8</encoding>
@@ -972,7 +976,6 @@ mm
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.3</version>
<configuration>
- <outputEncoding>UTF-8</outputEncoding>
<charset>UTF-8</charset>
<docencoding>UTF-8</docencoding>
<encoding>UTF-8</encoding>
http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/curator2/pom.xml
----------------------------------------------------------------------
diff --git a/curator2/pom.xml b/curator2/pom.xml
index fa2017e..7cd7edd 100644
--- a/curator2/pom.xml
+++ b/curator2/pom.xml
@@ -20,7 +20,7 @@ the License.
<parent>
<groupId>org.apache.oodt</groupId>
<artifactId>oodt-core</artifactId>
- <version>0.13-SNAPSHOT</version>
+ <version>1.1-SNAPSHOT</version>
<relativePath>../core/pom.xml</relativePath>
</parent>
<artifactId>cas-curator-engine</artifactId>
http://git-wip-us.apache.org/repos/asf/oodt/blob/9897fcff/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index 6199a33..1372ae2 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -53,13 +53,13 @@ import java.util.logging.Logger;
* @author bfoster
* @author luca
* @version $Revision$
- *
+ *
* <p>
* An implementation of a File {@link Catalog} using Apache's popular <a
* href="http://lucene.apache.org">Lucene</a> text indexing engine as a
* backend.
* </p>
- *
+ *
*/
public class LuceneCatalog implements Catalog {
Directory indexDir = null;
@@ -98,7 +98,7 @@ public class LuceneCatalog implements Catalog {
/**
- *
+ *
* @param idxFilePath
* A file path pointing to the lucene index directory for this
* catalog.
@@ -107,22 +107,22 @@ public class LuceneCatalog implements Catalog {
* @param pgSize
* The size of pages to be used when doing pagination of the
* catalog.
- *
+ *
* @param commitTimeout
* The amount of time (in seconds) that should be flowed down to
* the Lucene IndexReader and IndexWriters for their commit lock
* timeout property.
- *
+ *
* @param writeTimeout
* The amount of time (in seconds) that should be flowed down to
* the Lucene IndexWriters for their commit lock timeout
* property.
- *
+ *
* @param mergeFactor
* The merge factor to use when writing to the index.
*/
public LuceneCatalog(String idxFilePath, ValidationLayer vLayer,
- int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) {
+ int pgSize, long commitTimeout, long writeTimeout, int mergeFactor) {
this.indexFilePath = idxFilePath;
this.valLayer = vLayer;
this.pageSize = pgSize;
@@ -144,47 +144,55 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#addMetadata(org.apache.oodt.cas.metadata.Metadata,
* org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void addMetadata(Metadata m, Product product)
- throws CatalogException {
+ throws CatalogException {
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
CompleteProduct p = CATALOG_CACHE.get(product.getProductId());
- p.setMetadata(m);
- if (hasMetadataAndRefs(p)) {
- LOG.log(Level.FINE,
- "metadata and references present for product: ["
+ p.setMetadata(m);
+ if (hasMetadataAndRefs(p)) {
+ LOG.log(Level.FINE,
+ "metadata and references present for product: ["
+ product.getProductId() + "]");
- addCompleteProductToIndex(p);
- // now remove its entry from the cache
- CATALOG_CACHE.remove(product.getProductId());
+ addCompleteProductToIndex(p);
+ // now remove its entry from the cache
+ CATALOG_CACHE.remove(product.getProductId());
}
}
else{
- // move product from index to cache
- // it will be moved back after metadata is added
- getCompleteProductById(product.getProductId(), true, true);
- LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
- removeProduct(product);
+ // move product from index to cache
+ // it will be moved back after metadata is added
+ CompleteProduct p = getCompleteProductById(product.getProductId(), true, true);
+ LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
+ removeProduct(product);
+ if (hasMetadataAndRefs(p)) {
+ LOG.log(Level.FINE,
+ "metadata and references present for product: ["
+ + product.getProductId() + "]");
+ p.setMetadata(m);
+ addCompleteProductToIndex(p);
+ // now remove its entry from the cache
+ }
}
}
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeMetadata(org.apache.oodt.cas.metadata.Metadata,
* org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void removeMetadata(Metadata m, Product product)
- throws CatalogException {
+ throws CatalogException {
CompleteProduct p;
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
- p = CATALOG_CACHE.get(product.getProductId());
+ p = CATALOG_CACHE.get(product.getProductId());
}
else{
String prodId = product.getProductId();
@@ -199,21 +207,21 @@ public class LuceneCatalog implements Catalog {
List<String> metadataTypes = new ArrayList<String>();
if (valLayer!=null) {
- try {
- // remove metadata elements specified by validation layer
- for (Element element : valLayer.getElements(product.getProductType())) {
- metadataTypes.add(element.getElementName());
- }
- } catch (ValidationLayerException e) {
- LOG.log(Level.SEVERE, e.getMessage());
- throw new CatalogException(
- "ValidationLayerException when trying to obtain element list for product type: "
- + product.getProductType().getName()
- + ": Message: " + e.getMessage(), e);
- }
+ try {
+ // remove metadata elements specified by validation layer
+ for (Element element : valLayer.getElements(product.getProductType())) {
+ metadataTypes.add(element.getElementName());
+ }
+ } catch (ValidationLayerException e) {
+ LOG.log(Level.SEVERE, e.getMessage());
+ throw new CatalogException(
+ "ValidationLayerException when trying to obtain element list for product type: "
+ + product.getProductType().getName()
+ + ": Message: " + e.getMessage(), e);
+ }
} else {
- // remove all metadata
- metadataTypes = currMet.getAllKeys();
+ // remove all metadata
+ metadataTypes = currMet.getAllKeys();
}
for (String name : metadataTypes) {
@@ -224,8 +232,8 @@ public class LuceneCatalog implements Catalog {
if (hasMetadataAndRefs(p)) {
LOG.log(Level.FINE,
- "metadata and references present for product: ["
- + product.getProductId() + "]");
+ "metadata and references present for product: ["
+ + product.getProductId() + "]");
addCompleteProductToIndex(p);
// now remove its entry from the cache
CATALOG_CACHE.remove(product.getProductId());
@@ -234,15 +242,15 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProduct(org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void addProduct(Product product)
- throws CatalogException {
+ throws CatalogException {
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
throw new CatalogException(
"Attempt to add a product that already existed: product: ["
- + product.getProductName() + "]");
+ + product.getProductName() + "]");
@@ -270,16 +278,16 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#modifyProduct(org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void modifyProduct(Product product)
- throws CatalogException {
+ throws CatalogException {
if (product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
LOG.log(Level.FINE, "Modifying product: [" + product.getProductId()
- + "]: found product in cache!");
+ + "]: found product in cache!");
CompleteProduct cp = CATALOG_CACHE.get(product
- .getProductId());
+ .getProductId());
cp.setProduct(product);
} else {
// need to grab the metadata for the existing product, and make sure
@@ -300,62 +308,62 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#removeProduct(org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void removeProduct(Product product)
- throws CatalogException {
+ throws CatalogException {
removeProductDocument(product);
}
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#setProductTransferStatus(org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void setProductTransferStatus(Product product)
- throws CatalogException {
+ throws CatalogException {
LOG.log(Level.FINE,
- "LuceneCatalog: seting product transfer status to: ["
- + product.getTransferStatus() + "] for " + "product: ["
- + product.getProductId() + "]");
+ "LuceneCatalog: seting product transfer status to: ["
+ + product.getTransferStatus() + "] for " + "product: ["
+ + product.getProductId() + "]");
modifyProduct(product);
}
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#addProductReferences(org.apache.oodt.cas.filemgr.structs.Product)
*/
public synchronized void addProductReferences(Product product)
- throws CatalogException {
+ throws CatalogException {
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
CompleteProduct p = CATALOG_CACHE.get(product
.getProductId());
p.getProduct().setProductReferences(product.getProductReferences());
- if (hasMetadataAndRefs(p)) {
- LOG.log(Level.FINE,
- "metadata and references present for product: ["
+ if (hasMetadataAndRefs(p)) {
+ LOG.log(Level.FINE,
+ "metadata and references present for product: ["
+ product.getProductId() + "]");
- addCompleteProductToIndex(p);
- // now remove its entry from the cache
- CATALOG_CACHE.remove(product.getProductId());
- }
+ addCompleteProductToIndex(p);
+ // now remove its entry from the cache
+ CATALOG_CACHE.remove(product.getProductId());
+ }
}
else{
- // move product from index to cache
- // it will be moved back after metadata is added
- getCompleteProductById(product.getProductId(), true, true);
- LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
- removeProduct(product);
+ // move product from index to cache
+ // it will be moved back after metadata is added
+ getCompleteProductById(product.getProductId(), true, true);
+ LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
+ removeProduct(product);
}
}
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductById(java.lang.String)
*/
public Product getProductById(String productId) throws CatalogException {
@@ -364,23 +372,23 @@ public class LuceneCatalog implements Catalog {
}
private Product getProductById(String productId, boolean getRefs)
- throws CatalogException {
+ throws CatalogException {
CompleteProduct prod = getCompleteProductById(productId, getRefs);
return prod.getProduct();
}
private CompleteProduct getCompleteProductById(String productId)
- throws CatalogException {
+ throws CatalogException {
return getCompleteProductById(productId, false);
}
private CompleteProduct getCompleteProductById(String productId,
- boolean getRefs) throws CatalogException {
+ boolean getRefs) throws CatalogException {
return getCompleteProductById(productId, getRefs, false);
}
private CompleteProduct getCompleteProductById(String productId,
- boolean getRefs, boolean getMet) throws CatalogException {
+ boolean getRefs, boolean getMet) throws CatalogException {
IndexSearcher searcher = null;
try {
try {
@@ -398,7 +406,7 @@ public class LuceneCatalog implements Catalog {
// should be exactly 1 hit
if (topDocs.totalHits == 0) {
- throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!");
+ throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!");
}
if (topDocs.totalHits > 1) {
throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!");
@@ -406,12 +414,12 @@ public class LuceneCatalog implements Catalog {
Document productDoc = searcher.doc(hits[0].doc);
return toCompleteProduct(productDoc, getRefs,
- getMet);
+ getMet);
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
@@ -425,7 +433,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductByName(java.lang.String)
*/
public Product getProductByName(String productName) throws CatalogException {
@@ -433,7 +441,7 @@ public class LuceneCatalog implements Catalog {
}
private Product getProductByName(String productName, boolean getRefs)
- throws CatalogException {
+ throws CatalogException {
IndexSearcher searcher = null;
try {
try {
@@ -445,7 +453,7 @@ public class LuceneCatalog implements Catalog {
Term productIdTerm = new Term("product_name", productName);
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.Type.STRING, true));
+ SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
if(check.totalHits>0) {
@@ -458,11 +466,11 @@ public class LuceneCatalog implements Catalog {
// just get the first hit back
Document productDoc = searcher.doc(hits[0].doc);
CompleteProduct prod = toCompleteProduct(productDoc, getRefs,
- false);
+ false);
return prod.getProduct();
} else {
LOG.log(Level.FINEST, "Request for product by name: ["
- + productName + "] returned no results");
+ + productName + "] returned no results");
return null;
}
}
@@ -472,9 +480,9 @@ public class LuceneCatalog implements Catalog {
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
@@ -488,7 +496,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductReferences(org.apache.oodt.cas.filemgr.structs.Product)
*/
public List<Reference> getProductReferences(Product product) throws CatalogException {
@@ -502,7 +510,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProducts()
*/
public List<Product> getProducts() throws CatalogException {
@@ -523,7 +531,7 @@ public class LuceneCatalog implements Catalog {
Term productIdTerm = new Term("myfield", "myvalue");
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.Type.STRING, true));
+ SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
TopDocs topDocs = searcher.search(query, check.totalHits, sort);
@@ -536,20 +544,20 @@ public class LuceneCatalog implements Catalog {
for (ScoreDoc hit : hits) {
Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
- getRefs, false);
+ getRefs, false);
products.add(prod.getProduct());
}
} else {
LOG.log(Level.FINEST,
- "Request for products returned no results");
+ "Request for products returned no results");
return null;
}
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
@@ -565,16 +573,16 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getProductsByProductType(org.apache.oodt.cas.filemgr.structs.ProductType)
*/
public List<Product> getProductsByProductType(ProductType type)
- throws CatalogException {
+ throws CatalogException {
return getProductsByProductType(type, false);
}
private List<Product> getProductsByProductType(ProductType type, boolean getRefs)
- throws CatalogException {
+ throws CatalogException {
IndexSearcher searcher = null;
List<Product> products = null;
@@ -586,10 +594,10 @@ public class LuceneCatalog implements Catalog {
}
searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_type_id", type
- .getProductTypeId());
+ .getProductTypeId());
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.Type.STRING, true));
+ SortField.Type.STRING, true));
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(query, 1, sort);
TopDocs topDocs = searcher.search(query, check.totalHits, sort);
@@ -602,26 +610,26 @@ public class LuceneCatalog implements Catalog {
for (ScoreDoc hit : hits) {
Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
- getRefs, false);
+ getRefs, false);
products.add(prod.getProduct());
}
} else {
LOG.log(Level.FINEST, "Request for products by type: ["
- + type.getProductTypeId() + "] returned no results");
+ + type.getProductTypeId() + "] returned no results");
return null;
}
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
try {
//TODO CLOSE
- } catch (Exception ignore) {
+ } catch (Exception ignore) {
}
}
}
@@ -650,9 +658,9 @@ public class LuceneCatalog implements Catalog {
// should be exactly 1 hit
if (topDocs.totalHits != 1) {
throw new CatalogException("Product: ["
- + product.getProductId()
- + "] is not unique in the catalog! Num Hits: ["
- + hits.length + "]");
+ + product.getProductId()
+ + "] is not unique in the catalog! Num Hits: ["
+ + hits.length + "]");
}
Document productDoc = searcher.doc(hits[0].doc);
@@ -661,20 +669,20 @@ public class LuceneCatalog implements Catalog {
return prod.getMetadata();
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
try {
//TODO CLOSE
- } catch (Exception ignore) {
+ } catch (Exception ignore) {
}
}
}
}
-
+
public Metadata getReducedMetadata(Product product, List<String> elements) throws CatalogException {
Metadata fullMetadata = getMetadata(product);
Metadata reducedMetadata = new Metadata();
@@ -688,7 +696,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#query(org.apache.oodt.cas.filemgr.structs.Query,
* org.apache.oodt.cas.filemgr.structs.ProductType)
*/
@@ -711,7 +719,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int)
*/
public List<Product> getTopNProducts(int n) throws CatalogException {
@@ -732,9 +740,9 @@ public class LuceneCatalog implements Catalog {
booleanQuery.add(tq, BooleanClause.Occur.MUST);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.Type.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
- + "]");
+ + "]");
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(booleanQuery.build(), 1, sort);
if(check.totalHits>0) {
@@ -748,7 +756,7 @@ public class LuceneCatalog implements Catalog {
while (products.size() < Math.min(n, hits.length)) {
Document productDoc = searcher.doc(hits[i].doc);
CompleteProduct prod = toCompleteProduct(productDoc, false,
- false);
+ false);
products.add(prod.getProduct());
i++;
}
@@ -762,9 +770,9 @@ public class LuceneCatalog implements Catalog {
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (searcher != null) {
@@ -780,12 +788,12 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getTopNProducts(int,
* org.apache.oodt.cas.filemgr.structs.ProductType)
*/
public List<Product> getTopNProducts(int n, ProductType type)
- throws CatalogException {
+ throws CatalogException {
int numPages = 1;
if (n > this.pageSize) {
numPages = n / this.pageSize + (n % this.pageSize == 0 ? 0 : 1);
@@ -796,11 +804,11 @@ public class LuceneCatalog implements Catalog {
for (int pageNum = 1; pageNum < numPages + 1; pageNum++) {
List<Product> pageProducts = paginateQuery(query, type, pageNum, null);
- products.addAll(pageProducts);
+ products.addAll(pageProducts);
}
if(n<=products.size()) {
- return products.subList(0, n);
+ return products.subList(0, n);
}
return products;
@@ -808,7 +816,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getValidationLayer()
*/
public ValidationLayer getValidationLayer() {
@@ -817,7 +825,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#getNumProducts(org.apache.oodt.cas.filemgr.structs.ProductType)
*/
public int getNumProducts(ProductType type) throws CatalogException {
@@ -827,29 +835,29 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.util.Pagination#getFirstPage(org.apache.oodt.cas.filemgr.structs.ProductType)
*/
public ProductPage getFirstPage(ProductType type) {
ProductPage firstPage = new ProductPage();
List<Product> products;
Query query = new Query();
-
+
// now construct the page
firstPage.setPageNum(1);
firstPage.setPageSize(pageSize);
try {
- products = paginateQuery(query, type, 1, firstPage);
+ products = paginateQuery(query, type, 1, firstPage);
} catch (CatalogException e) {
LOG.log(Level.WARNING,
- "CatalogException getting first page for product type: ["
- + type.getProductTypeId()
- + "] from catalog: Message: " + e.getMessage());
+ "CatalogException getting first page for product type: ["
+ + type.getProductTypeId()
+ + "] from catalog: Message: " + e.getMessage());
return null;
}
// There are no products and thus no first page
if (products == null || (products.size() == 0)) {
- return null;
+ return null;
}
firstPage.setPageProducts(products);
@@ -859,7 +867,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.util.Pagination#getLastProductPage(org.apache.oodt.cas.filemgr.structs.ProductType)
*/
public ProductPage getLastProductPage(ProductType type) {
@@ -867,22 +875,22 @@ public class LuceneCatalog implements Catalog {
ProductPage firstPage = getFirstPage(type);
List<Product> products;
Query query = new Query();
-
+
// now construct the page
lastPage.setPageNum(firstPage.getTotalPages());
lastPage.setPageSize(pageSize);
try {
products = paginateQuery(query, type, firstPage.getTotalPages(), lastPage);
} catch (CatalogException e) {
- LOG.log(Level.WARNING,
- "CatalogException getting last page for product type: ["
- + type.getProductTypeId()
- + "] from catalog: Message: " + e.getMessage());
- return null;
+ LOG.log(Level.WARNING,
+ "CatalogException getting last page for product type: ["
+ + type.getProductTypeId()
+ + "] from catalog: Message: " + e.getMessage());
+ return null;
}
// There are no products thus there is no last page
if (products == null || (products.size() == 0)) {
- return null;
+ return null;
}
lastPage.setPageProducts(products);
@@ -891,7 +899,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.util.Pagination#getNextPage(org.apache.oodt.cas.filemgr.structs.ProductType,
* org.apache.oodt.cas.filemgr.structs.ProductPage)
*/
@@ -918,14 +926,14 @@ public class LuceneCatalog implements Catalog {
products = paginateQuery(query, type, currentPage.getPageNum() + 1, nextPage);
} catch (CatalogException e) {
LOG.log(Level.WARNING,
- "CatalogException getting next page for product type: ["
- + type.getProductTypeId()
- + "] from catalog: Message: " + e.getMessage());
+ "CatalogException getting next page for product type: ["
+ + type.getProductTypeId()
+ + "] from catalog: Message: " + e.getMessage());
return null;
}
// There are no products and thus no next page
if (products == null || (products.size() == 0)) {
- return null;
+ return null;
}
nextPage.setPageProducts(products);
@@ -934,7 +942,7 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.util.Pagination#getPrevPage(org.apache.oodt.cas.filemgr.structs.ProductType,
* org.apache.oodt.cas.filemgr.structs.ProductPage)
*/
@@ -962,15 +970,15 @@ public class LuceneCatalog implements Catalog {
products = paginateQuery(query, type, currentPage.getPageNum() - 1, prevPage);
} catch (CatalogException e) {
LOG.log(Level.WARNING,
- "CatalogException getting prev page for product type: ["
- + type.getProductTypeId()
- + "] from catalog: Message: " + e.getMessage());
+ "CatalogException getting prev page for product type: ["
+ + type.getProductTypeId()
+ + "] from catalog: Message: " + e.getMessage());
return null;
}
-
+
// There are no products and thus no pages
if (products == null || (products.size() == 0)) {
- return null;
+ return null;
}
prevPage.setPageProducts(products);
@@ -979,12 +987,12 @@ public class LuceneCatalog implements Catalog {
/*
* (non-Javadoc)
- *
+ *
* @see org.apache.oodt.cas.filemgr.catalog.Catalog#pagedQuery(org.apache.oodt.cas.filemgr.structs.Query,
* org.apache.oodt.cas.filemgr.structs.ProductType, int)
*/
public ProductPage pagedQuery(Query query, ProductType type, int pageNum)
- throws CatalogException {
+ throws CatalogException {
try {
ProductPage retPage = new ProductPage();
retPage.setPageNum(pageNum);
@@ -994,15 +1002,15 @@ public class LuceneCatalog implements Catalog {
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING,
- "CatalogException when doing paged product query: Message: "
- + e.getMessage());
+ "CatalogException when doing paged product query: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
}
}
private synchronized void removeProductDocument(Product product)
- throws CatalogException {
+ throws CatalogException {
try {
reader = DirectoryReader.open(indexDir);
@@ -1011,8 +1019,8 @@ public class LuceneCatalog implements Catalog {
}
try {
LOG.log(Level.FINE,
- "LuceneCatalog: remove document from index for product: ["
- + product.getProductId() + "]");
+ "LuceneCatalog: remove document from index for product: ["
+ + product.getProductId() + "]");
IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
@@ -1020,15 +1028,15 @@ public class LuceneCatalog implements Catalog {
lmp.setMergeFactor(mergeFactor);
config.setMergePolicy(lmp);
- IndexWriter writer = new IndexWriter(indexDir, config);
+ IndexWriter writer = new IndexWriter(indexDir, config);
writer.deleteDocuments(new Term("product_id", product
- .getProductId()));
+ .getProductId()));
writer.close();
} catch (IOException e) {
LOG.log(Level.WARNING, "Exception removing product: ["
- + product.getProductName() + "] from index: Message: "
- + e.getMessage());
+ + product.getProductName() + "] from index: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage(), e);
} finally {
if (reader != null) {
@@ -1043,7 +1051,7 @@ public class LuceneCatalog implements Catalog {
}
private synchronized void addCompleteProductToIndex(CompleteProduct cp)
- throws CatalogException {
+ throws CatalogException {
IndexWriter writer = null;
try {
/*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(),
@@ -1064,11 +1072,11 @@ public class LuceneCatalog implements Catalog {
// TODO: determine a better way to optimize the index
} catch (Exception e) {
LOG.log(Level.WARNING, "Unable to index product: ["
- + cp.getProduct().getProductName() + "]: Message: "
- + e.getMessage(), e);
+ + cp.getProduct().getProductName() + "]: Message: "
+ + e.getMessage(), e);
throw new CatalogException("Unable to index product: ["
- + cp.getProduct().getProductName() + "]: Message: "
- + e.getMessage(), e);
+ + cp.getProduct().getProductName() + "]: Message: "
+ + e.getMessage(), e);
} finally {
try {
if (writer != null) {
@@ -1086,7 +1094,7 @@ public class LuceneCatalog implements Catalog {
}
private CompleteProduct toCompleteProduct(Document doc, boolean getRefs,
- boolean getMetadata) {
+ boolean getMetadata) {
Product product = new Product();
Metadata metadata = new Metadata();
CompleteProduct completeProduct = new CompleteProduct();
@@ -1108,39 +1116,39 @@ public class LuceneCatalog implements Catalog {
List<String> names = new ArrayList<String>();
if (valLayer!=null) {
- // only add metadata elements specified by validation layer
- try {
- for (Element element : valLayer.getElements(type)) {
- names.add(element.getElementName());
- }
- } catch (ValidationLayerException e) {
- LOG.log(Level.WARNING,
- "Unable to obtain metadata for product: ["
- + product.getProductName() + "]: Message: "
- + e.getMessage());
- }
+ // only add metadata elements specified by validation layer
+ try {
+ for (Element element : valLayer.getElements(type)) {
+ names.add(element.getElementName());
+ }
+ } catch (ValidationLayerException e) {
+ LOG.log(Level.WARNING,
+ "Unable to obtain metadata for product: ["
+ + product.getProductName() + "]: Message: "
+ + e.getMessage());
+ }
} else {
- // add all metadata elements found in document
- List<IndexableField> fields = doc.getFields();
+ // add all metadata elements found in document
+ List<IndexableField> fields = doc.getFields();
for(IndexableField field: fields){
if (!names.contains(field.name())) {
names.add(field.name());
}
}
-
+
}
// loop over field names to add to metadata
for (String name : names) {
- if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) {
- String[] elemValues = doc.getValues(name);
-
- if (elemValues != null && elemValues.length > 0) {
+ if (metadata.getAllMetadata(name)==null || metadata.getAllMetadata(name).size()==0) {
+ String[] elemValues = doc.getValues(name);
+
+ if (elemValues != null && elemValues.length > 0) {
for (String elemValue : elemValues) {
metadata.addMetadata(name, elemValue);
}
- }
- }
+ }
+ }
}
completeProduct.setMetadata(metadata);
@@ -1154,7 +1162,7 @@ public class LuceneCatalog implements Catalog {
String[] refMimeTypes = doc.getValues("reference_mimeType");
if ((origRefs.length == dataStoreRefs.length)
- && (origRefs.length == refLengths.length)) {
+ && (origRefs.length == refLengths.length)) {
List<Reference> references = new Vector<Reference>();
for (int i = 0; i < origRefs.length; i++) {
Reference r = new Reference();
@@ -1170,11 +1178,11 @@ public class LuceneCatalog implements Catalog {
product.setProductReferences(references);
} else {
LOG.log(Level.WARNING, "Number of original refs: ["
- + origRefs.length + "] for product: ["
- + product.getProductName()
- + "] not equivalent to number of data store refs: ["
- + dataStoreRefs.length
- + "]: Skipping product references");
+ + origRefs.length + "] for product: ["
+ + product.getProductName()
+ + "] not equivalent to number of data store refs: ["
+ + dataStoreRefs.length
+ + "]: Skipping product references");
}
}
@@ -1187,63 +1195,63 @@ public class LuceneCatalog implements Catalog {
//TODO CHECK STORED TYPES
// add the product information
doc.add(new Field("product_id", product.getProductId(),
- StringField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc.add(new Field("product_name", product.getProductName(),
- StringField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc.add(new Field("product_structure", product.getProductStructure(),
- StringField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc
- .add(new Field("product_transfer_status", product
- .getTransferStatus(), StringField.TYPE_STORED));
+ .add(new Field("product_transfer_status", product
+ .getTransferStatus(), StringField.TYPE_STORED));
// product type
doc
- .add(new Field("product_type_id", product.getProductType()
- .getProductTypeId(), StringField.TYPE_STORED));
+ .add(new Field("product_type_id", product.getProductType()
+ .getProductTypeId(), StringField.TYPE_STORED));
doc.add(new Field("product_type_name", product.getProductType()
- .getName(), StringField.TYPE_STORED));
+ .getName(), StringField.TYPE_STORED));
doc.add(new Field("product_type_desc", product.getProductType()
- .getDescription() != null ? product.getProductType()
- .getDescription() : "", StringField.TYPE_STORED));
+ .getDescription() != null ? product.getProductType()
+ .getDescription() : "", StringField.TYPE_STORED));
doc.add(new Field("product_type_repoPath", product.getProductType()
- .getProductRepositoryPath() != null ? product.getProductType()
- .getProductRepositoryPath() : "", StringField.TYPE_STORED));
+ .getProductRepositoryPath() != null ? product.getProductType()
+ .getProductRepositoryPath() : "", StringField.TYPE_STORED));
doc.add(new Field("product_type_versioner", product.getProductType()
- .getVersioner() != null ? product.getProductType()
- .getVersioner() : "", StringField.TYPE_STORED));
-
+ .getVersioner() != null ? product.getProductType()
+ .getVersioner() : "", StringField.TYPE_STORED));
+
// write metadata fields to the Lucene document
List<String> keys = new ArrayList<String>();
// validation layer: add only specifically configured keys
if (valLayer!=null) {
- List<Element> elements = quietGetElements(product.getProductType());
+ List<Element> elements = quietGetElements(product.getProductType());
for (Element element : elements) {
String key = element.getElementName();
keys.add(key);
}
- // no validation layer: add all keys that are NOT already in doc
- // (otherwise some keys such as the product_* keys are duplicated)
+ // no validation layer: add all keys that are NOT already in doc
+ // (otherwise some keys such as the product_* keys are duplicated)
} else {
- for (String key : metadata.getAllKeys()) {
- if (doc.getField(key)==null) {
- keys.add(key);
- }
- }
+ for (String key : metadata.getAllKeys()) {
+ if (doc.getField(key)==null) {
+ keys.add(key);
+ }
+ }
}
for (String key : keys) {
- List<String> values = metadata.getAllMetadata(key);
+ List<String> values = metadata.getAllMetadata(key);
if (values == null) {
LOG
- .log(
- Level.WARNING,
- "No Metadata specified for product ["
- + product.getProductName()
- + "] for required field ["
- + key
- + "]: Attempting to continue processing metadata");
+ .log(
+ Level.WARNING,
+ "No Metadata specified for product ["
+ + product.getProductName()
+ + "] for required field ["
+ + key
+ + "]: Attempting to continue processing metadata");
continue;
}
@@ -1258,7 +1266,7 @@ public class LuceneCatalog implements Catalog {
// add the product references
for (Reference r : product.getProductReferences()) {
doc.add(new Field("reference_orig", r.getOrigReference(),
- StringField.TYPE_STORED));
+ StringField.TYPE_STORED));
doc
.add(new Field("reference_data_store", r
.getDataStoreReference(), StringField.TYPE_STORED));
@@ -1296,7 +1304,7 @@ public class LuceneCatalog implements Catalog {
}
private int getNumHits(Query query, ProductType type)
- throws CatalogException {
+ throws CatalogException {
IndexSearcher searcher = null;
int numHits = -1;
@@ -1313,7 +1321,7 @@ public class LuceneCatalog implements Catalog {
// add the product type as the first clause
org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term(
- "product_type_id", type.getProductTypeId()));
+ "product_type_id", type.getProductTypeId()));
booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST);
//convert filemgr query into a lucene query
@@ -1322,7 +1330,7 @@ public class LuceneCatalog implements Catalog {
}
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
- + "]");
+ + "]");
//TODO FIX returned records
TopDocs hits = searcher.search(booleanQuery.build(), 1);
@@ -1331,9 +1339,9 @@ public class LuceneCatalog implements Catalog {
numHits = hits.totalHits;
} catch (IOException e) {
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage());
} finally {
if (searcher != null) {
@@ -1348,7 +1356,7 @@ public class LuceneCatalog implements Catalog {
}
private List<Product> paginateQuery(Query query, ProductType type, int pageNum, ProductPage page)
- throws CatalogException {
+ throws CatalogException {
List<Product> products = null;
IndexSearcher searcher = null;
@@ -1370,25 +1378,25 @@ public class LuceneCatalog implements Catalog {
// add the product type as the first clause
TermQuery prodTypeTermQuery = new TermQuery(new Term(
- "product_type_id", type.getProductTypeId()));
+ "product_type_id", type.getProductTypeId()));
booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST);
-
+
//convert filemgr query into a lucene query
for (QueryCriteria queryCriteria : query.getCriteria()) {
booleanQuery.add(this.getQuery(queryCriteria), BooleanClause.Occur.MUST);
}
-
+
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.Type.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
- + "]");
+ + "]");
//TODO FIX NUMBER OF RECORDS
TopDocs check = searcher.search(booleanQuery.build(),1, sort);
TopDocs topDocs = searcher.search(booleanQuery.build(),check.totalHits, sort);
// Calculate page size and set it while we have the results
if (page != null) {
- page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
+ page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
}
ScoreDoc[] hits = topDocs.scoreDocs;
@@ -1404,11 +1412,11 @@ public class LuceneCatalog implements Catalog {
products = new Vector<Product>(pageSize);
for (int i = startNum; i < Math.min(hits.length,
- (startNum + pageSize)); i++) {
+ (startNum + pageSize)); i++) {
Document productDoc = searcher.doc(hits[i].doc);
CompleteProduct prod = toCompleteProduct(productDoc,
- false, false);
+ false, false);
products.add(prod.getProduct());
}
} else {
@@ -1417,22 +1425,22 @@ public class LuceneCatalog implements Catalog {
Document productDoc = searcher.doc(hits[i].doc);
CompleteProduct prod = toCompleteProduct(productDoc,
- false, false);
+ false, false);
products.add(prod.getProduct());
}
}
} else {
LOG.log(Level.WARNING, "Query: [" + query
- + "] for Product Type: [" + type.getProductTypeId()
- + "] returned no results");
+ + "] for Product Type: [" + type.getProductTypeId()
+ + "] returned no results");
}
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage());
LOG.log(Level.WARNING,
- "IOException when opening index directory: ["
- + indexFilePath + "] for search: Message: "
- + e.getMessage());
+ "IOException when opening index directory: ["
+ + indexFilePath + "] for search: Message: "
+ + e.getMessage());
throw new CatalogException(e.getMessage());
} finally {
if (searcher != null) {
@@ -1451,19 +1459,19 @@ public class LuceneCatalog implements Catalog {
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
BooleanClause.Occur occur;
switch (((BooleanQueryCriteria) queryCriteria).getOperator()) {
- case BooleanQueryCriteria.AND:
- occur = BooleanClause.Occur.MUST;
- break;
- case BooleanQueryCriteria.OR:
- occur = BooleanClause.Occur.SHOULD;
- break;
- case BooleanQueryCriteria.NOT:
- occur = BooleanClause.Occur.MUST_NOT;
- booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria)
+ case BooleanQueryCriteria.AND:
+ occur = BooleanClause.Occur.MUST;
+ break;
+ case BooleanQueryCriteria.OR:
+ occur = BooleanClause.Occur.SHOULD;
+ break;
+ case BooleanQueryCriteria.NOT:
+ occur = BooleanClause.Occur.MUST_NOT;
+ booleanQuery.add(new WildcardQuery(new Term(((BooleanQueryCriteria) queryCriteria)
.getTerms().get(0).getElementName(), "*")), BooleanClause.Occur.SHOULD);
- break;
- default:
- throw new CatalogException("Invalid BooleanQueryCriteria opertor ["
+ break;
+ default:
+ throw new CatalogException("Invalid BooleanQueryCriteria opertor ["
+ ((BooleanQueryCriteria) queryCriteria).getOperator() + "]");
}
for (QueryCriteria qc : ((BooleanQueryCriteria) queryCriteria).getTerms()) {
@@ -1486,10 +1494,10 @@ public class LuceneCatalog implements Catalog {
return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive);
}else {
throw new CatalogException("Invalid QueryCriteria ["
- + queryCriteria.getClass().getCanonicalName() + "]");
+ + queryCriteria.getClass().getCanonicalName() + "]");
}
}
-
+
private List<Element> quietGetElements(ProductType type) {
List<Element> elementList = new Vector<Element>();
@@ -1497,8 +1505,8 @@ public class LuceneCatalog implements Catalog {
elementList = valLayer.getElements(type);
} catch (Exception e) {
LOG.log(Level.WARNING,
- "Exception obtaining elements for product type: ["
- + type.getName() + "]: Message: " + e.getMessage());
+ "Exception obtaining elements for product type: ["
+ + type.getName() + "]: Message: " + e.getMessage());
}
return elementList;
@@ -1565,4 +1573,4 @@ public class LuceneCatalog implements Catalog {
}
-}
\ No newline at end of file
+}