You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by ma...@apache.org on 2017/10/12 17:57:41 UTC
[17/50] [abbrv] oodt git commit: lucene 6.1.0 upgrade
lucene 6.1.0 upgrade
Project: http://git-wip-us.apache.org/repos/asf/oodt/repo
Commit: http://git-wip-us.apache.org/repos/asf/oodt/commit/999e68dd
Tree: http://git-wip-us.apache.org/repos/asf/oodt/tree/999e68dd
Diff: http://git-wip-us.apache.org/repos/asf/oodt/diff/999e68dd
Branch: refs/heads/master
Commit: 999e68ddbed773aaeffe4a912288915a9eb4ab9c
Parents: 3939d0a
Author: Tom Barber <to...@analytical-labs.com>
Authored: Fri Jul 8 21:46:35 2016 +0100
Committer: Lewis John McGibbney <le...@gmail.com>
Committed: Thu Mar 9 21:34:33 2017 -0800
----------------------------------------------------------------------
core/pom.xml | 14 +-
filemgr/pom.xml | 11 +
.../oodt/cas/filemgr/catalog/LuceneCatalog.java | 373 +++++++++++--------
.../filemgr/catalog/LuceneCatalogFactory.java | 18 +-
.../cli/action/LuceneQueryCliAction.java | 25 +-
.../oodt/cas/filemgr/tools/CASAnalyzer.java | 36 +-
.../oodt/cas/filemgr/tools/CatalogSearch.java | 26 +-
.../filemgr/tools/OptimizeLuceneCatalog.java | 24 +-
.../oodt/cas/filemgr/tools/QueryTool.java | 23 +-
.../cas/filemgr/tools/RangeQueryTester.java | 50 +--
10 files changed, 374 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index ee8cb7e..e3ff15a 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -280,9 +280,21 @@ the License.
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
- <version>2.0.0</version>
+ <version>6.1.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>6.1.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>6.1.0</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
<version>${project.parent.version}</version>
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/pom.xml
----------------------------------------------------------------------
diff --git a/filemgr/pom.xml b/filemgr/pom.xml
index 6e102dc..54b4e31 100644
--- a/filemgr/pom.xml
+++ b/filemgr/pom.xml
@@ -125,6 +125,15 @@
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.oodt</groupId>
<artifactId>cas-cli</artifactId>
</dependency>
@@ -275,6 +284,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
+ <forkMode>never</forkMode>
+ <!--forkMode>pertest</forkMode-->
<forkMode>pertest</forkMode>
<!-- so we can DEBUG -->
<argLine>-enableassertions</argLine>
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
index 26bb983..c8936bf 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalog.java
@@ -17,44 +17,29 @@
package org.apache.oodt.cas.filemgr.catalog;
-//JDK imports
-
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.WildcardQuery;
-import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.Element;
-import org.apache.oodt.cas.filemgr.structs.Product;
-import org.apache.oodt.cas.filemgr.structs.ProductPage;
-import org.apache.oodt.cas.filemgr.structs.ProductType;
+import org.apache.lucene.document.*;
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.Version;
+import org.apache.oodt.cas.filemgr.structs.*;
import org.apache.oodt.cas.filemgr.structs.Query;
-import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
-import org.apache.oodt.cas.filemgr.structs.Reference;
-import org.apache.oodt.cas.filemgr.structs.TermQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.exceptions.CatalogException;
import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException;
import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
import org.apache.oodt.cas.metadata.Metadata;
import org.apache.oodt.commons.pagination.PaginationUtils;
-
+import org.apache.poi.hssf.record.formula.functions.Text;
+import org.apache.solr.schema.FieldType;
import org.safehaus.uuid.UUID;
import org.safehaus.uuid.UUIDGenerator;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
@@ -63,10 +48,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
-//Lucene imports
-//OODT imports
-//JUG imports
-
/**
* @author mattmann
* @author bfoster
@@ -81,7 +62,9 @@ import java.util.logging.Logger;
*
*/
public class LuceneCatalog implements Catalog {
+ Directory indexDir = null;
+ private DirectoryReader reader;
/* the path to the index directory for this catalog */
private String indexFilePath = null;
@@ -92,7 +75,7 @@ public class LuceneCatalog implements Catalog {
* temporary Cache of product/metadata/reference information before it is
* written to the index
*/
- private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<String, CompleteProduct>();
+ private static ConcurrentHashMap<String, CompleteProduct> CATALOG_CACHE = new ConcurrentHashMap<>();
/* our product ID generator */
private static UUIDGenerator generator = UUIDGenerator.getInstance();
@@ -112,6 +95,8 @@ public class LuceneCatalog implements Catalog {
/* lucene index merge factor */
private int mergeFactor = -1;
+
+
/**
*
* @param idxFilePath
@@ -144,6 +129,17 @@ public class LuceneCatalog implements Catalog {
this.writeLockTimeout = writeTimeout;
this.commitLockTimeout = commitTimeout;
this.mergeFactor = mergeFactor;
+
+ try {
+ indexDir = FSDirectory.open(new File( indexFilePath ).toPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+
+
+
+
}
/*
@@ -170,7 +166,7 @@ public class LuceneCatalog implements Catalog {
else{
// move product from index to cache
// it will be moved back after metadata is added
- CompleteProduct p = getCompleteProductById(product.getProductId(), true, true);
+ getCompleteProductById(product.getProductId(), true, true);
LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
removeProduct(product);
@@ -185,7 +181,7 @@ public class LuceneCatalog implements Catalog {
*/
public synchronized void removeMetadata(Metadata m, Product product)
throws CatalogException {
- CompleteProduct p=null;
+ CompleteProduct p;
if(product.getProductId()!=null && CATALOG_CACHE.containsKey(product.getProductId())) {
p = CATALOG_CACHE.get(product.getProductId());
@@ -350,7 +346,7 @@ public class LuceneCatalog implements Catalog {
else{
// move product from index to cache
// it will be moved back after metadata is added
- CompleteProduct p = getCompleteProductById(product.getProductId(), true, true);
+ getCompleteProductById(product.getProductId(), true, true);
LOG.log(Level.FINE, "Product not found in local cache, retrieved from index");
removeProduct(product);
@@ -387,19 +383,28 @@ public class LuceneCatalog implements Catalog {
boolean getRefs, boolean getMet) throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_id", productId);
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
- Hits hits = searcher.search(query);
+ TopDocs topDocs = searcher.search(query,1);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be exactly 1 hit
- if (hits.length() == 0) {
+ if (topDocs.totalHits == 0) {
throw new CatalogException("Product: [" + productId + "] NOT found in the catalog!");
- } else if (hits.length() > 1) {
+ }
+ if (topDocs.totalHits > 1) {
throw new CatalogException("Product: [" + productId+ "] is not unique in the catalog!");
}
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
return toCompleteProduct(productDoc, getRefs,
getMet);
} catch (IOException e) {
@@ -411,7 +416,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO shutdown reader
} catch (Exception ignore) {
}
}
@@ -431,17 +436,25 @@ public class LuceneCatalog implements Catalog {
throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_name", productName);
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
+ if (hits.length > 0) {
// just get the first hit back
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
CompleteProduct prod = toCompleteProduct(productDoc, getRefs,
false);
return prod.getProduct();
@@ -460,7 +473,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+//TODO CLOSE SEARCHER
} catch (Exception ignore) {
}
}
@@ -495,18 +508,26 @@ public class LuceneCatalog implements Catalog {
List<Product> products = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("myfield", "myvalue");
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
products.add(prod.getProduct());
@@ -526,7 +547,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO close searcher
} catch (Exception ignore) {
}
}
@@ -551,19 +572,27 @@ public class LuceneCatalog implements Catalog {
List<Product> products = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
Term productIdTerm = new Term("product_type_id", type
.getProductTypeId());
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(query, sort);
+ SortField.Type.STRING, true));
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be > 0 hits
- if (hits.length() > 0) {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ if (hits.length > 0) {
+ products = new Vector<Product>(hits.length);
+ for (ScoreDoc hit : hits) {
+ Document productDoc = searcher.doc(hit.doc);
CompleteProduct prod = toCompleteProduct(productDoc,
getRefs, false);
products.add(prod.getProduct());
@@ -583,8 +612,8 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
- } catch (Exception ignore) {
+//TODO CLOSE
+ } catch (Exception ignore) {
}
}
}
@@ -595,20 +624,31 @@ public class LuceneCatalog implements Catalog {
public Metadata getMetadata(Product product) throws CatalogException {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
+ TermQuery qry = new TermQuery(new Term("*", "*"));
+ TopDocs tdocks = searcher.search(qry, 100);
Term productIdTerm = new Term("product_id", product.getProductId());
org.apache.lucene.search.Query query = new TermQuery(productIdTerm);
- Hits hits = searcher.search(query);
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(query, 1);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
// should be exactly 1 hit
- if (hits.length() != 1) {
+ if (topDocs.totalHits != 1) {
throw new CatalogException("Product: ["
+ product.getProductId()
+ "] is not unique in the catalog! Num Hits: ["
- + hits.length() + "]");
+ + hits.length + "]");
}
- Document productDoc = hits.doc(0);
+ Document productDoc = searcher.doc(hits[0].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc, false, true);
return prod.getMetadata();
} catch (IOException e) {
@@ -620,8 +660,8 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
- } catch (Exception ignore) {
+//TODO CLOSE
+ } catch (Exception ignore) {
}
}
}
@@ -671,23 +711,32 @@ public class LuceneCatalog implements Catalog {
IndexSearcher searcher = null;
try {
- searcher = new IndexSearcher(indexFilePath);
+ try {
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
TermQuery tq = new TermQuery(new Term("myfield", "myvalue"));
booleanQuery.add(tq, BooleanClause.Occur.MUST);
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery, sort);
- if (hits.length() > 0) {
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(booleanQuery.build(), 1, sort);
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (hits.length > 0) {
products = new Vector<Product>(n);
int i = 0;
- while (products.size() < Math.min(n, hits.length())) {
- Document productDoc = hits.doc(i);
+ while (products.size() < Math.min(n, hits.length)) {
+ Document productDoc = searcher.doc(hits[i].doc);
CompleteProduct prod = toCompleteProduct(productDoc, false,
false);
products.add(prod.getProduct());
@@ -706,7 +755,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -940,15 +989,28 @@ public class LuceneCatalog implements Catalog {
private synchronized void removeProductDocument(Product product)
throws CatalogException {
- IndexReader reader = null;
try {
- reader = IndexReader.open(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
LOG.log(Level.FINE,
"LuceneCatalog: remove document from index for product: ["
+ product.getProductId() + "]");
- reader.deleteDocuments(new Term("product_id", product
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
+
+ IndexWriter writer = new IndexWriter(indexDir, config);
+ writer.deleteDocuments(new Term("product_id", product
.getProductId()));
+ writer.close();
+
} catch (IOException e) {
LOG.log(Level.WARNING, "Exception removing product: ["
+ product.getProductName() + "] from index: Message: "
@@ -969,24 +1031,24 @@ public class LuceneCatalog implements Catalog {
private synchronized void addCompleteProductToIndex(CompleteProduct cp)
throws CatalogException {
IndexWriter writer = null;
+ try {
+ /*writer = new IndexWriter(indexFilePath, new StandardAnalyzer(),
+ createIndex);*/
+ //writer.setCommitLockTimeout(this.commitLockTimeout * 1000);
+ //writer.setWriteLockTimeout(this.writeLockTimeout * 1000);
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
- File indexDir = new File(indexFilePath);
-
- boolean createIndex;
-
- createIndex = !(indexDir.exists() && indexDir.isDirectory());
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(mergeFactor);
+ config.setMergePolicy(lmp);
- try {
- writer = new IndexWriter(indexFilePath, new StandardAnalyzer(),
- createIndex);
- writer.setCommitLockTimeout(this.commitLockTimeout * 1000);
- writer.setWriteLockTimeout(this.writeLockTimeout * 1000);
- writer.setMergeFactor(this.mergeFactor);
+ writer = new IndexWriter(indexDir, config);
Document doc = toDoc(cp.getProduct(), cp.getMetadata());
writer.addDocument(doc);
// TODO: determine a better way to optimize the index
- } catch (IOException e) {
+ } catch (Exception e) {
LOG.log(Level.WARNING, "Unable to index product: ["
+ cp.getProduct().getProductName() + "]: Message: "
+ e.getMessage(), e);
@@ -998,7 +1060,8 @@ public class LuceneCatalog implements Catalog {
if (writer != null) {
writer.close();
}
- } catch (Exception ignore) {
+ } catch (Exception e) {
+ System.out.println("failed"+e.getLocalizedMessage());
}
}
@@ -1044,13 +1107,12 @@ public class LuceneCatalog implements Catalog {
}
} else {
// add all metadata elements found in document
- Enumeration<Field> fields = doc.fields();
- while (fields.hasMoreElements()) {
- Field field = fields.nextElement();
- if (!names.contains(field.name())) {
- names.add(field.name());
- }
- }
+ List<IndexableField> fields = doc.getFields();
+ for(IndexableField field: fields){
+ if (!names.contains(field.name())) {
+ names.add(field.name());
+ }
+ }
}
@@ -1108,36 +1170,33 @@ public class LuceneCatalog implements Catalog {
private Document toDoc(Product product, Metadata metadata) {
Document doc = new Document();
-
+//TODO CHECK STORED TYPES
// add the product information
doc.add(new Field("product_id", product.getProductId(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ StringField.TYPE_STORED));
doc.add(new Field("product_name", product.getProductName(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ TextField.TYPE_STORED));
doc.add(new Field("product_structure", product.getProductStructure(),
- Field.Store.YES, Field.Index.UN_TOKENIZED));
+ TextField.TYPE_STORED));
doc
.add(new Field("product_transfer_status", product
- .getTransferStatus(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getTransferStatus(), TextField.TYPE_STORED));
// product type
doc
.add(new Field("product_type_id", product.getProductType()
- .getProductTypeId(), Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getProductTypeId(), StringField.TYPE_STORED));
doc.add(new Field("product_type_name", product.getProductType()
- .getName(), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ .getName(), TextField.TYPE_STORED));
doc.add(new Field("product_type_desc", product.getProductType()
.getDescription() != null ? product.getProductType()
- .getDescription() : "", Field.Store.YES, Field.Index.NO));
+ .getDescription() : "", TextField.TYPE_STORED));
doc.add(new Field("product_type_repoPath", product.getProductType()
.getProductRepositoryPath() != null ? product.getProductType()
- .getProductRepositoryPath() : "", Field.Store.YES,
- Field.Index.NO));
+ .getProductRepositoryPath() : "", StringField.TYPE_STORED));
doc.add(new Field("product_type_versioner", product.getProductType()
.getVersioner() != null ? product.getProductType()
- .getVersioner() : "", Field.Store.YES, Field.Index.NO));
+ .getVersioner() : "", StringField.TYPE_STORED));
// write metadata fields to the Lucene document
List<String> keys = new ArrayList<String>();
@@ -1175,30 +1234,29 @@ public class LuceneCatalog implements Catalog {
}
for (String val : values) {
- doc.add(new Field(key, val, Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ doc.add(new Field(key, val, StringField.TYPE_STORED));
+ if(values.size()==1) {
+ doc.add(new SortedDocValuesField(key, new BytesRef(val)));
+ }
}
}
// add the product references
for (Reference r : product.getProductReferences()) {
doc.add(new Field("reference_orig", r.getOrigReference(),
- Field.Store.YES, Field.Index.NO));
+ StringField.TYPE_STORED));
doc
.add(new Field("reference_data_store", r
- .getDataStoreReference(), Field.Store.YES,
- Field.Index.NO));
+ .getDataStoreReference(), StringField.TYPE_STORED));
doc.add(new Field("reference_fileSize", String.valueOf(r
- .getFileSize()), Field.Store.YES, Field.Index.NO));
+ .getFileSize()), StringField.TYPE_STORED));
doc.add(new Field("reference_mimeType", r.getMimeType() != null ? r
- .getMimeType().getName() : "", Field.Store.YES,
- Field.Index.UN_TOKENIZED));
+ .getMimeType().getName() : "", StringField.TYPE_STORED));
}
// add special field for all products
// then can use that field to retrieve back all products
- doc.add(new Field("myfield", "myvalue", Field.Store.NO,
- Field.Index.TOKENIZED));
+ doc.add(new Field("myfield", "myvalue", TextField.TYPE_STORED));
return doc;
}
@@ -1228,15 +1286,19 @@ public class LuceneCatalog implements Catalog {
IndexSearcher searcher = null;
int numHits = -1;
-
try {
- searcher = new IndexSearcher(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
- TermQuery prodTypeTermQuery = new TermQuery(new Term(
+ org.apache.lucene.search.Query prodTypeTermQuery = new TermQuery(new Term(
"product_type_id", type.getProductTypeId()));
booleanQuery.add(prodTypeTermQuery, BooleanClause.Occur.MUST);
@@ -1247,8 +1309,12 @@ public class LuceneCatalog implements Catalog {
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery);
- numHits = hits.length();
+
+ //TODO FIX returned records
+ TopDocs hits = searcher.search(booleanQuery.build(), 1);
+
+
+ numHits = hits.totalHits;
} catch (IOException e) {
LOG.log(Level.WARNING,
"IOException when opening index directory: ["
@@ -1258,7 +1324,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -1277,12 +1343,16 @@ public class LuceneCatalog implements Catalog {
if (pageNum == -1) {
doSkip = false;
}
-
try {
- searcher = new IndexSearcher(indexFilePath);
+ reader = DirectoryReader.open(indexDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
TermQuery prodTypeTermQuery = new TermQuery(new Term(
@@ -1295,37 +1365,42 @@ public class LuceneCatalog implements Catalog {
}
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
+ SortField.Type.STRING, true));
LOG.log(Level.FINE, "Querying LuceneCatalog: q: [" + booleanQuery
+ "]");
- Hits hits = searcher.search(booleanQuery, sort);
+ //TODO FIX NUMBER OF RECORDS
+ TopDocs topDocs = searcher.search(booleanQuery.build(),1, sort);
// Calculate page size and set it while we have the results
if (page != null) {
- page.setTotalPages(PaginationUtils.getTotalPage(hits.length(), pageSize));
+ page.setTotalPages(PaginationUtils.getTotalPage(topDocs.totalHits, pageSize));
}
-
- if (hits.length() > 0) {
+
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (hits.length > 0) {
int startNum = (pageNum - 1) * pageSize;
if (doSkip) {
- if (startNum > hits.length()) {
+ if (startNum > hits.length) {
startNum = 0;
}
products = new Vector<Product>(pageSize);
- for (int i = startNum; i < Math.min(hits.length(),
+ for (int i = startNum; i < Math.min(hits.length,
(startNum + pageSize)); i++) {
- Document productDoc = hits.doc(i);
+ Document productDoc = searcher.doc(hits[i].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
products.add(prod.getProduct());
}
} else {
- products = new Vector<Product>(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ products = new Vector<Product>(hits.length);
+ for (int i = 0; i < hits.length; i++) {
+ Document productDoc = searcher.doc(hits[i].doc);
+
CompleteProduct prod = toCompleteProduct(productDoc,
false, false);
products.add(prod.getProduct());
@@ -1347,7 +1422,7 @@ public class LuceneCatalog implements Catalog {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE
} catch (Exception ignore) {
}
}
@@ -1358,7 +1433,7 @@ public class LuceneCatalog implements Catalog {
private org.apache.lucene.search.Query getQuery(QueryCriteria queryCriteria) throws CatalogException {
if (queryCriteria instanceof BooleanQueryCriteria) {
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
BooleanClause.Occur occur;
switch (((BooleanQueryCriteria) queryCriteria).getOperator()) {
case BooleanQueryCriteria.AND:
@@ -1380,7 +1455,7 @@ public class LuceneCatalog implements Catalog {
booleanQuery.add(this.getQuery(qc), occur);
}
- return booleanQuery;
+ return booleanQuery.build();
} else if (queryCriteria instanceof TermQueryCriteria) {
String val = ((TermQueryCriteria) queryCriteria).getValue();
return new TermQuery(new Term(queryCriteria.getElementName(), val));
@@ -1397,7 +1472,7 @@ public class LuceneCatalog implements Catalog {
endTerm = new Term(queryCriteria.getElementName(), endVal);
}
- return new RangeQuery(startTerm, endTerm, inclusive);
+ return TermRangeQuery.newStringRange(startTerm.field(), startVal, endVal, inclusive,inclusive);
}else {
throw new CatalogException("Invalid QueryCriteria ["
+ queryCriteria.getClass().getCanonicalName() + "]");
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
index f70fa71..2512964 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/catalog/LuceneCatalogFactory.java
@@ -19,9 +19,15 @@ package org.apache.oodt.cas.filemgr.catalog;
//JDK imports
import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
import java.util.logging.Logger;
//OODT imports
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
import org.apache.oodt.cas.filemgr.util.GenericFileManagerObjectFactory;
import org.apache.oodt.cas.metadata.util.PathUtils;
import org.apache.oodt.cas.filemgr.validation.ValidationLayer;
@@ -47,6 +53,7 @@ public class LuceneCatalogFactory implements CatalogFactory {
public static final int VAL3 = 20;
/* path to the index directory for lucene catalogs */
private String indexFilePath = null;
+ private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
/* our validation layer */
private ValidationLayer validationLayer = null;
@@ -119,9 +126,16 @@ public class LuceneCatalogFactory implements CatalogFactory {
File indexDir = new File(indexFilePath);
// Create the index if it does not already exist
IndexWriter writer = null;
+ config.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
if (!indexDir.exists()) {
- try {
- writer = new IndexWriter(indexDir, new StandardAnalyzer(), true);
+ try {
+ try {
+ Directory indexDir2 = FSDirectory.open(new File( indexFilePath ).toPath());
+ writer = new IndexWriter(indexDir2, config);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
} catch (Exception e) {
LOG.severe("Unable to create index: " + e.getMessage());
} finally {
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
index f688478..85e06cd 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/cli/action/LuceneQueryCliAction.java
@@ -21,14 +21,10 @@ import com.google.common.collect.Lists;
import org.apache.commons.lang.Validate;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.BooleanQueryCriteria;
import org.apache.oodt.cas.filemgr.structs.QueryCriteria;
import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
@@ -110,13 +106,14 @@ public class LuceneQueryCliAction extends AbstractQueryCliAction {
}
return bqc;
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- return new RangeQueryCriteria(startT.field(), startT
- .text(), endT.text(), ((RangeQuery) luceneQuery).isInclusive());
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ //TODO CHECK Inclusive
+ return new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT
+ .utf8ToString(), endT.utf8ToString()/*, ((TermRangeQuery) luceneQuery).isInclusive()*/);
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
BooleanQueryCriteria bqc = new BooleanQueryCriteria();
bqc.setOperator(BooleanQueryCriteria.AND);
for (BooleanClause clause : clauses) {
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
index d9bdaaf..9470c62 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CASAnalyzer.java
@@ -20,7 +20,12 @@ package org.apache.oodt.cas.filemgr.tools;
//Lucene imports
import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.core.StopAnalyzer;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
//JDK imports
import java.io.Reader;
@@ -44,26 +49,47 @@ public class CASAnalyzer extends Analyzer {
* An array containing some common English words that are usually not useful
* for searching.
*/
- public static final String[] STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS;
+ public static final CharArraySet STOP_WORDS = StopAnalyzer.ENGLISH_STOP_WORDS_SET;
+ private Reader reader;
/** Builds an analyzer. */
public CASAnalyzer() {
this(STOP_WORDS);
}
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ TokenStream result = new WhitespaceTokenizer(/*reader*/);
+ result = new StandardFilter(result);
+ result = new StopFilter(result, STOP_WORDS);
+
+
+ //TODO FIX
+ try {
+ throw new Exception("needs fixing");
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return null; //new TokenStreamComponents();
+ }
+
+ public void tokenStreams(String fname, Reader reader){
+ this.reader = reader;
+ }
/** Builds an analyzer with the given stop words. */
- public CASAnalyzer(String[] stopWords) {
- stopSet = StopFilter.makeStopSet(stopWords);
+ public CASAnalyzer(CharArraySet stopWords) {
+ //stopSet = StopFilter.makeStopSet(stopWords);
}
/**
* Constructs a {@link org.apache.lucene.analysis.standard.StandardTokenizer} filtered by a {@link
* StandardFilter}, a {@link LowerCaseFilter} and a {@link StopFilter}.
*/
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ /*public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new WhitespaceTokenizer(reader);
result = new StandardFilter(result);
result = new StopFilter(result, stopSet);
return result;
- }
+ }*/
+
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
index e086940..5dbb0da 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/CatalogSearch.java
@@ -18,14 +18,10 @@
package org.apache.oodt.cas.filemgr.tools;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.Element;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
@@ -41,6 +37,7 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
+import java.util.List;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.logging.Level;
@@ -255,13 +252,14 @@ public class CatalogSearch {
aT.text()));
}
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(startT.field(), startT
- .text(), endT.text()));
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+
+ //TODO CHECK THIS RANGE!
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
for (BooleanClause clause : clauses) {
GenerateCASQuery(casQuery, (clause).getQuery());
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
index dd92f28..16cf966 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/OptimizeLuceneCatalog.java
@@ -19,12 +19,14 @@ package org.apache.oodt.cas.filemgr.tools;
//JDK imports
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.logging.Level;
import java.util.logging.Logger;
//Lucene imports
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.FSDirectory;
/**
* @author mattmann
@@ -39,6 +41,9 @@ public class OptimizeLuceneCatalog {
public static final double DOUBLE = 1000.0;
public static final int INT = 20;
+ private DirectoryReader reader;
+ private IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+
/* the path to the lucene index directory */
private String catalogPath = null;
@@ -55,6 +60,12 @@ public class OptimizeLuceneCatalog {
public OptimizeLuceneCatalog(String catPath, int mf) {
this.catalogPath = catPath;
this.mergeFactor = mf;
+ try {
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(catalogPath)));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
}
public void doOptimize() {
@@ -62,11 +73,14 @@ public class OptimizeLuceneCatalog {
boolean createIndex = false;
try {
- writer = new IndexWriter(catalogPath, new StandardAnalyzer(),
- false);
- writer.setMergeFactor(this.mergeFactor);
+ writer = new IndexWriter(reader.directory(), config);
+ LogMergePolicy lmp =new LogDocMergePolicy();
+ lmp.setMergeFactor(this.mergeFactor);
+ config.setMergePolicy(lmp);
+
long timeBefore = System.currentTimeMillis();
- writer.optimize();
+ //TODO http://blog.trifork.com/2011/11/21/simon-says-optimize-is-bad-for-you/
+ //writer.optimize();
long timeAfter = System.currentTimeMillis();
double numSeconds = ((timeAfter - timeBefore) * 1.0) / DOUBLE;
LOG.log(Level.INFO, "LuceneCatalog: [" + this.catalogPath
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
index f7ba6e6..4a9641f 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/QueryTool.java
@@ -18,14 +18,10 @@
package org.apache.oodt.cas.filemgr.tools;
import org.apache.lucene.index.Term;
-import org.apache.lucene.queryParser.ParseException;
-import org.apache.lucene.queryParser.QueryParser;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.queryparser.classic.ParseException;
+import org.apache.lucene.queryparser.classic.QueryParser;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.BytesRef;
import org.apache.oodt.cas.filemgr.structs.Product;
import org.apache.oodt.cas.filemgr.structs.ProductType;
import org.apache.oodt.cas.filemgr.structs.RangeQueryCriteria;
@@ -139,13 +135,12 @@ public final class QueryTool {
aT.field(), aT.text()));
}
}
- } else if (luceneQuery instanceof RangeQuery) {
- Term startT = ((RangeQuery) luceneQuery).getLowerTerm();
- Term endT = ((RangeQuery) luceneQuery).getUpperTerm();
- casQuery.addCriterion(new RangeQueryCriteria(startT
- .field(), startT.text(), endT.text()));
+ } else if (luceneQuery instanceof TermRangeQuery) {
+ BytesRef startT = ((TermRangeQuery) luceneQuery).getLowerTerm();
+ BytesRef endT = ((TermRangeQuery) luceneQuery).getUpperTerm();
+ casQuery.addCriterion(new RangeQueryCriteria(((TermRangeQuery) luceneQuery).getField(), startT.utf8ToString(), endT.utf8ToString()));
} else if (luceneQuery instanceof BooleanQuery) {
- BooleanClause[] clauses = ((BooleanQuery) luceneQuery).getClauses();
+ List<BooleanClause> clauses = ((BooleanQuery) luceneQuery).clauses();
for (BooleanClause clause : clauses) {
generateCASQuery(casQuery, (clause).getQuery());
}
http://git-wip-us.apache.org/repos/asf/oodt/blob/999e68dd/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
----------------------------------------------------------------------
diff --git a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
index 3b476e4..a765b44 100644
--- a/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
+++ b/filemgr/src/main/java/org/apache/oodt/cas/filemgr/tools/RangeQueryTester.java
@@ -19,6 +19,7 @@ package org.apache.oodt.cas.filemgr.tools;
//JDK imports
import java.io.IOException;
+import java.nio.file.Paths;
import java.util.List;
import java.util.Vector;
import java.util.logging.Level;
@@ -26,15 +27,10 @@ import java.util.logging.Logger;
//Lucene imports
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.Hits;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.RangeQuery;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.FSDirectory;
/**
* @author mattmann
@@ -66,6 +62,7 @@ public final class RangeQueryTester {
private static final Logger LOG = Logger.getLogger(RangeQueryTester.class
.getName());
+ DirectoryReader reader;
/**
*
*/
@@ -75,12 +72,16 @@ public final class RangeQueryTester {
public List doRangeQuery(String productTypeId) {
List products = null;
IndexSearcher searcher = null;
-
try {
- searcher = new IndexSearcher(this.indexPath);
+ reader = DirectoryReader.open(FSDirectory.open(Paths.get(this.indexPath)));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ searcher = new IndexSearcher(reader);
// construct a Boolean query here
- BooleanQuery booleanQuery = new BooleanQuery();
+ BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
// add the product type as the first clause
TermQuery prodTypeTermQuery = new TermQuery(new Term(
@@ -99,8 +100,8 @@ public final class RangeQueryTester {
this.startFieldEndValue);
}
- RangeQuery query1 = new RangeQuery(startFieldStartTerm,
- startFieldEndTerm, true);
+ TermRangeQuery query1 = new TermRangeQuery(startFieldEndTerm.field(),startFieldStartTerm.bytes(),
+ startFieldEndTerm.bytes(), true, true);
booleanQuery.add(query1, BooleanClause.Occur.MUST);
if (this.endFieldName != null
@@ -117,18 +118,22 @@ public final class RangeQueryTester {
this.endFieldEndValue);
}
- RangeQuery query2 = new RangeQuery(endFieldStartTerm,
- endFieldEndTerm, true);
+ TermRangeQuery query2 = new TermRangeQuery(endFieldEndTerm.field(),endFieldStartTerm.bytes(),
+ endFieldEndTerm.bytes(), true, true);
booleanQuery.add(query2, BooleanClause.Occur.MUST);
}
Sort sort = new Sort(new SortField("CAS.ProductReceivedTime",
- SortField.STRING, true));
- Hits hits = searcher.search(booleanQuery, sort);
- if (hits.length() > 0) {
- products = new Vector(hits.length());
- for (int i = 0; i < hits.length(); i++) {
- Document productDoc = hits.doc(i);
+ SortField.Type.STRING, true));
+ //TODO Fix number
+ TopFieldDocs topDocs = searcher.search(booleanQuery.build(), 1, sort);
+ ScoreDoc[] hits = topDocs.scoreDocs;
+
+ if (topDocs.totalHits > 0) {
+ products = new Vector(topDocs.totalHits);
+ for (int i = 0; i < topDocs.totalHits; i++) {
+ Document productDoc = searcher.doc(hits[i].doc);
+
products.add(productDoc.get("reference_data_store"));
}
} else {
@@ -146,7 +151,8 @@ public final class RangeQueryTester {
} finally {
if (searcher != null) {
try {
- searcher.close();
+ //TODO CLOSE SEARCH
+ // searcher.close();
} catch (Exception ignore) {
}
}