You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/07/28 13:28:27 UTC
svn commit: r1366643 [13/19] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/mave...
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/ScoredDocIdCollector.java Sat Jul 28 11:27:51 2012
@@ -9,7 +9,7 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.FixedBitSet;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -52,7 +52,7 @@ public abstract class ScoredDocIdCollect
@Override
public void collect(int doc) {
- docIds.fastSet(docBase + doc);
+ docIds.set(docBase + doc);
++numDocIds;
}
@@ -103,7 +103,9 @@ public abstract class ScoredDocIdCollect
@SuppressWarnings("synthetic-access")
public ScoringDocIdCollector(int maxDoc) {
super(maxDoc);
- scores = new float[maxDoc];
+ // only matching documents have an entry in the scores array. Therefore start with
+ // a small array and grow when needed.
+ scores = new float[64];
}
@Override
@@ -111,7 +113,7 @@ public abstract class ScoredDocIdCollect
@Override
public void collect(int doc) throws IOException {
- docIds.fastSet(docBase + doc);
+ docIds.set(docBase + doc);
float score = this.scorer.score();
if (numDocIds >= scores.length) {
@@ -167,7 +169,7 @@ public abstract class ScoredDocIdCollect
protected int numDocIds;
protected int docBase;
- protected final OpenBitSet docIds;
+ protected final FixedBitSet docIds;
/**
* Creates a new {@link ScoredDocIdCollector} with the given parameters.
@@ -187,7 +189,7 @@ public abstract class ScoredDocIdCollect
private ScoredDocIdCollector(int maxDoc) {
numDocIds = 0;
- docIds = new OpenBitSet(maxDoc);
+ docIds = new FixedBitSet(maxDoc);
}
/** Returns the default score used when scoring is disabled. */
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/search/results/package.html Sat Jul 28 11:27:51 2012
@@ -20,15 +20,8 @@
</head>
<body>
<h1>Results of Faceted Search</h1>
- <!--
<p>
The results of facets accumulation are obtained as a list of {@link org.apache.lucene.facet.search.results.FacetResult} elements.
- See two API calls for obtaining these results:
- <ul>
- <li></li>
- </ul>
-
</p>
- -->
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java Sat Jul 28 11:27:51 2012
@@ -375,7 +375,7 @@ public class DirectoryTaxonomyWriter imp
* returning the category's ordinal, or a negative number in case the
* category does not yet exist in the taxonomy.
*/
- protected int findCategory(CategoryPath categoryPath) throws IOException {
+ protected synchronized int findCategory(CategoryPath categoryPath) throws IOException {
// If we can find the category in the cache, or we know the cache is
// complete, we can return the response directly from it
int res = cache.get(categoryPath);
@@ -474,12 +474,11 @@ public class DirectoryTaxonomyWriter imp
@Override
public int addCategory(CategoryPath categoryPath) throws IOException {
ensureOpen();
- // If the category is already in the cache and/or the taxonomy, we
- // should return its existing ordinal
- int res = findCategory(categoryPath);
+ // check the cache outside the synchronized block. this results in better
+ // concurrency when categories are there.
+ int res = cache.get(categoryPath);
if (res < 0) {
- // the category is neither in the cache nor in the index - following code
- // cannot be executed in parallel.
+ // the category is not in the cache - following code cannot be executed in parallel.
synchronized (this) {
res = findCategory(categoryPath);
if (res < 0) {
@@ -494,7 +493,6 @@ public class DirectoryTaxonomyWriter imp
}
}
return res;
-
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/writercache/TaxonomyWriterCache.java Sat Jul 28 11:27:51 2012
@@ -22,22 +22,25 @@ import org.apache.lucene.facet.taxonomy.
/**
* TaxonomyWriterCache is a relatively simple interface for a cache of
- * category->ordinal mappings, used in TaxonomyWriter implementations
- * (such as {@link DirectoryTaxonomyWriter}).
- * <P>
- * It basically has put() methods for adding a mapping, and get() for looking
- * a mapping up the cache. The cache does <B>not</B> guarantee to hold
- * everything that has been put into it, and might in fact selectively
- * delete some of the mappings (e.g., the ones least recently used).
- * This means that if get() returns a negative response, it does not
- * necessarily mean that the category doesn't exist - just that it is not
- * in the cache. The caller can only infer that the category doesn't exist
- * if it knows the cache to be complete (because all the categories were
- * loaded into the cache, and since then no put() returned true).
- * <P> However,
- * if it does so, it should clear out large parts of the cache at once, because
- * the user will typically need to work hard to recover from every cache
+ * category->ordinal mappings, used in TaxonomyWriter implementations (such as
+ * {@link DirectoryTaxonomyWriter}).
+ * <p>
+ * It basically has put() methods for adding a mapping, and get() for looking a
+ * mapping up the cache. The cache does <B>not</B> guarantee to hold everything
+ * that has been put into it, and might in fact selectively delete some of the
+ * mappings (e.g., the ones least recently used). This means that if get()
+ * returns a negative response, it does not necessarily mean that the category
+ * doesn't exist - just that it is not in the cache. The caller can only infer
+ * that the category doesn't exist if it knows the cache to be complete (because
+ * all the categories were loaded into the cache, and since then no put()
+ * returned true).
+ * <p>
+ * However, if it does so, it should clear out large parts of the cache at once,
+ * because the user will typically need to work hard to recover from every cache
* cleanup (see {@link #put(CategoryPath, int)}'s return value).
+ * <p>
+ * <b>NOTE:</b> the cache may be accessed concurrently by multiple threads,
+ * therefore cache implementations should take this into consideration.
*
* @lucene.experimental
*/
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ArrayHashMap.java Sat Jul 28 11:27:51 2012
@@ -389,10 +389,8 @@ public class ArrayHashMap<K,V> implement
/** Prints the baseHash array, used for debugging purposes. */
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i : baseHash) {
- System.out.println(i + ".\t" + i);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(this.baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/FloatToObjectMap.java Sat Jul 28 11:27:51 2012
@@ -462,10 +462,8 @@ public class FloatToObjectMap<T> impleme
* Prints the baseHash array, used for DEBUG purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(this.baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntHashSet.java Sat Jul 28 11:27:51 2012
@@ -389,12 +389,9 @@ public class IntHashSet {
/**
* Prints the baseHash array, used for debug purposes.
*/
- public void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- if (baseHash[i] != 0) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
- }
+ @SuppressWarnings("unused")
+ private String getBaseHashAsString() {
+ return Arrays.toString(this.baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToDoubleMap.java Sat Jul 28 11:27:51 2012
@@ -461,10 +461,8 @@ public class IntToDoubleMap {
* Prints the baseHash array, used for debug purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(this.baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToIntMap.java Sat Jul 28 11:27:51 2012
@@ -458,10 +458,8 @@ public class IntToIntMap {
* Prints the baseHash array, used for debug purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(this.baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/IntToObjectMap.java Sat Jul 28 11:27:51 2012
@@ -462,10 +462,8 @@ public class IntToObjectMap<T> implement
* Prints the baseHash array, used for debug purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToFloatMap.java Sat Jul 28 11:27:51 2012
@@ -463,10 +463,8 @@ public class ObjectToFloatMap<K> {
* Prints the baseHash array, used for debug purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/org/apache/lucene/util/collections/ObjectToIntMap.java Sat Jul 28 11:27:51 2012
@@ -462,10 +462,8 @@ public class ObjectToIntMap<K> {
* Prints the baseHash array, used for debug purposes.
*/
@SuppressWarnings("unused")
- private void printBaseHash() {
- for (int i = 0; i < this.baseHash.length; i++) {
- System.out.println(i + ".\t" + baseHash[i]);
- }
+ private String getBaseHashAsString() {
+ return Arrays.toString(baseHash);
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/java/overview.html?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/java/overview.html (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/java/overview.html Sat Jul 28 11:27:51 2012
@@ -21,7 +21,6 @@
</title>
</head>
<body>
- <!-- NOTE: if you update the userguide link, also update the one in o.a.l.facet/package.html -->
Provides faceted indexing and search capabilities (checkout the <a href="org/apache/lucene/facet/doc-files/userguide.html">userguide</a>).
</body>
</html>
Modified: lucene/dev/branches/lucene3312/lucene/grouping/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/grouping/build.xml?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/grouping/build.xml (original)
+++ lucene/dev/branches/lucene3312/lucene/grouping/build.xml Sat Jul 28 11:27:51 2012
@@ -35,8 +35,8 @@
<path refid="base.classpath"/>
</path>
- <target name="compile" depends="jar-queries,common.compile-core" description="Compiles grouping classes" />
- <target name="jar-core" depends="common.jar-core" />
+ <target name="init" depends="module-build.init,jar-queries"/>
+
<target name="javadocs" depends="javadocs-queries,compile-core">
<invoke-module-javadoc>
<links>
Modified: lucene/dev/branches/lucene3312/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (original)
+++ lucene/dev/branches/lucene3312/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java Sat Jul 28 11:27:51 2012
@@ -97,6 +97,11 @@ public class BlockGroupingCollector exte
public float score() {
return score;
}
+
+ @Override
+ public float freq() {
+ throw new UnsupportedOperationException(); // TODO: wtf does this class do?
+ }
@Override
public int docID() {
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Sat Jul 28 11:27:51 2012
@@ -38,6 +38,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -129,6 +130,8 @@ public class TokenSources {
if (termsEnum.next() != null) {
DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false);
if (dpEnum != null) {
+ int doc = dpEnum.nextDoc();
+ assert doc >= 0 && doc != DocIdSetIterator.NO_MORE_DOCS;
int pos = dpEnum.nextPosition();
if (pos >= 0) {
return true;
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Sat Jul 28 11:27:51 2012
@@ -17,10 +17,6 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
@@ -29,10 +25,19 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.search.highlight.DefaultEncoder;
import org.apache.lucene.search.highlight.Encoder;
-import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
+import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
protected String[] preTags, postTags;
@@ -48,6 +53,7 @@ public abstract class BaseFragmentsBuild
public static final String[] COLORED_POST_TAGS = { "</b>" };
private char multiValuedSeparator = ' ';
private final BoundaryScanner boundaryScanner;
+ private boolean discreteMultiValueHighlighting = false;
protected BaseFragmentsBuilder(){
this( new String[]{ "<b>" }, new String[]{ "</b>" } );
@@ -76,7 +82,7 @@ public abstract class BaseFragmentsBuild
public abstract List<WeightedFragInfo> getWeightedFragInfoList( List<WeightedFragInfo> src );
private static final Encoder NULL_ENCODER = new DefaultEncoder();
-
+
public String createFragment( IndexReader reader, int docId,
String fieldName, FieldFragList fieldFragList ) throws IOException {
return createFragment( reader, docId, fieldName, fieldFragList,
@@ -102,14 +108,23 @@ public abstract class BaseFragmentsBuild
public String[] createFragments( IndexReader reader, int docId,
String fieldName, FieldFragList fieldFragList, int maxNumFragments,
String[] preTags, String[] postTags, Encoder encoder ) throws IOException {
- if( maxNumFragments < 0 )
+
+ if( maxNumFragments < 0 ) {
throw new IllegalArgumentException( "maxNumFragments(" + maxNumFragments + ") must be positive number." );
+ }
- List<WeightedFragInfo> fragInfos = getWeightedFragInfoList( fieldFragList.getFragInfos() );
-
+ List<WeightedFragInfo> fragInfos = fieldFragList.getFragInfos();
List<String> fragments = new ArrayList<String>( maxNumFragments );
Field[] values = getFields( reader, docId, fieldName );
- if( values.length == 0 ) return null;
+ if( values.length == 0 ) {
+ return null;
+ }
+
+ if (discreteMultiValueHighlighting && values.length > 1) {
+ fragInfos = discreteMultiValueHighlighting(fragInfos, values);
+ }
+
+ fragInfos = getWeightedFragInfoList(fragInfos);
StringBuilder buffer = new StringBuilder();
int[] nextValueIndex = { 0 };
for( int n = 0; n < maxNumFragments && n < fragInfos.size(); n++ ){
@@ -186,7 +201,92 @@ public abstract class BaseFragmentsBuild
int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
return buffer.substring( startOffset, eo );
}
-
+
+ protected List<WeightedFragInfo> discreteMultiValueHighlighting(List<WeightedFragInfo> fragInfos, Field[] fields) {
+ Map<String, List<WeightedFragInfo>> fieldNameToFragInfos = new HashMap<String, List<WeightedFragInfo>>();
+ for (Field field : fields) {
+ fieldNameToFragInfos.put(field.name(), new ArrayList<WeightedFragInfo>());
+ }
+
+ fragInfos: for (WeightedFragInfo fragInfo : fragInfos) {
+ int fieldStart;
+ int fieldEnd = 0;
+ for (Field field : fields) {
+ if (field.stringValue().isEmpty()) {
+ fieldEnd++;
+ continue;
+ }
+ fieldStart = fieldEnd;
+ fieldEnd += field.stringValue().length() + 1; // + 1 for going to next field with same name.
+
+ if (fragInfo.getStartOffset() >= fieldStart && fragInfo.getEndOffset() >= fieldStart &&
+ fragInfo.getStartOffset() <= fieldEnd && fragInfo.getEndOffset() <= fieldEnd) {
+ fieldNameToFragInfos.get(field.name()).add(fragInfo);
+ continue fragInfos;
+ }
+
+ if (fragInfo.getSubInfos().isEmpty()) {
+ continue fragInfos;
+ }
+
+ Toffs firstToffs = fragInfo.getSubInfos().get(0).getTermsOffsets().get(0);
+ if (fragInfo.getStartOffset() >= fieldEnd || firstToffs.getStartOffset() >= fieldEnd) {
+ continue;
+ }
+
+ int fragStart = fieldStart;
+ if (fragInfo.getStartOffset() > fieldStart && fragInfo.getStartOffset() < fieldEnd) {
+ fragStart = fragInfo.getStartOffset();
+ }
+
+ int fragEnd = fieldEnd;
+ if (fragInfo.getEndOffset() > fieldStart && fragInfo.getEndOffset() < fieldEnd) {
+ fragEnd = fragInfo.getEndOffset();
+ }
+
+
+ List<SubInfo> subInfos = new ArrayList<SubInfo>();
+ WeightedFragInfo weightedFragInfo = new WeightedFragInfo(fragStart, fragEnd, subInfos, fragInfo.getTotalBoost());
+
+ Iterator<SubInfo> subInfoIterator = fragInfo.getSubInfos().iterator();
+ while (subInfoIterator.hasNext()) {
+ SubInfo subInfo = subInfoIterator.next();
+ List<Toffs> toffsList = new ArrayList<Toffs>();
+ Iterator<Toffs> toffsIterator = subInfo.getTermsOffsets().iterator();
+ while (toffsIterator.hasNext()) {
+ Toffs toffs = toffsIterator.next();
+ if (toffs.getStartOffset() >= fieldStart && toffs.getEndOffset() <= fieldEnd) {
+ toffsList.add(toffs);
+ toffsIterator.remove();
+ }
+ }
+ if (!toffsList.isEmpty()) {
+ subInfos.add(new SubInfo(subInfo.getText(), toffsList, subInfo.getSeqnum()));
+ }
+
+ if (subInfo.getTermsOffsets().isEmpty()) {
+ subInfoIterator.remove();
+ }
+ }
+ fieldNameToFragInfos.get(field.name()).add(weightedFragInfo);
+ }
+ }
+
+ List<WeightedFragInfo> result = new ArrayList<WeightedFragInfo>();
+ for (List<WeightedFragInfo> weightedFragInfos : fieldNameToFragInfos.values()) {
+ result.addAll(weightedFragInfos);
+ }
+ Collections.sort(result, new Comparator<WeightedFragInfo>() {
+
+ public int compare(FieldFragList.WeightedFragInfo info1, FieldFragList.WeightedFragInfo info2) {
+ return info1.getStartOffset() - info2.getStartOffset();
+ }
+
+ });
+
+ return result;
+ }
+
public void setMultiValuedSeparator( char separator ){
multiValuedSeparator = separator;
}
@@ -195,6 +295,14 @@ public abstract class BaseFragmentsBuild
return multiValuedSeparator;
}
+ public boolean isDiscreteMultiValueHighlighting() {
+ return discreteMultiValueHighlighting;
+ }
+
+ public void setDiscreteMultiValueHighlighting(boolean discreteMultiValueHighlighting) {
+ this.discreteMultiValueHighlighting = discreteMultiValueHighlighting;
+ }
+
protected String getPreTag( int num ){
return getPreTag( preTags, num );
}
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java Sat Jul 28 11:27:51 2012
@@ -17,12 +17,12 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* FieldFragList has a list of "frag info" that is used by FragmentsBuilder class
* to create fragments (snippets).
@@ -116,7 +116,11 @@ public abstract class FieldFragList {
public int getSeqnum(){
return seqnum;
}
-
+
+ public String getText(){
+ return text;
+ }
+
@Override
public String toString(){
StringBuilder sb = new StringBuilder();
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Sat Jul 28 11:27:51 2012
@@ -317,8 +317,8 @@ public abstract class AbstractTestCase e
}
@Override
- public void reset( Reader input ) throws IOException {
- super.reset( input );
+ public void setReader( Reader input ) throws IOException {
+ super.setReader( input );
reset();
}
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Sat Jul 28 11:27:51 2012
@@ -17,20 +17,32 @@ package org.apache.lucene.search.vectorh
* limitations under the License.
*/
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util._TestUtil;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class SimpleFragmentsBuilderTest extends AbstractTestCase {
@@ -175,4 +187,152 @@ public class SimpleFragmentsBuilderTest
sfb.setMultiValuedSeparator( '/' );
assertEquals( "//a b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
+
+ public void testDiscreteMultiValueHighlighting() throws Exception {
+ makeIndexShortMV();
+
+ FieldQuery fq = new FieldQuery( tq( "d" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ SimpleFragListBuilder sflb = new SimpleFragListBuilder();
+ FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
+ SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
+ sfb.setDiscreteMultiValueHighlighting(true);
+ assertEquals( "<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
+
+ make1dmfIndex("some text to highlight", "highlight other text");
+ fq = new FieldQuery( tq( "text" ), true, true );
+ stack = new FieldTermStack( reader, 0, F, fq );
+ fpl = new FieldPhraseList( stack, fq );
+ sflb = new SimpleFragListBuilder();
+ ffl = sflb.createFieldFragList( fpl, 32 );
+ String[] result = sfb.createFragments(reader, 0, F, ffl, 3);
+ assertEquals(2, result.length);
+ assertEquals("some <b>text</b> to highlight", result[0]);
+ assertEquals("other <b>text</b>", result[1]);
+
+ fq = new FieldQuery( tq( "highlight" ), true, true );
+ stack = new FieldTermStack( reader, 0, F, fq );
+ fpl = new FieldPhraseList( stack, fq );
+ sflb = new SimpleFragListBuilder();
+ ffl = sflb.createFieldFragList( fpl, 32 );
+ result = sfb.createFragments(reader, 0, F, ffl, 3);
+ assertEquals(2, result.length);
+ assertEquals("text to <b>highlight</b>", result[0]);
+ assertEquals("<b>highlight</b> other text", result[1]);
+ }
+
+ public void testRandomDiscreteMultiValueHighlighting() throws Exception {
+ String[] randomValues = new String[3 + random().nextInt(10 * RANDOM_MULTIPLIER)];
+ for (int i = 0; i < randomValues.length; i++) {
+ String randomValue;
+ do {
+ randomValue = _TestUtil.randomSimpleString(random());
+ } while ("".equals(randomValue));
+ randomValues[i] = randomValue;
+ }
+
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(
+ random(),
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setStoreTermVectors(true);
+ customType.setStoreTermVectorOffsets(true);
+ customType.setStoreTermVectorPositions(true);
+
+ int numDocs = randomValues.length * 5;
+ int numFields = 2 + random().nextInt(5);
+ int numTerms = 2 + random().nextInt(3);
+ List<Doc> docs = new ArrayList<Doc>(numDocs);
+ List<Document> documents = new ArrayList<Document>(numDocs);
+ Map<String, Set<Integer>> valueToDocId = new HashMap<String, Set<Integer>>();
+ for (int i = 0; i < numDocs; i++) {
+ Document document = new Document();
+ String[][] fields = new String[numFields][numTerms];
+ for (int j = 0; j < numFields; j++) {
+ String[] fieldValues = new String[numTerms];
+ fieldValues[0] = getRandomValue(randomValues, valueToDocId, i);
+ StringBuilder builder = new StringBuilder(fieldValues[0]);
+ for (int k = 1; k < numTerms; k++) {
+ fieldValues[k] = getRandomValue(randomValues, valueToDocId, i);
+ builder.append(' ').append(fieldValues[k]);
+ }
+ document.add(new Field(F, builder.toString(), customType));
+ fields[j] = fieldValues;
+ }
+ docs.add(new Doc(fields));
+ documents.add(document);
+ }
+ writer.addDocuments(documents);
+ writer.close();
+ IndexReader reader = DirectoryReader.open(dir);
+
+ try {
+ int highlightIters = 1 + random().nextInt(120 * RANDOM_MULTIPLIER);
+ for (int highlightIter = 0; highlightIter < highlightIters; highlightIter++) {
+ String queryTerm = randomValues[random().nextInt(randomValues.length)];
+ int randomHit = valueToDocId.get(queryTerm).iterator().next();
+ List<StringBuilder> builders = new ArrayList<StringBuilder>();
+ for (String[] fieldValues : docs.get(randomHit).fieldValues) {
+ StringBuilder builder = new StringBuilder();
+ boolean hit = false;
+ for (int i = 0; i < fieldValues.length; i++) {
+ if (queryTerm.equals(fieldValues[i])) {
+ builder.append("<b>").append(queryTerm).append("</b>");
+ hit = true;
+ } else {
+ builder.append(fieldValues[i]);
+ }
+ if (i != fieldValues.length - 1) {
+ builder.append(' ');
+ }
+ }
+ if (hit) {
+ builders.add(builder);
+ }
+ }
+
+ FieldQuery fq = new FieldQuery(tq(queryTerm), true, true);
+ FieldTermStack stack = new FieldTermStack(reader, randomHit, F, fq);
+
+ FieldPhraseList fpl = new FieldPhraseList(stack, fq);
+ SimpleFragListBuilder sflb = new SimpleFragListBuilder(100);
+ FieldFragList ffl = sflb.createFieldFragList(fpl, 300);
+
+ SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
+ sfb.setDiscreteMultiValueHighlighting(true);
+ String[] actualFragments = sfb.createFragments(reader, randomHit, F, ffl, numFields);
+ assertEquals(builders.size(), actualFragments.length);
+ for (int i = 0; i < actualFragments.length; i++) {
+ assertEquals(builders.get(i).toString(), actualFragments[i]);
+ }
+ }
+ } finally {
+ reader.close();
+ dir.close();
+ }
+ }
+
+ private String getRandomValue(String[] randomValues, Map<String, Set<Integer>> valueToDocId, int docId) {
+ String value = randomValues[random().nextInt(randomValues.length)];
+ if (!valueToDocId.containsKey(value)) {
+ valueToDocId.put(value, new HashSet<Integer>());
+ }
+ valueToDocId.get(value).add(docId);
+ return value;
+ }
+
+ private static class Doc {
+
+ final String[][] fieldValues;
+
+ private Doc(String[][] fieldValues) {
+ this.fieldValues = fieldValues;
+ }
+ }
+
}
Modified: lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java Sat Jul 28 11:27:51 2012
@@ -208,6 +208,11 @@ class TermsIncludingScoreQuery extends Q
} while (docId != DocIdSetIterator.NO_MORE_DOCS);
return docId;
}
+
+ @Override
+ public float freq() {
+ return 1;
+ }
}
// This impl that tracks whether a docid has already been emitted. This check makes sure that docs aren't emitted
Modified: lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java Sat Jul 28 11:27:51 2012
@@ -161,6 +161,7 @@ public class ToChildBlockJoinQuery exten
private final Bits acceptDocs;
private float parentScore;
+ private float parentFreq = 1;
private int childDoc = -1;
private int parentDoc;
@@ -175,7 +176,7 @@ public class ToChildBlockJoinQuery exten
@Override
public Collection<ChildScorer> getChildren() {
- return Collections.singletonList(new ChildScorer(parentScorer, "BLOCK_JOIN"));
+ return Collections.singleton(new ChildScorer(parentScorer, "BLOCK_JOIN"));
}
@Override
@@ -218,6 +219,7 @@ public class ToChildBlockJoinQuery exten
if (childDoc < parentDoc) {
if (doScores) {
parentScore = parentScorer.score();
+ parentFreq = parentScorer.freq();
}
//System.out.println(" " + childDoc);
return childDoc;
@@ -248,6 +250,11 @@ public class ToChildBlockJoinQuery exten
}
@Override
+ public float freq() throws IOException {
+ return parentFreq;
+ }
+
+ @Override
public int advance(int childTarget) throws IOException {
assert childTarget >= parentBits.length() || !parentBits.get(childTarget);
@@ -269,6 +276,7 @@ public class ToChildBlockJoinQuery exten
}
if (doScores) {
parentScore = parentScorer.score();
+ parentFreq = parentScorer.freq();
}
final int firstChild = parentBits.prevSetBit(parentDoc-1);
//System.out.println(" firstChild=" + firstChild);
Modified: lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (original)
+++ lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java Sat Jul 28 11:27:51 2012
@@ -327,6 +327,11 @@ public class ToParentBlockJoinCollector
public float score() {
return score;
}
+
+ @Override
+ public float freq() {
+ return 1; // TODO: does anything else make sense?... duplicate of grouping's FakeScorer btw?
+ }
@Override
public int docID() {
Modified: lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java Sat Jul 28 11:27:51 2012
@@ -218,6 +218,7 @@ public class ToParentBlockJoinQuery exte
private int parentDoc = -1;
private int prevParentDoc;
private float parentScore;
+ private float parentFreq;
private int nextChildDoc;
private int[] pendingChildDocs = new int[5];
@@ -239,7 +240,7 @@ public class ToParentBlockJoinQuery exte
@Override
public Collection<ChildScorer> getChildren() {
- return Collections.singletonList(new ChildScorer(childScorer, "BLOCK_JOIN"));
+ return Collections.singleton(new ChildScorer(childScorer, "BLOCK_JOIN"));
}
int getChildCount() {
@@ -299,7 +300,9 @@ public class ToParentBlockJoinQuery exte
}
float totalScore = 0;
+ float totalFreq = 0;
float maxScore = Float.NEGATIVE_INFINITY;
+ float maxFreq = 0;
childDocUpto = 0;
do {
@@ -315,9 +318,12 @@ public class ToParentBlockJoinQuery exte
if (scoreMode != ScoreMode.None) {
// TODO: specialize this into dedicated classes per-scoreMode
final float childScore = childScorer.score();
+ final float childFreq = childScorer.freq();
pendingChildScores[childDocUpto] = childScore;
maxScore = Math.max(childScore, maxScore);
+ maxFreq = Math.max(childFreq, maxFreq);
totalScore += childScore;
+ totalFreq += childFreq;
}
childDocUpto++;
nextChildDoc = childScorer.nextDoc();
@@ -329,12 +335,15 @@ public class ToParentBlockJoinQuery exte
switch(scoreMode) {
case Avg:
parentScore = totalScore / childDocUpto;
+ parentFreq = totalFreq / childDocUpto;
break;
case Max:
parentScore = maxScore;
+ parentFreq = maxFreq;
break;
case Total:
parentScore = totalScore;
+ parentFreq = totalFreq;
break;
case None:
break;
@@ -354,6 +363,11 @@ public class ToParentBlockJoinQuery exte
public float score() throws IOException {
return parentScore;
}
+
+ @Override
+ public float freq() {
+ return parentFreq;
+ }
@Override
public int advance(int parentTarget) throws IOException {
Modified: lucene/dev/branches/lucene3312/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java (original)
+++ lucene/dev/branches/lucene3312/lucene/join/src/test/org/apache/lucene/search/join/TestJoinUtil.java Sat Jul 28 11:27:51 2012
@@ -232,7 +232,7 @@ public class TestJoinUtil extends Lucene
public void testSingleValueRandomJoin() throws Exception {
int maxIndexIter = _TestUtil.nextInt(random(), 6, 12);
int maxSearchIter = _TestUtil.nextInt(random(), 13, 26);
- executeRandomJoin(false, maxIndexIter, maxSearchIter);
+ executeRandomJoin(false, maxIndexIter, maxSearchIter, _TestUtil.nextInt(random(), 87, 764));
}
@Test
@@ -240,10 +240,10 @@ public class TestJoinUtil extends Lucene
public void testMultiValueRandomJoin() throws Exception {
int maxIndexIter = _TestUtil.nextInt(random(), 3, 6);
int maxSearchIter = _TestUtil.nextInt(random(), 6, 12);
- executeRandomJoin(true, maxIndexIter, maxSearchIter);
+ executeRandomJoin(true, maxIndexIter, maxSearchIter, _TestUtil.nextInt(random(), 11, 57));
}
- private void executeRandomJoin(boolean multipleValuesPerDocument, int maxIndexIter, int maxSearchIter) throws Exception {
+ private void executeRandomJoin(boolean multipleValuesPerDocument, int maxIndexIter, int maxSearchIter, int numberOfDocumentsToIndex) throws Exception {
for (int indexIter = 1; indexIter <= maxIndexIter; indexIter++) {
if (VERBOSE) {
System.out.println("indexIter=" + indexIter);
@@ -254,7 +254,6 @@ public class TestJoinUtil extends Lucene
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)).setMergePolicy(newLogMergePolicy())
);
- int numberOfDocumentsToIndex = _TestUtil.nextInt(random(), 87, 764);
IndexIterationContext context = createContext(numberOfDocumentsToIndex, w, multipleValuesPerDocument);
IndexReader topLevelReader = w.getReader();
Modified: lucene/dev/branches/lucene3312/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java (original)
+++ lucene/dev/branches/lucene3312/lucene/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java Sat Jul 28 11:27:51 2012
@@ -56,6 +56,7 @@ public class TestHighFreqTerms extends L
dir.close();
dir = null;
reader = null;
+ writer = null;
}
/******************** Tests for getHighFreqTerms **********************************/
Modified: lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java Sat Jul 28 11:27:51 2012
@@ -18,6 +18,8 @@ package org.apache.lucene.queries;
*/
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
import java.util.Arrays;
@@ -325,6 +327,16 @@ public class CustomScoreQuery extends Qu
}
@Override
+ public float freq() throws IOException {
+ return subQueryScorer.freq();
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return Collections.singleton(new ChildScorer(subQueryScorer, "CUSTOM"));
+ }
+
+ @Override
public int advance(int target) throws IOException {
int doc = subQueryScorer.advance(target);
if (doc != NO_MORE_DOCS) {
Modified: lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java Sat Jul 28 11:27:51 2012
@@ -25,6 +25,8 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
import java.util.Map;
@@ -164,6 +166,16 @@ public class BoostedQuery extends Query
return score>Float.NEGATIVE_INFINITY ? score : -Float.MAX_VALUE;
}
+ @Override
+ public float freq() throws IOException {
+ return scorer.freq();
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return Collections.singleton(new ChildScorer(scorer, "CUSTOM"));
+ }
+
public Explanation explain(int doc) throws IOException {
Explanation subQueryExpl = weight.qWeight.explain(readerContext ,doc);
if (!subQueryExpl.isMatch()) {
Modified: lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java Sat Jul 28 11:27:51 2012
@@ -158,6 +158,11 @@ public class FunctionQuery extends Query
return score>Float.NEGATIVE_INFINITY ? score : -Float.MAX_VALUE;
}
+ @Override
+ public float freq() throws IOException {
+ return 1;
+ }
+
public Explanation explain(int doc) throws IOException {
float sc = qWeight * vals.floatVal(doc);
Modified: lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java Sat Jul 28 11:27:51 2012
@@ -82,4 +82,9 @@ public class ValueSourceScorer extends S
public float score() throws IOException {
return values.floatVal(doc);
}
+
+ @Override
+ public float freq() throws IOException {
+ return 1;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/messages/NLS.java Sat Jul 28 11:27:51 2012
@@ -174,13 +174,13 @@ public class NLS {
Locale.getDefault());
if (resourceBundle != null) {
Object obj = resourceBundle.getObject(key);
- if (obj == null)
- System.err.println("WARN: Message with key:" + key + " and locale: "
- + Locale.getDefault() + " not found.");
+ //if (obj == null)
+ // System.err.println("WARN: Message with key:" + key + " and locale: "
+ // + Locale.getDefault() + " not found.");
}
} catch (MissingResourceException e) {
- System.err.println("WARN: Message with key:" + key + " and locale: "
- + Locale.getDefault() + " not found.");
+ //System.err.println("WARN: Message with key:" + key + " and locale: "
+ // + Locale.getDefault() + " not found.");
} catch (Throwable e) {
// ignore all other errors and exceptions
// since this code is just a test to see if the message is present on the
Modified: lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/query/FieldsQuery.java Sat Jul 28 11:27:51 2012
@@ -60,7 +60,7 @@ public class FieldsQuery extends SrndQue
OrQuery oq = new OrQuery(queries,
true /* infix OR for field names */,
OrOperatorName);
- System.out.println(getClass().toString() + ", fields expanded: " + oq.toString()); /* needs testing */
+ // System.out.println(getClass().toString() + ", fields expanded: " + oq.toString()); /* needs testing */
return oq.makeLuceneQueryField(null, qf);
}
}
Modified: lucene/dev/branches/lucene3312/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java (original)
+++ lucene/dev/branches/lucene3312/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java Sat Jul 28 11:27:51 2012
@@ -81,8 +81,8 @@ public class TestMultiPhraseQueryParsing
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
this.upto = 0;
this.lastPos = 0;
}
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Sat Jul 28 11:27:51 2012
@@ -28,7 +28,28 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.spatial.query.SpatialArgs;
/**
- * must be thread safe
+ * The SpatialStrategy encapsulates an approach to indexing and searching based
+ * on shapes.
+ * <p/>
+ * Different implementations will support different features. A strategy should
+ * document these common elements:
+ * <ul>
+ * <li>Can it index more than one shape per field?</li>
+ * <li>What types of shapes can be indexed?</li>
+ * <li>What types of query shapes can be used?</li>
+ * <li>What types of query operations are supported?
+ * This might vary per shape.</li>
+ * <li>Are there caches? Under what circumstances are they used?
+ * Roughly how big are they? Is it segmented by Lucene segments, such as is
+ * done by the Lucene {@link org.apache.lucene.search.FieldCache} and
+ * {@link org.apache.lucene.index.DocValues} (ideal) or is it for the entire
+ * index?
+ * </ul>
+ * <p/>
+ * Note that a SpatialStrategy is not involved with the Lucene stored field
+ * values of shapes, which is immaterial to indexing & search.
+ * <p/>
+ * Thread-safe.
*
* @lucene.experimental
*/
@@ -54,11 +75,6 @@ public abstract class SpatialStrategy {
return ctx;
}
- /** Corresponds with Solr's FieldType.isPolyField(). */
- public boolean isPolyField() {
- return false;
- }
-
/**
* The name of the field or the prefix of them if there are multiple
* fields needed internally.
@@ -69,18 +85,19 @@ public abstract class SpatialStrategy {
}
/**
- * Corresponds with Solr's FieldType.createField().
+ * Returns the IndexableField(s) from the <code>shape</code> that are to be
+ * added to the {@link org.apache.lucene.document.Document}. These fields
+ * are expected to be marked as indexed and not stored.
+ * <p/>
+ * Note: If you want to <i>store</i> the shape as a string for retrieval in
+ * search results, you could add it like this:
+ * <pre>document.add(new StoredField(fieldName,ctx.toString(shape)));</pre>
+ * The particular string representation used doesn't matter to the Strategy
+ * since it doesn't use it.
*
- * This may return a null field if it does not want to make anything.
- * This is reasonable behavior if 'ignoreIncompatibleGeometry=true' and the
- * geometry is incompatible
- */
- public abstract IndexableField createField(Shape shape, boolean index, boolean store);
-
- /** Corresponds with Solr's FieldType.createFields(). */
- public IndexableField[] createFields(Shape shape, boolean index, boolean store) {
- return new IndexableField[] { createField(shape, index, store) };
- }
+ * @return Not null nor will it have null elements.
+ */
+ public abstract IndexableField[] createIndexableFields(Shape shape);
/**
* The value source yields a number that is proportional to the distance between the query shape and indexed data.
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Sat Jul 28 11:27:51 2012
@@ -23,7 +23,7 @@ import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.FunctionQuery;
@@ -41,9 +41,6 @@ import org.apache.lucene.spatial.query.S
import org.apache.lucene.spatial.query.SpatialOperation;
import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
-import java.text.NumberFormat;
-import java.util.Locale;
-
/**
* Based on GeoPortal's
@@ -95,69 +92,19 @@ public class BBoxStrategy extends Spatia
//---------------------------------
@Override
- public IndexableField[] createFields(Shape shape, boolean index, boolean store) {
-
+ public IndexableField[] createIndexableFields(Shape shape) {
Rectangle bbox = shape.getBoundingBox();
- IndexableField[] fields = new IndexableField[store?6:5];
- fields[0] = createDouble(field_minX, bbox.getMinX(), index, store);
- fields[1] = createDouble(field_maxX, bbox.getMaxX(), index, store);
- fields[2] = createDouble(field_minY, bbox.getMinY(), index, store);
- fields[3] = createDouble(field_maxY, bbox.getMaxY(), index, store);
-
- FieldType ft = new FieldType();
- ft.setIndexed(index);
- ft.setStored(store);
- ft.setTokenized(false);
- ft.setOmitNorms(true);
- ft.setIndexOptions(IndexOptions.DOCS_ONLY);
- ft.freeze();
-
- Field xdl = new Field( field_xdl, bbox.getCrossesDateLine()?"T":"F", ft );
- fields[4] = xdl;
- if( store ) {
- FieldType ff = new FieldType();
- ff.setIndexed(false);
- ff.setStored(true);
- ff.setOmitNorms(true);
- ff.setIndexOptions(IndexOptions.DOCS_ONLY);
- ff.freeze();
-
- NumberFormat nf = NumberFormat.getInstance( Locale.ROOT );
- nf.setMaximumFractionDigits( 5 );
- nf.setMinimumFractionDigits( 5 );
- nf.setGroupingUsed(false);
- String ext =
- nf.format( bbox.getMinX() ) + ' ' +
- nf.format( bbox.getMinY() ) + ' ' +
- nf.format( bbox.getMaxX() ) + ' ' +
- nf.format( bbox.getMaxY() ) + ' ';
- fields[5] = new Field( field_bbox, ext, ff );
- }
+ FieldType doubleFieldType = new FieldType(DoubleField.TYPE_NOT_STORED);
+ doubleFieldType.setNumericPrecisionStep(precisionStep);
+ IndexableField[] fields = new IndexableField[5];
+ fields[0] = new DoubleField(field_minX, bbox.getMinX(), doubleFieldType);
+ fields[1] = new DoubleField(field_maxX, bbox.getMaxX(), doubleFieldType);
+ fields[2] = new DoubleField(field_minY, bbox.getMinY(), doubleFieldType);
+ fields[3] = new DoubleField(field_maxY, bbox.getMaxY(), doubleFieldType);
+ fields[4] = new Field( field_xdl, bbox.getCrossesDateLine()?"T":"F", StringField.TYPE_NOT_STORED);
return fields;
}
- private IndexableField createDouble(String name, double v, boolean index, boolean store) {
- if (!store && !index)
- throw new IllegalArgumentException("field must be indexed or stored");
-
- FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED);
- fieldType.setStored(store);
- fieldType.setIndexed(index);
- fieldType.setNumericPrecisionStep(precisionStep);
- return new DoubleField(name,v,fieldType);
- }
-
- @Override
- public IndexableField createField(Shape shape,
- boolean index, boolean store) {
- throw new UnsupportedOperationException("BBOX is poly field");
- }
-
- @Override
- public boolean isPolyField() {
- return true;
- }
-
//---------------------------------
// Query Builder
//---------------------------------
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixCellsTokenizer.java Sat Jul 28 11:27:51 2012
@@ -83,7 +83,7 @@ class PrefixCellsTokenizer extends Token
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
}
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Sat Jul 28 11:27:51 2012
@@ -24,14 +24,14 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.StoredField;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.spatial.SpatialStrategy;
import org.apache.lucene.spatial.prefix.tree.Node;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgs;
-import org.apache.lucene.spatial.util.CachedDistanceValueSource;
+import org.apache.lucene.spatial.util.ShapeFieldCacheDistanceValueSource;
import java.util.Iterator;
import java.util.List;
@@ -63,7 +63,7 @@ public abstract class PrefixTreeStrategy
}
@Override
- public IndexableField createField(Shape shape, boolean index, boolean store) {
+ public IndexableField[] createIndexableFields(Shape shape) {
int detailLevel = grid.getMaxLevelForPrecision(shape,distErrPct);
List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
//If shape isn't a point, add a full-resolution center-point so that
@@ -78,42 +78,19 @@ public abstract class PrefixTreeStrategy
//TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
// http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4
- String fname = getFieldName();
- if( store ) {
- //TODO figure out how to re-use original string instead of reconstituting it.
- String wkt = grid.getSpatialContext().toString(shape);
- if( index ) {
- Field f = new Field(fname,wkt,TYPE_STORED);
- f.setTokenStream(new CellTokenStream(cells.iterator()));
- return f;
- }
- return new StoredField(fname,wkt);
- }
-
- if( index ) {
- return new Field(fname,new CellTokenStream(cells.iterator()),TYPE_NOT_STORED);
- }
-
- throw new UnsupportedOperationException("Fields need to be indexed or store ["+fname+"]");
+ Field field = new Field(getFieldName(), new CellTokenStream(cells.iterator()), FIELD_TYPE);
+ return new IndexableField[]{field};
}
/* Indexed, tokenized, not stored. */
- public static final FieldType TYPE_NOT_STORED = new FieldType();
-
- /* Indexed, tokenized, stored. */
- public static final FieldType TYPE_STORED = new FieldType();
+ public static final FieldType FIELD_TYPE = new FieldType();
static {
- TYPE_NOT_STORED.setIndexed(true);
- TYPE_NOT_STORED.setTokenized(true);
- TYPE_NOT_STORED.setOmitNorms(true);
- TYPE_NOT_STORED.freeze();
-
- TYPE_STORED.setStored(true);
- TYPE_STORED.setIndexed(true);
- TYPE_STORED.setTokenized(true);
- TYPE_STORED.setOmitNorms(true);
- TYPE_STORED.freeze();
+ FIELD_TYPE.setIndexed(true);
+ FIELD_TYPE.setTokenized(true);
+ FIELD_TYPE.setOmitNorms(true);
+ FIELD_TYPE.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY);
+ FIELD_TYPE.freeze();
}
/** Outputs the tokenString of a cell, and if its a leaf, outputs it again with the leaf byte. */
@@ -169,7 +146,7 @@ public abstract class PrefixTreeStrategy
}
}
Point point = args.getShape().getCenter();
- return new CachedDistanceValueSource(point, calc, p);
+ return new ShapeFieldCacheDistanceValueSource(point, calc, p);
}
public SpatialPrefixTree getGrid() {
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,7 @@ import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
import com.spatial4j.core.shape.simple.PointImpl;
+import java.io.PrintStream;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collection;
@@ -107,14 +108,14 @@ public class QuadPrefixTree extends Spat
this(ctx, ctx.getWorldBounds(), maxLevels);
}
- public void printInfo() {
+ public void printInfo(PrintStream out) {
NumberFormat nf = NumberFormat.getNumberInstance(Locale.ROOT);
nf.setMaximumFractionDigits(5);
nf.setMinimumFractionDigits(5);
nf.setMinimumIntegerDigits(3);
for (int i = 0; i < maxLevels; i++) {
- System.out.println(i + "]\t" + nf.format(levelW[i]) + "\t" + nf.format(levelH[i]) + "\t" +
+ out.println(i + "]\t" + nf.format(levelW[i]) + "\t" + nf.format(levelH[i]) + "\t" +
levelS[i] + "\t" + (levelS[i] * levelS[i]));
}
}
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java Sat Jul 28 11:27:51 2012
@@ -24,7 +24,6 @@ import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Rectangle;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.DoubleField;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.queries.function.FunctionQuery;
@@ -78,25 +77,14 @@ public class TwoDoublesStrategy extends
}
@Override
- public boolean isPolyField() {
- return true;
- }
-
- @Override
- public IndexableField[] createFields(Shape shape, boolean index, boolean store) {
+ public IndexableField[] createIndexableFields(Shape shape) {
if( shape instanceof Point ) {
Point point = (Point)shape;
-
- IndexableField[] f = new IndexableField[(index ? 2 : 0) + (store ? 1 : 0)];
- if (index) {
- f[0] = createDouble(fieldNameX, point.getX(), index, store);
- f[1] = createDouble(fieldNameY, point.getY(), index, store);
- }
- if(store) {
- FieldType customType = new FieldType();
- customType.setStored(true);
- f[f.length-1] = new Field( getFieldName(), ctx.toString( shape ), customType );
- }
+ FieldType doubleFieldType = new FieldType(DoubleField.TYPE_NOT_STORED);
+ doubleFieldType.setNumericPrecisionStep(precisionStep);
+ IndexableField[] f = new IndexableField[2];
+ f[0] = new DoubleField(fieldNameX, point.getX(), doubleFieldType);
+ f[1] = new DoubleField(fieldNameY, point.getY(), doubleFieldType);
return f;
}
if( !ignoreIncompatibleGeometry ) {
@@ -105,23 +93,6 @@ public class TwoDoublesStrategy extends
return new IndexableField[0]; // nothing (solr does not support null)
}
- private IndexableField createDouble(String name, double v, boolean index, boolean store) {
- if (!store && !index)
- throw new IllegalArgumentException("field must be indexed or stored");
-
- FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED);
- fieldType.setStored(store);
- fieldType.setIndexed(index);
- fieldType.setNumericPrecisionStep(precisionStep);
- return new DoubleField(name,v,fieldType);
- }
-
- @Override
- public IndexableField createField(Shape shape,
- boolean index, boolean store) {
- throw new UnsupportedOperationException("Point is poly field");
- }
-
@Override
public ValueSource makeValueSource(SpatialArgs args) {
Point p = args.getShape().getCenter();
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/java/overview.html?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/java/overview.html (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/java/overview.html Sat Jul 28 11:27:51 2012
@@ -16,8 +16,49 @@
-->
<html>
<head>
- <title>Apache Lucene Spatial Strategies</title>
+ <title>Apache Lucene Spatial Module</title>
</head>
<body>
+
+ <h1>The Spatial Module for Apache Lucene</h1>
+
+ <p>
+ The spatial module is new is Lucene 4, replacing the old contrib module
+ that came before it. The principle interface to the module is
+ a {@link org.apache.lucene.spatial.SpatialStrategy}
+ which encapsulates an approach to indexing and searching
+ based on shapes. Different Strategies have different features and
+ performance profiles, which are documented at each Strategy class level.
+ </p>
+ <p>
+ For some sample code showing how to use the API, see SpatialExample.java in
+ the tests.
+ </p>
+ <p>
+ The spatial module uses
+ <a href="https://github.com/spatial4j/spatial4j">Spatial4j</a>
+ heavily. Spatial4j is an ASL licensed library with these capabilities:
+ <ul>
+ <li>Provides shape implementations, namely point, rectangle,
+ and circle. Both geospatial contexts and plain 2D Euclidean/Cartesian contexts
+ are supported.
+ With an additional dependency, it adds polygon and other geometry shape
+ support via integration with
+ <a href="http://sourceforge.net/projects/jts-topo-suite/">JTS Topology Suite</a>.
+ This includes dateline wrap support.</li>
+ <li>Shape parsing and serialization, including
+ <a href="http://en.wikipedia.org/wiki/Well-known_text">Well-Known Text (WKT)</a>
+ (via JTS).</li>
+ <li>Distance and other spatial related math calculations.</li>
+ </ul>
+ </p>
+ <p>
+ Historical note: The new spatial module was once known as
+ Lucene Spatial Playground (LSP) as an external project. In ~March 2012, LSP
+ split into this new module as part of Lucene and Spatial4j externally. A
+ large chunk of the LSP implementation originated as SOLR-2155 which uses
+ trie/prefix-tree algorithms with a geohash encoding.
+ </p>
+
</body>
</html>
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/PortedSolr3Test.java Sat Jul 28 11:27:51 2012
@@ -25,6 +25,7 @@ import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.FilteredQuery;
@@ -77,7 +78,7 @@ public class PortedSolr3Test extends Str
static class Param {
SpatialStrategy strategy;
String description;
-
+
Param(SpatialStrategy strategy, String description) {
this.strategy = strategy;
this.description = description;
@@ -193,9 +194,11 @@ public class PortedSolr3Test extends Str
private Document newDoc(String id, Shape shape) {
Document doc = new Document();
doc.add(new StringField("id", id, Field.Store.YES));
- for (IndexableField f : strategy.createFields(shape, true, storeShape)) {
+ for (IndexableField f : strategy.createIndexableFields(shape)) {
doc.add(f);
}
+ if (storeShape)
+ doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
return doc;
}
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/StrategyTestCase.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,7 @@ import com.spatial4j.core.io.sample.Samp
import com.spatial4j.core.shape.Shape;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.spatial.query.SpatialArgsParser;
@@ -84,11 +85,12 @@ public abstract class StrategyTestCase e
document.add(new StringField("id", data.id, Field.Store.YES));
document.add(new StringField("name", data.name, Field.Store.YES));
Shape shape = ctx.readShape(data.shape);
- for (IndexableField f : strategy.createFields(shape, true, storeShape)) {
- if( f != null ) { // null if incompatibleGeometry && ignore
- document.add(f);
- }
+ for (IndexableField f : strategy.createIndexableFields(shape)) {
+ document.add(f);
}
+ if (storeShape)
+ document.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
+
documents.add(document);
}
return documents;
@@ -113,6 +115,10 @@ public abstract class StrategyTestCase e
String msg = q.line; //"Query: " + q.args.toString(ctx);
SearchResults got = executeQuery(strategy.makeQuery(q.args), 100);
+ if (storeShape && got.numFound > 0) {
+ //check stored value is there & parses
+ assertNotNull(ctx.readShape(got.results.get(0).document.get(strategy.getFieldName())));
+ }
if (concern.orderIsImportant) {
Iterator<String> ids = q.ids.iterator();
for (SearchResult r : got.results) {
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestRecursivePrefixTreeStrategy.java Sat Jul 28 11:27:51 2012
@@ -26,6 +26,7 @@ import com.spatial4j.core.shape.simple.P
import com.spatial4j.core.util.GeohashUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.spatial.SpatialMatchConcern;
@@ -153,9 +154,11 @@ public class TestRecursivePrefixTreeStra
private Document newDoc(String id, Shape shape) {
Document doc = new Document();
doc.add(new StringField("id", id, Field.Store.YES));
- for (IndexableField f : strategy.createFields(shape, true, storeShape)) {
+ for (IndexableField f : strategy.createIndexableFields(shape)) {
doc.add(f);
}
+ if (storeShape)
+ doc.add(new StoredField(strategy.getFieldName(), ctx.toString(shape)));
return doc;
}
Modified: lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/spatial/src/test/org/apache/lucene/spatial/prefix/TestTermQueryPrefixGridStrategy.java Sat Jul 28 11:27:51 2012
@@ -23,7 +23,9 @@ import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.simple.PointImpl;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.spatial.SpatialTestCase;
import org.apache.lucene.spatial.prefix.tree.QuadPrefixTree;
import org.apache.lucene.spatial.query.SpatialArgsParser;
@@ -44,7 +46,10 @@ public class TestTermQueryPrefixGridStra
Document losAngeles = new Document();
losAngeles.add(new StringField("name", "Los Angeles", Field.Store.YES));
- losAngeles.add(prefixGridStrategy.createField(point, true, true));
+ for (IndexableField field : prefixGridStrategy.createIndexableFields(point)) {
+ losAngeles.add(field);
+ }
+ losAngeles.add(new StoredField(prefixGridStrategy.getFieldName(), ctx.toString(point)));
addDocumentsAndCommit(Arrays.asList(losAngeles));
Modified: lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java (original)
+++ lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java Sat Jul 28 11:27:51 2012
@@ -128,13 +128,13 @@ public class SortedTermFreqIteratorWrapp
}
private void close() throws IOException {
+ IOUtils.close(reader);
if (tempInput != null) {
tempInput.delete();
}
if (tempSorted != null) {
tempSorted.delete();
}
- IOUtils.close(reader);
}
private final static class BytesOnlyComparator implements Comparator<BytesRef> {
Modified: lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene3312/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Sat Jul 28 11:27:51 2012
@@ -150,6 +150,7 @@ public class FSTCompletionLookup extends
Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
Sort.ByteSequencesReader reader = null;
+ ExternalRefSorter sorter = null;
// Push floats up front before sequences to sort them. For now, assume they are non-negative.
// If negative floats are allowed some trickery needs to be done to find their byte order.
@@ -175,7 +176,7 @@ public class FSTCompletionLookup extends
SortInfo info = new Sort().sort(tempInput, tempSorted);
tempInput.delete();
FSTCompletionBuilder builder = new FSTCompletionBuilder(
- buckets, new ExternalRefSorter(new Sort()), sharedTailLength);
+ buckets, sorter = new ExternalRefSorter(new Sort()), sharedTailLength);
final int inputLines = info.lines;
reader = new Sort.ByteSequencesReader(tempSorted);
@@ -215,9 +216,9 @@ public class FSTCompletionLookup extends
success = true;
} finally {
if (success)
- IOUtils.close(reader, writer);
+ IOUtils.close(reader, writer, sorter);
else
- IOUtils.closeWhileHandlingException(reader, writer);
+ IOUtils.closeWhileHandlingException(reader, writer, sorter);
tempInput.delete();
tempSorted.delete();