You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/13 12:38:54 UTC
svn commit: r1531666 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/suggest/ lucene/suggest/src/java/org/apache/lucene/search/spell/
lucene/suggest/src/java/org/apache/lucene/search/suggest/
lucene/suggest/src/java/org/apache/lucene/searc...
Author: mikemccand
Date: Sun Oct 13 10:38:53 2013
New Revision: 1531666
URL: http://svn.apache.org/r1531666
Log:
LUCENE-5260: cutover all suggesters to TermFreqPayloadIterator
Added:
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java
- copied, changed from r1531664, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java
- copied, changed from r1531664, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java
- copied unchanged from r1531664, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java
- copied, changed from r1531664, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java
Removed:
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sun Oct 13 10:38:53 2013
@@ -97,6 +97,10 @@ API Changes:
numBits parameter to allow growing/shrinking the copied bitset. You can
use FixedBitSet.clone() if you only need to clone the bitset. (Shai Erera)
+* LUCENE-5260: Use TermFreqPayloadIterator for all suggesters; those
+ suggesters that can't support payloads will throw an exception if
+ hasPayloads() is true. (Areek Zillur via Mike McCandless)
+
Optimizations
* LUCENE-5225: The ToParentBlockJoinQuery only keeps tracks of the the child
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Sun Oct 13 10:38:53 2013
@@ -60,7 +60,7 @@ public class HighFrequencyDictionary imp
return new HighFrequencyIterator();
}
- final class HighFrequencyIterator implements TermFreqIterator {
+ final class HighFrequencyIterator implements TermFreqPayloadIterator {
private final BytesRef spare = new BytesRef();
private final TermsEnum termsEnum;
private int minNumDocs;
@@ -108,5 +108,15 @@ public class HighFrequencyDictionary imp
return termsEnum.getComparator();
}
}
+
+ @Override
+ public BytesRef payload() {
+ return null;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java Sun Oct 13 10:38:53 2013
@@ -17,20 +17,73 @@ package org.apache.lucene.search.spell;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Comparator;
+
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
+import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; // javadocs
import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; // javadocs
import org.apache.lucene.search.suggest.analyzing.FuzzySuggester; // javadocs
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
/**
* Interface for enumerating term,weight,payload triples;
- * currently only {@link AnalyzingSuggester} and {@link
- * FuzzySuggester} support payloads.
+ * currently only {@link AnalyzingSuggester}, {@link
+ * FuzzySuggester} and {@link AnalyzingInfixSuggester} support payloads.
*/
-public interface TermFreqPayloadIterator extends TermFreqIterator {
+public interface TermFreqPayloadIterator extends BytesRefIterator {
+ /** A term's weight, higher numbers mean better suggestions. */
+ public long weight();
+
/** An arbitrary byte[] to record per suggestion. See
* {@link LookupResult#payload} to retrieve the payload
* for each suggestion. */
public BytesRef payload();
+
+ /** Returns true if the iterator has payloads */
+ public boolean hasPayloads();
+
+ /**
+ * Wraps a BytesRefIterator as a TermFreqPayloadIterator, with all weights
+ * set to <code>1</code> and carries no payload
+ */
+ public static class TermFreqPayloadIteratorWrapper implements TermFreqPayloadIterator {
+ private final BytesRefIterator wrapped;
+
+ /**
+ * Creates a new wrapper, wrapping the specified iterator and
+ * specifying a weight value of <code>1</code> for all terms
+ * and nullifies associated payloads.
+ */
+ public TermFreqPayloadIteratorWrapper(BytesRefIterator wrapped) {
+ this.wrapped = wrapped;
+ }
+
+ @Override
+ public long weight() {
+ return 1;
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return wrapped.getComparator();
+ }
+
+ @Override
+ public BytesRef next() throws IOException {
+ return wrapped.next();
+ }
+
+ @Override
+ public BytesRef payload() {
+ return null;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
+ }
}
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java (from r1531664, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java&r1=1531664&r2=1531666&rev=1531666&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java Sun Oct 13 10:38:53 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.search.suggest
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.ArrayUtil;
@@ -41,6 +42,7 @@ public class BufferingTermFreqPayloadIte
private final BytesRef spare = new BytesRef();
private final BytesRef payloadSpare = new BytesRef();
private final boolean hasPayloads;
+ private final Comparator<BytesRef> comp;
/** Creates a new iterator, buffering entries from the specified iterator */
public BufferingTermFreqPayloadIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
@@ -57,7 +59,7 @@ public class BufferingTermFreqPayloadIte
}
freqs[freqIndex++] = source.weight();
}
-
+ comp = source.getComparator();
}
@Override
@@ -86,4 +88,9 @@ public class BufferingTermFreqPayloadIte
public boolean hasPayloads() {
return hasPayloads;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return comp;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java Sun Oct 13 10:38:53 2013
@@ -48,12 +48,6 @@ import org.apache.lucene.util.BytesRefIt
* The term, weight and (optionally) payload fields supplied
* are required for ALL documents and has to be stored
* </li>
- * <li>
- * This Dictionary implementation is not compatible with the following Suggesters:
- * {@link JaspellLookup}, {@link TSTLookup}, {@link FSTCompletionLookup},
- * {@link WFSTCompletionLookup} and {@link AnalyzingInfixSuggester}.
- * see https://issues.apache.org/jira/browse/LUCENE-5260
- * </li>
* </ul>
*/
public class DocumentDictionary implements Dictionary {
@@ -96,7 +90,7 @@ public class DocumentDictionary implemen
final class TermWeightPayloadIterator implements TermFreqPayloadIterator {
private final int docCount;
private final Set<String> relevantFields;
- private final boolean withPayload;
+ private final boolean hasPayloads;
private final Bits liveDocs;
private int currentDocId = -1;
private long currentWeight;
@@ -107,13 +101,13 @@ public class DocumentDictionary implemen
* index. setting <code>withPayload</code> to false, implies an iterator
* over only term and weight.
*/
- public TermWeightPayloadIterator(boolean withPayload) throws IOException {
+ public TermWeightPayloadIterator(boolean hasPayloads) throws IOException {
docCount = reader.maxDoc() - 1;
- this.withPayload = withPayload;
+ this.hasPayloads = hasPayloads;
currentPayload = null;
liveDocs = MultiFields.getLiveDocs(reader);
List<String> relevantFieldList;
- if(withPayload) {
+ if(hasPayloads) {
relevantFieldList = Arrays.asList(field, weightField, payloadField);
} else {
relevantFieldList = Arrays.asList(field, weightField);
@@ -141,7 +135,7 @@ public class DocumentDictionary implemen
Document doc = reader.document(currentDocId, relevantFields);
- if (withPayload) {
+ if (hasPayloads) {
IndexableField payload = doc.getField(payloadField);
if (payload == null) {
throw new IllegalArgumentException(payloadField + " does not exist");
@@ -175,6 +169,11 @@ public class DocumentDictionary implemen
public BytesRef payload() {
return currentPayload;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Sun Oct 13 10:38:53 2013
@@ -22,7 +22,7 @@ import java.io.*;
import java.util.Comparator;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqIterator;
+import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -58,11 +58,11 @@ public class FileDictionary implements D
}
@Override
- public TermFreqIterator getWordsIterator() {
+ public TermFreqPayloadIterator getWordsIterator() {
return new FileIterator();
}
- final class FileIterator implements TermFreqIterator {
+ final class FileIterator implements TermFreqPayloadIterator {
private long curFreq;
private final BytesRef spare = new BytesRef();
@@ -104,6 +104,16 @@ public class FileDictionary implements D
public Comparator<BytesRef> getComparator() {
return null;
}
+
+ @Override
+ public BytesRef payload() {
+ return null;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java Sun Oct 13 10:38:53 2013
@@ -24,7 +24,7 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqIterator;
+import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.PriorityQueue;
@@ -154,25 +154,25 @@ public abstract class Lookup {
/** Build lookup from a dictionary. Some implementations may require sorted
* or unsorted keys from the dictionary's iterator - use
- * {@link SortedTermFreqIteratorWrapper} or
- * {@link UnsortedTermFreqIteratorWrapper} in such case.
+ * {@link SortedTermFreqPayloadIteratorWrapper} or
+ * {@link UnsortedTermFreqPayloadIteratorWrapper} in such case.
*/
public void build(Dictionary dict) throws IOException {
BytesRefIterator it = dict.getWordsIterator();
- TermFreqIterator tfit;
- if (it instanceof TermFreqIterator) {
- tfit = (TermFreqIterator)it;
+ TermFreqPayloadIterator tfit;
+ if (it instanceof TermFreqPayloadIterator) {
+ tfit = (TermFreqPayloadIterator)it;
} else {
- tfit = new TermFreqIterator.TermFreqIteratorWrapper(it);
+ tfit = new TermFreqPayloadIterator.TermFreqPayloadIteratorWrapper(it);
}
build(tfit);
}
/**
- * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqIterator}.
+ * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqPayloadIterator}.
* The implementation might re-sort the data internally.
*/
- public abstract void build(TermFreqIterator tfit) throws IOException;
+ public abstract void build(TermFreqPayloadIterator tfit) throws IOException;
/**
* Look up a key and return possible completion for this key.
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java (from r1531664, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java&r1=1531664&r2=1531666&rev=1531666&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java Sun Oct 13 10:38:53 2013
@@ -112,6 +112,11 @@ public class SortedTermFreqPayloadIterat
return hasPayloads;
}
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return comparator;
+ }
+
/** Sortes by BytesRef (ascending) then cost (ascending). */
private final Comparator<BytesRef> tieBreakByCostComparator = new Comparator<BytesRef>() {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Sun Oct 13 10:38:53 2013
@@ -65,7 +65,6 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
import org.apache.lucene.search.suggest.Lookup;
@@ -176,19 +175,14 @@ public class AnalyzingInfixSuggester ext
}
@Override
- public void build(TermFreqIterator iter) throws IOException {
+ public void build(TermFreqPayloadIterator iter) throws IOException {
if (searcher != null) {
searcher.getIndexReader().close();
searcher = null;
}
- TermFreqPayloadIterator payloads;
- if (iter instanceof TermFreqPayloadIterator) {
- payloads = (TermFreqPayloadIterator) iter;
- } else {
- payloads = null;
- }
+
Directory dirTmp = getDirectory(new File(indexPath.toString() + ".tmp"));
IndexWriter w = null;
@@ -236,7 +230,7 @@ public class AnalyzingInfixSuggester ext
doc.add(weightField);
Field payloadField;
- if (payloads != null) {
+ if (iter.hasPayloads()) {
payloadField = new BinaryDocValuesField("payloads", new BytesRef());
doc.add(payloadField);
} else {
@@ -250,8 +244,8 @@ public class AnalyzingInfixSuggester ext
textGramField.setStringValue(textString);
textDVField.setBytesValue(text);
weightField.setLongValue(iter.weight());
- if (payloads != null) {
- payloadField.setBytesValue(payloads.payload());
+ if (iter.hasPayloads()) {
+ payloadField.setBytesValue(iter.payload());
}
w.addDocument(doc);
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Sun Oct 13 10:38:53 2013
@@ -31,7 +31,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenStreamToAutomaton;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
@@ -381,19 +380,13 @@ public class AnalyzingSuggester extends
}
@Override
- public void build(TermFreqIterator iterator) throws IOException {
+ public void build(TermFreqPayloadIterator iterator) throws IOException {
String prefix = getClass().getSimpleName();
File directory = Sort.defaultTempDir();
File tempInput = File.createTempFile(prefix, ".input", directory);
File tempSorted = File.createTempFile(prefix, ".sorted", directory);
- TermFreqPayloadIterator payloads;
- if (iterator instanceof TermFreqPayloadIterator) {
- payloads = (TermFreqPayloadIterator) iterator;
- } else {
- payloads = null;
- }
- hasPayloads = payloads != null;
+ hasPayloads = iterator.hasPayloads();
Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
Sort.ByteSequencesReader reader = null;
@@ -432,7 +425,7 @@ public class AnalyzingSuggester extends
if (surfaceForm.length > (Short.MAX_VALUE-2)) {
throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")");
}
- payload = payloads.payload();
+ payload = iterator.payload();
// payload + surfaceLength (short)
requiredLength += payload.length + 2;
} else {
@@ -470,7 +463,7 @@ public class AnalyzingSuggester extends
writer.close();
// Sort all input/output pairs (required by FST.Builder):
- new Sort(new AnalyzingComparator(payloads != null)).sort(tempInput, tempSorted);
+ new Sort(new AnalyzingComparator(hasPayloads)).sort(tempInput, tempSorted);
// Free disk space:
tempInput.delete();
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Sun Oct 13 10:38:53 2013
@@ -54,7 +54,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
@@ -274,15 +273,15 @@ public class FreeTextSuggester extends L
}
@Override
- public void build(TermFreqIterator iterator) throws IOException {
+ public void build(TermFreqPayloadIterator iterator) throws IOException {
build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
}
/** Build the suggest index, using up to the specified
* amount of temporary RAM while building. Note that
* the weights for the suggestions are ignored. */
- public void build(TermFreqIterator iterator, double ramBufferSizeMB) throws IOException {
- if (iterator instanceof TermFreqPayloadIterator) {
+ public void build(TermFreqPayloadIterator iterator, double ramBufferSizeMB) throws IOException {
+ if (iterator.hasPayloads()) {
throw new IllegalArgumentException("payloads are not supported");
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Sun Oct 13 10:38:53 2013
@@ -24,7 +24,6 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.SortInfo;
@@ -43,7 +42,7 @@ import org.apache.lucene.util.fst.NoOutp
* An adapter from {@link Lookup} API to {@link FSTCompletion}.
*
* <p>This adapter differs from {@link FSTCompletion} in that it attempts
- * to discretize any "weights" as passed from in {@link TermFreqIterator#weight()}
+ * to discretize any "weights" as passed from in {@link TermFreqPayloadIterator#weight()}
* to match the number of buckets. For the rationale for bucketing, see
* {@link FSTCompletion}.
*
@@ -96,7 +95,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqIterator)} method. The number of weight
+ * {@link #build(TermFreqPayloadIterator)} method. The number of weight
* discretization buckets is set to {@link FSTCompletion#DEFAULT_BUCKETS} and
* exact matches are promoted to the top of the suggestions list.
*/
@@ -106,7 +105,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqIterator)} method.
+ * {@link #build(TermFreqPayloadIterator)} method.
*
* @param buckets
* The number of weight discretization buckets (see
@@ -141,8 +140,8 @@ public class FSTCompletionLookup extends
}
@Override
- public void build(TermFreqIterator tfit) throws IOException {
- if (tfit instanceof TermFreqPayloadIterator) {
+ public void build(TermFreqPayloadIterator tfit) throws IOException {
+ if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
File tempInput = File.createTempFile(
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Sun Oct 13 10:38:53 2013
@@ -25,11 +25,10 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
-import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.InputStreamDataInput;
@@ -93,12 +92,12 @@ public class WFSTCompletionLookup extend
}
@Override
- public void build(TermFreqIterator iterator) throws IOException {
- if (iterator instanceof TermFreqPayloadIterator) {
+ public void build(TermFreqPayloadIterator iterator) throws IOException {
+ if (iterator.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
BytesRef scratch = new BytesRef();
- TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
+ TermFreqPayloadIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
IntsRef scratchInts = new IntsRef();
BytesRef previous = null;
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
@@ -255,14 +254,15 @@ public class WFSTCompletionLookup extend
return Integer.MAX_VALUE - (int)value;
}
- private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqIteratorWrapper {
+ private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqPayloadIteratorWrapper {
- WFSTTermFreqIteratorWrapper(TermFreqIterator source) throws IOException {
+ WFSTTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
super(source);
+ assert source.hasPayloads() == false;
}
@Override
- protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException {
+ protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, long weight) throws IOException {
if (spare.length + 4 >= buffer.length) {
buffer = ArrayUtil.grow(buffer, spare.length + 4);
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Sun Oct 13 10:38:53 2013
@@ -25,10 +25,9 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.UnsortedTermFreqIteratorWrapper;
+import org.apache.lucene.search.suggest.UnsortedTermFreqPayloadIteratorWrapper;
import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
@@ -48,19 +47,19 @@ public class JaspellLookup extends Looku
/**
* Creates a new empty trie
- * @see #build(TermFreqIterator)
+ * @see #build(TermFreqPayloadIterator)
* */
public JaspellLookup() {}
@Override
- public void build(TermFreqIterator tfit) throws IOException {
- if (tfit instanceof TermFreqPayloadIterator) {
+ public void build(TermFreqPayloadIterator tfit) throws IOException {
+ if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
if (tfit.getComparator() != null) {
// make sure it's unsorted
// WTF - this could result in yet another sorted iteration....
- tfit = new UnsortedTermFreqIteratorWrapper(tfit);
+ tfit = new UnsortedTermFreqPayloadIteratorWrapper(tfit);
}
trie = new JaspellTernarySearchTrie();
trie.setMatchAlmostDiff(editDistance);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Sun Oct 13 10:38:53 2013
@@ -25,10 +25,9 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
@@ -46,20 +45,20 @@ public class TSTLookup extends Lookup {
/**
* Creates a new TSTLookup with an empty Ternary Search Tree.
- * @see #build(TermFreqIterator)
+ * @see #build(TermFreqPayloadIterator)
*/
public TSTLookup() {}
@Override
- public void build(TermFreqIterator tfit) throws IOException {
- if (tfit instanceof TermFreqPayloadIterator) {
+ public void build(TermFreqPayloadIterator tfit) throws IOException {
+ if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
root = new TernaryTreeNode();
// buffer first
if (tfit.getComparator() != BytesRef.getUTF8SortedAsUTF16Comparator()) {
// make sure it's sorted and the comparator uses UTF16 sort order
- tfit = new SortedTermFreqIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
+ tfit = new SortedTermFreqPayloadIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
}
ArrayList<String> tokens = new ArrayList<String>();
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Sun Oct 13 10:38:53 2013
@@ -72,12 +72,12 @@ public class LookupBenchmarkTest extends
/**
* Input term/weight pairs.
*/
- private static TermFreq [] dictionaryInput;
+ private static TermFreqPayload [] dictionaryInput;
/**
* Benchmark term/weight pairs (randomized order).
*/
- private static List<TermFreq> benchmarkInput;
+ private static List<TermFreqPayload> benchmarkInput;
/**
* Loads terms and frequencies from Wikipedia (cached).
@@ -85,9 +85,9 @@ public class LookupBenchmarkTest extends
@BeforeClass
public static void setup() throws Exception {
assert false : "disable assertions before running benchmarks!";
- List<TermFreq> input = readTop50KWiki();
+ List<TermFreqPayload> input = readTop50KWiki();
Collections.shuffle(input, random);
- LookupBenchmarkTest.dictionaryInput = input.toArray(new TermFreq [input.size()]);
+ LookupBenchmarkTest.dictionaryInput = input.toArray(new TermFreqPayload [input.size()]);
Collections.shuffle(input, random);
LookupBenchmarkTest.benchmarkInput = input;
}
@@ -97,8 +97,8 @@ public class LookupBenchmarkTest extends
/**
* Collect the multilingual input for benchmarks/ tests.
*/
- public static List<TermFreq> readTop50KWiki() throws Exception {
- List<TermFreq> input = new ArrayList<TermFreq>();
+ public static List<TermFreqPayload> readTop50KWiki() throws Exception {
+ List<TermFreqPayload> input = new ArrayList<TermFreqPayload>();
URL resource = LookupBenchmarkTest.class.getResource("Top50KWiki.utf8");
assert resource != null : "Resource missing: Top50KWiki.utf8";
@@ -109,7 +109,7 @@ public class LookupBenchmarkTest extends
assertTrue("No | separator?: " + line, tab >= 0);
int weight = Integer.parseInt(line.substring(tab + 1));
String key = line.substring(0, tab);
- input.add(new TermFreq(key, weight));
+ input.add(new TermFreqPayload(key, weight));
}
br.close();
return input;
@@ -163,7 +163,7 @@ public class LookupBenchmarkTest extends
/**
* Create {@link Lookup} instance and populate it.
*/
- private Lookup buildLookup(Class<? extends Lookup> cls, TermFreq[] input) throws Exception {
+ private Lookup buildLookup(Class<? extends Lookup> cls, TermFreqPayload[] input) throws Exception {
Lookup lookup = null;
try {
lookup = cls.newInstance();
@@ -176,7 +176,7 @@ public class LookupBenchmarkTest extends
lookup = ctor.newInstance(a);
}
}
- lookup.build(new TermFreqArrayIterator(input));
+ lookup.build(new TermFreqPayloadArrayIterator(input));
return lookup;
}
@@ -220,7 +220,7 @@ public class LookupBenchmarkTest extends
final Lookup lookup = buildLookup(cls, dictionaryInput);
final List<String> input = new ArrayList<String>(benchmarkInput.size());
- for (TermFreq tf : benchmarkInput) {
+ for (TermFreqPayload tf : benchmarkInput) {
String s = tf.term.utf8ToString();
String sub = s.substring(0, Math.min(s.length(),
minPrefixLen + random.nextInt(maxPrefixLen - minPrefixLen + 1)));
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java Sun Oct 13 10:38:53 2013
@@ -65,10 +65,10 @@ public class PersistenceTest extends Luc
// Add all input keys.
Lookup lookup = lookupClass.newInstance();
- TermFreq[] keys = new TermFreq[this.keys.length];
+ TermFreqPayload[] keys = new TermFreqPayload[this.keys.length];
for (int i = 0; i < keys.length; i++)
- keys[i] = new TermFreq(this.keys[i], i);
- lookup.build(new TermFreqArrayIterator(keys));
+ keys[i] = new TermFreqPayload(this.keys[i], i);
+ lookup.build(new TermFreqPayloadArrayIterator(keys));
// Store the suggester.
File storeDir = TEMP_DIR;
@@ -81,7 +81,7 @@ public class PersistenceTest extends Luc
// Assert validity.
Random random = random();
long previous = Long.MIN_VALUE;
- for (TermFreq k : keys) {
+ for (TermFreqPayload k : keys) {
List<LookupResult> list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1);
assertEquals(1, list.size());
LookupResult lookupResult = list.get(0);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java Sun Oct 13 10:38:53 2013
@@ -23,14 +23,32 @@ public final class TermFreqPayload {
public final BytesRef term;
public final long v;
public final BytesRef payload;
+ public final boolean hasPayloads;
+ public TermFreqPayload(BytesRef term, long v, BytesRef payload) {
+ this(term, v, payload, true);
+ }
+
public TermFreqPayload(String term, long v, BytesRef payload) {
- this(new BytesRef(term), v, payload);
+ this(new BytesRef(term), v, payload, true);
}
- public TermFreqPayload(BytesRef term, long v, BytesRef payload) {
+ public TermFreqPayload(BytesRef term, long v) {
+ this(term, v, null, false);
+ }
+
+ public TermFreqPayload(String term, long v) {
+ this(new BytesRef(term), v, null, false);
+ }
+
+ public TermFreqPayload(BytesRef term, long v, BytesRef payload, boolean hasPayloads) {
this.term = term;
this.v = v;
this.payload = payload;
+ this.hasPayloads = hasPayloads;
+ }
+
+ public boolean hasPayloads() {
+ return hasPayloads;
}
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java Sun Oct 13 10:38:53 2013
@@ -21,26 +21,33 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
-import org.apache.lucene.search.spell.TermFreqIterator;
import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
/**
- * A {@link TermFreqIterator} over a sequence of {@link TermFreq}s.
+ * A {@link TermFreqPayloadIterator} over a sequence of {@link TermFreqPayload}s.
*/
public final class TermFreqPayloadArrayIterator implements TermFreqPayloadIterator {
private final Iterator<TermFreqPayload> i;
+ private final boolean hasPayloads;
+ private boolean first;
private TermFreqPayload current;
private final BytesRef spare = new BytesRef();
public TermFreqPayloadArrayIterator(Iterator<TermFreqPayload> i) {
this.i = i;
+ if (i.hasNext()) {
+ current = i.next();
+ first = true;
+ this.hasPayloads = current.hasPayloads;
+ } else {
+ this.hasPayloads = false;
+ }
}
public TermFreqPayloadArrayIterator(TermFreqPayload[] i) {
this(Arrays.asList(i));
}
-
public TermFreqPayloadArrayIterator(Iterable<TermFreqPayload> i) {
this(i.iterator());
}
@@ -52,8 +59,12 @@ public final class TermFreqPayloadArrayI
@Override
public BytesRef next() {
- if (i.hasNext()) {
- current = i.next();
+ if (i.hasNext() || (first && current!=null)) {
+ if (first) {
+ first = false;
+ } else {
+ current = i.next();
+ }
spare.copyBytes(current.term);
return spare;
}
@@ -69,4 +80,9 @@ public final class TermFreqPayloadArrayI
public Comparator<BytesRef> getComparator() {
return null;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
}
\ No newline at end of file
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java (from r1531664, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java&r1=1531664&r2=1531666&rev=1531666&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java Sun Oct 13 10:38:53 2013
@@ -47,8 +47,8 @@ public class TestTermFreqPayloadIterator
int num = atLeast(10000);
Comparator<BytesRef> comparator = random.nextBoolean() ? BytesRef.getUTF8SortedAsUnicodeComparator() : BytesRef.getUTF8SortedAsUTF16Comparator();
- TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> sorted = new TreeMap<>(comparator);
- TreeMap<BytesRef, Long> sortedWithoutPayload = new TreeMap<>(comparator);
+ TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> sorted = new TreeMap<BytesRef,SimpleEntry<Long,BytesRef>>(comparator);
+ TreeMap<BytesRef, Long> sortedWithoutPayload = new TreeMap<BytesRef,Long>(comparator);
TermFreqPayload[] unsorted = new TermFreqPayload[num];
TermFreqPayload[] unsortedWithoutPayload = new TermFreqPayload[num];
@@ -61,7 +61,7 @@ public class TestTermFreqPayloadIterator
} while (sorted.containsKey(key));
long value = random.nextLong();
sortedWithoutPayload.put(key, value);
- sorted.put(key, new SimpleEntry<>(value, payload));
+ sorted.put(key, new SimpleEntry<Long,BytesRef>(value, payload));
unsorted[i] = new TermFreqPayload(key, value, payload);
unsortedWithoutPayload[i] = new TermFreqPayload(key, value);
}
@@ -80,12 +80,12 @@ public class TestTermFreqPayloadIterator
// test the unsorted iterator wrapper with payloads
wrapper = new UnsortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsorted));
- TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> actual = new TreeMap<>();
+ TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> actual = new TreeMap<BytesRef,SimpleEntry<Long,BytesRef>>();
BytesRef key;
while ((key = wrapper.next()) != null) {
long value = wrapper.weight();
BytesRef payload = wrapper.payload();
- actual.put(BytesRef.deepCopyOf(key), new SimpleEntry<>(value, BytesRef.deepCopyOf(payload)));
+ actual.put(BytesRef.deepCopyOf(key), new SimpleEntry<Long,BytesRef>(value, BytesRef.deepCopyOf(payload)));
}
assertEquals(sorted, actual);
@@ -103,7 +103,7 @@ public class TestTermFreqPayloadIterator
// test the unsorted iterator wrapper without payloads
wrapperWithoutPayload = new UnsortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsortedWithoutPayload));
- TreeMap<BytesRef, Long> actualWithoutPayload = new TreeMap<>();
+ TreeMap<BytesRef, Long> actualWithoutPayload = new TreeMap<BytesRef,Long>();
while ((key = wrapperWithoutPayload.next()) != null) {
long value = wrapperWithoutPayload.weight();
assertNull(wrapperWithoutPayload.payload());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Sun Oct 13 10:38:53 2013
@@ -52,8 +52,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreq;
-import org.apache.lucene.search.suggest.TermFreqArrayIterator;
import org.apache.lucene.search.suggest.TermFreqPayload;
import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
@@ -65,18 +63,18 @@ public class AnalyzingSuggesterTest exte
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo", 50),
- new TermFreq("bar", 10),
- new TermFreq("barbar", 10),
- new TermFreq("barbar", 12),
- new TermFreq("barbara", 6),
- new TermFreq("bar", 5),
- new TermFreq("barbara", 1)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo", 50),
+ new TermFreqPayload("bar", 10),
+ new TermFreqPayload("barbar", 10),
+ new TermFreqPayload("barbar", 12),
+ new TermFreqPayload("barbara", 6),
+ new TermFreqPayload("bar", 5),
+ new TermFreqPayload("barbara", 1)
);
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
@@ -165,14 +163,14 @@ public class AnalyzingSuggesterTest exte
public void testRandomRealisticKeys() throws IOException {
LineFileDocs lineFile = new LineFileDocs(random());
Map<String, Long> mapping = new HashMap<String, Long>();
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
int howMany = atLeast(100); // this might bring up duplicates
for (int i = 0; i < howMany; i++) {
Document nextDoc = lineFile.nextDoc();
String title = nextDoc.getField("title").stringValue();
int randomWeight = random().nextInt(100);
- keys.add(new TermFreq(title, randomWeight));
+ keys.add(new TermFreqPayload(title, randomWeight));
if (!mapping.containsKey(title) || mapping.get(title) < randomWeight) {
mapping.put(title, Long.valueOf(randomWeight));
}
@@ -183,15 +181,15 @@ public class AnalyzingSuggesterTest exte
boolean doPayloads = random().nextBoolean();
if (doPayloads) {
List<TermFreqPayload> keysAndPayloads = new ArrayList<TermFreqPayload>();
- for (TermFreq termFreq : keys) {
+ for (TermFreqPayload termFreq : keys) {
keysAndPayloads.add(new TermFreqPayload(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
}
analyzingSuggester.build(new TermFreqPayloadArrayIterator(keysAndPayloads));
} else {
- analyzingSuggester.build(new TermFreqArrayIterator(keys));
+ analyzingSuggester.build(new TermFreqPayloadArrayIterator(keys));
}
- for (TermFreq termFreq : keys) {
+ for (TermFreqPayload termFreq : keys) {
List<LookupResult> lookup = analyzingSuggester.lookup(termFreq.term.utf8ToString(), false, keys.size());
for (LookupResult lookupResult : lookup) {
assertEquals(mapping.get(lookupResult.key), Long.valueOf(lookupResult.value));
@@ -211,14 +209,14 @@ public class AnalyzingSuggesterTest exte
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("the ghost of christmas past", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -241,23 +239,23 @@ public class AnalyzingSuggesterTest exte
public void testEmpty() throws Exception {
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
- suggester.build(new TermFreqArrayIterator(new TermFreq[0]));
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}
public void testNoSeps() throws Exception {
- TermFreq[] keys = new TermFreq[] {
- new TermFreq("ab cd", 0),
- new TermFreq("abcd", 1),
+ TermFreqPayload[] keys = new TermFreqPayload[] {
+ new TermFreqPayload("ab cd", 0),
+ new TermFreqPayload("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -318,13 +316,13 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("wifi network is slow", 50),
- new TermFreq("wi fi network is fast", 10),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("wifi network is slow", 50),
+ new TermFreqPayload("wi fi network is fast", 10),
};
//AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
System.out.println("Results: " + results);
@@ -384,12 +382,12 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("ab xc", 50),
- new TermFreq("ba xd", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("ab xc", 50),
+ new TermFreqPayload("ba xd", 50),
};
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -462,11 +460,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -502,11 +500,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -657,12 +655,12 @@ public class AnalyzingSuggesterTest exte
boolean doPayloads = random().nextBoolean();
- TermFreq[] keys = null;
+ TermFreqPayload[] keys = null;
TermFreqPayload[] payloadKeys = null;
if (doPayloads) {
payloadKeys = new TermFreqPayload[numQueries];
} else {
- keys = new TermFreq[numQueries];
+ keys = new TermFreqPayload[numQueries];
}
boolean preserveSep = random().nextBoolean();
@@ -735,7 +733,7 @@ public class AnalyzingSuggesterTest exte
payload = new BytesRef(bytes);
payloadKeys[i] = new TermFreqPayload(key, weight, payload);
} else {
- keys[i] = new TermFreq(key, weight);
+ keys[i] = new TermFreqPayload(key, weight);
payload = null;
}
@@ -758,7 +756,7 @@ public class AnalyzingSuggesterTest exte
if (doPayloads) {
suggester.build(new TermFreqPayloadArrayIterator(shuffle(payloadKeys)));
} else {
- suggester.build(new TermFreqArrayIterator(shuffle(keys)));
+ suggester.build(new TermFreqPayloadArrayIterator(shuffle(keys)));
}
for (String prefix : allPrefixes) {
@@ -876,8 +874,8 @@ public class AnalyzingSuggesterTest exte
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1);
- suggester.build(new TermFreqArrayIterator(shuffle(new TermFreq("a", 40),
- new TermFreq("a ", 50), new TermFreq(" a", 60))));
+ suggester.build(new TermFreqPayloadArrayIterator(shuffle(new TermFreqPayload("a", 40),
+ new TermFreqPayload("a ", 50), new TermFreqPayload(" a", 60))));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -891,11 +889,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("a", 2),
- new TermFreq("a b c", 3),
- new TermFreq("a c a", 1),
- new TermFreq("a c b", 1),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("a", 2),
+ new TermFreqPayload("a b c", 3),
+ new TermFreqPayload("a c a", 1),
+ new TermFreqPayload("a c b", 1),
}));
suggester.lookup("a", false, 4);
@@ -907,10 +905,10 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("a", 5),
- new TermFreq("a b", 3),
- new TermFreq("a c", 4),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("a", 5),
+ new TermFreqPayload("a b", 3),
+ new TermFreqPayload("a c", 4),
}));
List<LookupResult> results = suggester.lookup("a", false, 3);
@@ -972,9 +970,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqArrayIterator(shuffle(
- new TermFreq("hambone", 6),
- new TermFreq("nellie", 5))));
+ suggester.build(new TermFreqPayloadArrayIterator(shuffle(
+ new TermFreqPayload("hambone", 6),
+ new TermFreqPayload("nellie", 5))));
List<LookupResult> results = suggester.lookup("nellie", false, 2);
assertEquals(2, results.size());
@@ -1041,9 +1039,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("a", 6),
- new TermFreq("b", 5),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("a", 6),
+ new TermFreqPayload("b", 5),
}));
List<LookupResult> results = suggester.lookup("a", false, 2);
@@ -1114,21 +1112,21 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("a a", 50),
- new TermFreq("a b", 50),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("a a", 50),
+ new TermFreqPayload("a b", 50),
}));
}
public void testDupSurfaceFormsMissingResults3() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("a a", 7),
- new TermFreq("a a", 7),
- new TermFreq("a c", 6),
- new TermFreq("a c", 3),
- new TermFreq("a b", 5),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("a a", 7),
+ new TermFreqPayload("a a", 7),
+ new TermFreqPayload("a c", 6),
+ new TermFreqPayload("a c", 3),
+ new TermFreqPayload("a b", 5),
}));
assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString());
}
@@ -1136,9 +1134,9 @@ public class AnalyzingSuggesterTest exte
public void testEndingSpace() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("i love lucy", 7),
- new TermFreq("isla de muerta", 8),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("i love lucy", 7),
+ new TermFreqPayload("isla de muerta", 8),
}));
assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
@@ -1169,15 +1167,15 @@ public class AnalyzingSuggesterTest exte
};
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {new TermFreq("a", 1)}));
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {new TermFreqPayload("a", 1)}));
assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
}
public void testIllegalLookupArgument() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("а где ÐÑÑи?", 7),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("а где ÐÑÑи?", 7),
}));
try {
suggester.lookup("а\u001E", false, 3);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1531666&r1=1531665&r2=1531666&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Sun Oct 13 10:38:53 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreq;
-import org.apache.lucene.search.suggest.TermFreqArrayIterator;
+import org.apache.lucene.search.suggest.TermFreqPayload;
+import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -54,16 +54,16 @@ import org.apache.lucene.util.fst.Util;
public class FuzzySuggesterTest extends LuceneTestCase {
public void testRandomEdits() throws IOException {
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreq("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new TermFreqPayload("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreq("foo bar boo far", 12));
+ keys.add(new TermFreqPayload("foo bar boo far", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("foo bar boo", FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX);
@@ -75,16 +75,16 @@ public class FuzzySuggesterTest extends
}
public void testNonLatinRandomEdits() throws IOException {
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreq("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new TermFreqPayload("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreq("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
+ keys.add(new TermFreqPayload("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ", 0);
@@ -97,15 +97,15 @@ public class FuzzySuggesterTest extends
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("foo", 50),
- new TermFreq("bar", 10),
- new TermFreq("barbar", 12),
- new TermFreq("barbara", 6)
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("foo", 50),
+ new TermFreqPayload("bar", 10),
+ new TermFreqPayload("barbar", 12),
+ new TermFreqPayload("barbara", 6)
};
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("bariar", random()), false, 2);
assertEquals(2, results.size());
@@ -172,14 +172,14 @@ public class FuzzySuggesterTest extends
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("the ghost of christmas past", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
FuzzySuggester suggester = new FuzzySuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -200,16 +200,16 @@ public class FuzzySuggesterTest extends
}
public void testNoSeps() throws Exception {
- TermFreq[] keys = new TermFreq[] {
- new TermFreq("ab cd", 0),
- new TermFreq("abcd", 1),
+ TermFreqPayload[] keys = new TermFreqPayload[] {
+ new TermFreqPayload("ab cd", 0),
+ new TermFreqPayload("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -270,12 +270,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("wifi network is slow", 50),
- new TermFreq("wi fi network is fast", 10),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("wifi network is slow", 50),
+ new TermFreqPayload("wi fi network is fast", 10),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
@@ -290,7 +290,7 @@ public class FuzzySuggesterTest extends
public void testEmpty() throws Exception {
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqArrayIterator(new TermFreq[0]));
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
@@ -344,12 +344,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("ab xc", 50),
- new TermFreq("ba xd", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("ab xc", 50),
+ new TermFreqPayload("ba xd", 50),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -418,11 +418,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -458,11 +458,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -491,19 +491,19 @@ public class FuzzySuggesterTest extends
}
// Holds surface form separately:
- private static class TermFreq2 implements Comparable<TermFreq2> {
+ private static class TermFreqPayload2 implements Comparable<TermFreqPayload2> {
public final String surfaceForm;
public final String analyzedForm;
public final long weight;
- public TermFreq2(String surfaceForm, String analyzedForm, long weight) {
+ public TermFreqPayload2(String surfaceForm, String analyzedForm, long weight) {
this.surfaceForm = surfaceForm;
this.analyzedForm = analyzedForm;
this.weight = weight;
}
@Override
- public int compareTo(TermFreq2 other) {
+ public int compareTo(TermFreqPayload2 other) {
int cmp = analyzedForm.compareTo(other.analyzedForm);
if (cmp != 0) {
return cmp;
@@ -596,11 +596,11 @@ public class FuzzySuggesterTest extends
int numQueries = atLeast(100);
- final List<TermFreq2> slowCompletor = new ArrayList<TermFreq2>();
+ final List<TermFreqPayload2> slowCompletor = new ArrayList<TermFreqPayload2>();
final TreeSet<String> allPrefixes = new TreeSet<String>();
final Set<String> seen = new HashSet<String>();
- TermFreq[] keys = new TermFreq[numQueries];
+ TermFreqPayload[] keys = new TermFreqPayload[numQueries];
boolean preserveSep = random().nextBoolean();
boolean unicodeAware = random().nextBoolean();
@@ -666,17 +666,17 @@ public class FuzzySuggesterTest extends
}
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
- keys[i] = new TermFreq(key, weight);
+ keys[i] = new TermFreqPayload(key, weight);
- slowCompletor.add(new TermFreq2(key, analyzedKey, weight));
+ slowCompletor.add(new TermFreqPayload2(key, analyzedKey, weight));
}
if (VERBOSE) {
// Don't just sort original list, to avoid VERBOSE
// altering the test:
- List<TermFreq2> sorted = new ArrayList<TermFreq2>(slowCompletor);
+ List<TermFreqPayload2> sorted = new ArrayList<TermFreqPayload2>(slowCompletor);
Collections.sort(sorted);
- for(TermFreq2 ent : sorted) {
+ for(TermFreqPayload2 ent : sorted) {
System.out.println(" surface='" + ent.surfaceForm + " analyzed='" + ent.analyzedForm + "' weight=" + ent.weight);
}
}
@@ -684,7 +684,7 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
FuzzySuggester suggester = new FuzzySuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, 1, false, 1, 3, unicodeAware);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
for (String prefix : allPrefixes) {
@@ -756,7 +756,7 @@ public class FuzzySuggesterTest extends
assertTrue(automaton.isDeterministic());
// TODO: could be faster... but its slowCompletor for a reason
BytesRef spare = new BytesRef();
- for (TermFreq2 e : slowCompletor) {
+ for (TermFreqPayload2 e : slowCompletor) {
spare.copyChars(e.analyzedForm);
Set<IntsRef> finiteStrings = suggester.toFiniteStrings(spare, tokenStreamToAutomaton);
for (IntsRef intsRef : finiteStrings) {
@@ -825,14 +825,14 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, 1, true, 1, 3, false);
- List<TermFreq> keys = Arrays.asList(new TermFreq[] {
- new TermFreq("a", 40),
- new TermFreq("a ", 50),
- new TermFreq(" a", 60),
+ List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
+ new TermFreqPayload("a", 40),
+ new TermFreqPayload("a ", 50),
+ new TermFreqPayload(" a", 60),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -846,15 +846,15 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, 2, true, 1, 3, false);
- List<TermFreq> keys = Arrays.asList(new TermFreq[] {
- new TermFreq("foo bar", 40),
- new TermFreq("foo bar baz", 50),
- new TermFreq("barbaz", 60),
- new TermFreq("barbazfoo", 10),
+ List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
+ new TermFreqPayload("foo bar", 40),
+ new TermFreqPayload("foo bar baz", 50),
+ new TermFreqPayload("barbaz", 60),
+ new TermFreqPayload("barbazfoo", 10),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
assertEquals("[foo bar baz/50, foo bar/40]", suggester.lookup("foobar", false, 5).toString());
assertEquals("[foo bar baz/50]", suggester.lookup("foobarbaz", false, 5).toString());
@@ -929,25 +929,25 @@ public class FuzzySuggesterTest extends
public void testRandom2() throws Throwable {
final int NUM = atLeast(200);
- final List<TermFreq> answers = new ArrayList<TermFreq>();
+ final List<TermFreqPayload> answers = new ArrayList<TermFreqPayload>();
final Set<String> seen = new HashSet<String>();
for(int i=0;i<NUM;i++) {
final String s = randomSimpleString(8);
if (!seen.contains(s)) {
- answers.add(new TermFreq(s, random().nextInt(1000)));
+ answers.add(new TermFreqPayload(s, random().nextInt(1000)));
seen.add(s);
}
}
- Collections.sort(answers, new Comparator<TermFreq>() {
+ Collections.sort(answers, new Comparator<TermFreqPayload>() {
@Override
- public int compare(TermFreq a, TermFreq b) {
+ public int compare(TermFreqPayload a, TermFreqPayload b) {
return a.term.compareTo(b.term);
}
});
if (VERBOSE) {
System.out.println("\nTEST: targets");
- for(TermFreq tf : answers) {
+ for(TermFreqPayload tf : answers) {
System.out.println(" " + tf.term.utf8ToString() + " freq=" + tf.v);
}
}
@@ -965,7 +965,7 @@ public class FuzzySuggesterTest extends
}
Collections.shuffle(answers, random());
- suggest.build(new TermFreqArrayIterator(answers.toArray(new TermFreq[answers.size()])));
+ suggest.build(new TermFreqPayloadArrayIterator(answers.toArray(new TermFreqPayload[answers.size()])));
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
@@ -1004,10 +1004,10 @@ public class FuzzySuggesterTest extends
}
}
- private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<TermFreq> answers, String frag) {
+ private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<TermFreqPayload> answers, String frag) {
final List<LookupResult> results = new ArrayList<LookupResult>();
final int fragLen = frag.length();
- for(TermFreq tf : answers) {
+ for(TermFreqPayload tf : answers) {
//System.out.println(" check s=" + tf.term.utf8ToString());
boolean prefixMatches = true;
for(int i=0;i<prefixLen;i++) {