You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/14 19:49:14 UTC
svn commit: r1532001 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/suggest/ lucene/suggest/src/java/org/apache/lucene/search/spell/
lucene/suggest/src/java/org/apache/lucene/search/suggest/
lucene/suggest/src/java/org/apache/lucene/searc...
Author: mikemccand
Date: Mon Oct 14 17:49:13 2013
New Revision: 1532001
URL: http://svn.apache.org/r1532001
Log:
LUCENE-5280: rename TermFreqPayloadIterator -> InputIterator
Added:
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java
- copied, changed from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java
- copied, changed from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
- copied, changed from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java
- copied unchanged from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java
- copied unchanged from r1531987, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java
- copied, changed from r1531987, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
- copied, changed from r1531987, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
Removed:
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon Oct 14 17:49:13 2013
@@ -101,6 +101,10 @@ API Changes:
suggesters that can't support payloads will throw an exception if
hasPayloads() is true. (Areek Zillur via Mike McCandless)
+* LUCENE-5280: Rename TermFreqPayloadIterator -> InputIterator, along
+ with associated suggest/spell classes. (Areek Zillur via Mike
+ McCandless)
+
Optimizations
* LUCENE-5225: The ToParentBlockJoinQuery only keeps tracks of the the child
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Mon Oct 14 17:49:13 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.BytesRef;
@@ -60,7 +61,7 @@ public class HighFrequencyDictionary imp
return new HighFrequencyIterator();
}
- final class HighFrequencyIterator implements TermFreqPayloadIterator {
+ final class HighFrequencyIterator implements InputIterator {
private final BytesRef spare = new BytesRef();
private final TermsEnum termsEnum;
private int minNumDocs;
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java (from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java&r1=1531987&r2=1532001&rev=1532001&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java Mon Oct 14 17:49:13 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.search.suggest
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -40,7 +41,8 @@ public class BufferedInputIterator imple
private final BytesRef spare = new BytesRef();
private final BytesRef payloadSpare = new BytesRef();
private final boolean hasPayloads;
-
+ private final Comparator<BytesRef> comp;
+
/** Creates a new iterator, buffering entries from the specified iterator */
public BufferedInputIterator(InputIterator source) throws IOException {
BytesRef spare;
@@ -56,7 +58,7 @@ public class BufferedInputIterator imple
}
freqs[freqIndex++] = source.weight();
}
-
+ comp = source.getComparator();
}
@Override
@@ -85,4 +87,9 @@ public class BufferedInputIterator imple
public boolean hasPayloads() {
return hasPayloads;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return comp;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java Mon Oct 14 17:49:13 2013
@@ -28,12 +28,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
-import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; // javadoc
-import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; // javadoc
-import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup; // javadoc
-import org.apache.lucene.search.suggest.jaspell.JaspellLookup; // javadoc
-import org.apache.lucene.search.suggest.tst.TSTLookup; // javadoc
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -84,10 +78,10 @@ public class DocumentDictionary implemen
@Override
public BytesRefIterator getWordsIterator() throws IOException {
- return new TermWeightPayloadIterator(payloadField!=null);
+ return new DocumentInputIterator(payloadField!=null);
}
- final class TermWeightPayloadIterator implements TermFreqPayloadIterator {
+ final class DocumentInputIterator implements InputIterator {
private final int docCount;
private final Set<String> relevantFields;
private final boolean hasPayloads;
@@ -101,7 +95,7 @@ public class DocumentDictionary implemen
* index. setting <code>withPayload</code> to false, implies an iterator
* over only term and weight.
*/
- public TermWeightPayloadIterator(boolean hasPayloads) throws IOException {
+ public DocumentInputIterator(boolean hasPayloads) throws IOException {
docCount = reader.maxDoc() - 1;
this.hasPayloads = hasPayloads;
currentPayload = null;
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Mon Oct 14 17:49:13 2013
@@ -22,7 +22,6 @@ import java.io.*;
import java.util.Comparator;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -58,11 +57,11 @@ public class FileDictionary implements D
}
@Override
- public TermFreqPayloadIterator getWordsIterator() {
+ public InputIterator getWordsIterator() {
return new FileIterator();
}
- final class FileIterator implements TermFreqPayloadIterator {
+ final class FileIterator implements InputIterator {
private long curFreq;
private final BytesRef spare = new BytesRef();
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java (from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java&r1=1531987&r2=1532001&rev=1532001&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java Mon Oct 14 17:49:13 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.search.suggest
*/
import java.io.IOException;
+import java.util.Comparator;
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; // javadocs
@@ -79,5 +80,10 @@ public interface InputIterator extends B
public boolean hasPayloads() {
return false;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return wrapped.getComparator();
+ }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java Mon Oct 14 17:49:13 2013
@@ -24,7 +24,6 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.PriorityQueue;
@@ -154,25 +153,25 @@ public abstract class Lookup {
/** Build lookup from a dictionary. Some implementations may require sorted
* or unsorted keys from the dictionary's iterator - use
- * {@link SortedTermFreqPayloadIteratorWrapper} or
- * {@link UnsortedTermFreqPayloadIteratorWrapper} in such case.
+ * {@link SortedInputIterator} or
+ * {@link UnsortedInputIterator} in such case.
*/
public void build(Dictionary dict) throws IOException {
BytesRefIterator it = dict.getWordsIterator();
- TermFreqPayloadIterator tfit;
- if (it instanceof TermFreqPayloadIterator) {
- tfit = (TermFreqPayloadIterator)it;
+ InputIterator tfit;
+ if (it instanceof InputIterator) {
+ tfit = (InputIterator)it;
} else {
- tfit = new TermFreqPayloadIterator.TermFreqPayloadIteratorWrapper(it);
+ tfit = new InputIterator.InputIteratorWrapper(it);
}
build(tfit);
}
/**
- * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqPayloadIterator}.
+ * Builds up a new internal {@link Lookup} representation based on the given {@link InputIterator}.
* The implementation might re-sort the data internally.
*/
- public abstract void build(TermFreqPayloadIterator tfit) throws IOException;
+ public abstract void build(InputIterator tfit) throws IOException;
/**
* Look up a key and return possible completion for this key.
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java (from r1531987, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java&r1=1531987&r2=1532001&rev=1532001&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java Mon Oct 14 17:49:13 2013
@@ -111,6 +111,11 @@ public class SortedInputIterator impleme
return hasPayloads;
}
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return tieBreakByCostComparator;
+ }
+
/** Sortes by BytesRef (ascending) then cost (ascending). */
private final Comparator<BytesRef> tieBreakByCostComparator = new Comparator<BytesRef>() {
@@ -137,7 +142,14 @@ public class SortedInputIterator impleme
if (cmp != 0) {
return cmp;
}
- return Long.compare(leftCost, rightCost);
+ if (leftCost < rightCost) {
+ return -1;
+ } else if (leftCost > rightCost) {
+ return 1;
+ } else {
+ return 0;
+ }
+
}
};
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Mon Oct 14 17:49:13 2013
@@ -65,8 +65,8 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -175,7 +175,7 @@ public class AnalyzingInfixSuggester ext
}
@Override
- public void build(TermFreqPayloadIterator iter) throws IOException {
+ public void build(InputIterator iter) throws IOException {
if (searcher != null) {
searcher.getIndexReader().close();
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Mon Oct 14 17:49:13 2013
@@ -31,7 +31,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenStreamToAutomaton;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -380,7 +380,7 @@ public class AnalyzingSuggester extends
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
String prefix = getClass().getSimpleName();
File directory = Sort.defaultTempDir();
File tempInput = File.createTempFile(prefix, ".input", directory);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Mon Oct 14 17:49:13 2013
@@ -54,7 +54,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -273,14 +273,14 @@ public class FreeTextSuggester extends L
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
}
/** Build the suggest index, using up to the specified
* amount of temporary RAM while building. Note that
* the weights for the suggestions are ignored. */
- public void build(TermFreqPayloadIterator iterator, double ramBufferSizeMB) throws IOException {
+ public void build(InputIterator iterator, double ramBufferSizeMB) throws IOException {
if (iterator.hasPayloads()) {
throw new IllegalArgumentException("payloads are not supported");
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Mon Oct 14 17:49:13 2013
@@ -24,7 +24,7 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.SortInfo;
import org.apache.lucene.search.suggest.Sort;
@@ -42,7 +42,7 @@ import org.apache.lucene.util.fst.NoOutp
* An adapter from {@link Lookup} API to {@link FSTCompletion}.
*
* <p>This adapter differs from {@link FSTCompletion} in that it attempts
- * to discretize any "weights" as passed from in {@link TermFreqPayloadIterator#weight()}
+ * to discretize any "weights" as passed from in {@link InputIterator#weight()}
* to match the number of buckets. For the rationale for bucketing, see
* {@link FSTCompletion}.
*
@@ -95,7 +95,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqPayloadIterator)} method. The number of weight
+ * {@link #build(InputIterator)} method. The number of weight
* discretization buckets is set to {@link FSTCompletion#DEFAULT_BUCKETS} and
* exact matches are promoted to the top of the suggestions list.
*/
@@ -105,7 +105,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqPayloadIterator)} method.
+ * {@link #build(InputIterator)} method.
*
* @param buckets
* The number of weight discretization buckets (see
@@ -140,7 +140,7 @@ public class FSTCompletionLookup extends
}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Mon Oct 14 17:49:13 2013
@@ -25,10 +25,10 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
-import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.InputStreamDataInput;
@@ -92,12 +92,12 @@ public class WFSTCompletionLookup extend
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
if (iterator.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
BytesRef scratch = new BytesRef();
- TermFreqPayloadIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
+ InputIterator iter = new WFSTInputIterator(iterator);
IntsRef scratchInts = new IntsRef();
BytesRef previous = null;
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
@@ -254,9 +254,9 @@ public class WFSTCompletionLookup extend
return Integer.MAX_VALUE - (int)value;
}
- private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqPayloadIteratorWrapper {
+ private final class WFSTInputIterator extends SortedInputIterator {
- WFSTTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
+ WFSTInputIterator(InputIterator source) throws IOException {
super(source);
assert source.hasPayloads() == false;
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Mon Oct 14 17:49:13 2013
@@ -25,9 +25,9 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.UnsortedTermFreqPayloadIteratorWrapper;
+import org.apache.lucene.search.suggest.UnsortedInputIterator;
import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
@@ -47,19 +47,19 @@ public class JaspellLookup extends Looku
/**
* Creates a new empty trie
- * @see #build(TermFreqPayloadIterator)
+ * @see #build(InputIterator)
* */
public JaspellLookup() {}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
if (tfit.getComparator() != null) {
// make sure it's unsorted
// WTF - this could result in yet another sorted iteration....
- tfit = new UnsortedTermFreqPayloadIteratorWrapper(tfit);
+ tfit = new UnsortedInputIterator(tfit);
}
trie = new JaspellTernarySearchTrie();
trie.setMatchAlmostDiff(editDistance);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Mon Oct 14 17:49:13 2013
@@ -25,9 +25,9 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
@@ -45,12 +45,12 @@ public class TSTLookup extends Lookup {
/**
* Creates a new TSTLookup with an empty Ternary Search Tree.
- * @see #build(TermFreqPayloadIterator)
+ * @see #build(InputIterator)
*/
public TSTLookup() {}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
@@ -58,7 +58,7 @@ public class TSTLookup extends Lookup {
// buffer first
if (tfit.getComparator() != BytesRef.getUTF8SortedAsUTF16Comparator()) {
// make sure it's sorted and the comparator uses UTF16 sort order
- tfit = new SortedTermFreqPayloadIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
+ tfit = new SortedInputIterator(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
}
ArrayList<String> tokens = new ArrayList<String>();
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java Mon Oct 14 17:49:13 2013
@@ -7,7 +7,6 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -19,8 +18,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
-import org.apache.lucene.search.suggest.DocumentDictionary;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -78,7 +75,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -105,7 +102,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -153,7 +150,7 @@ public class DocumentDictionaryTest exte
IndexReader ir = DirectoryReader.open(dir);
assertEquals(ir.numDocs(), docs.size());
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java (from r1531987, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java&r1=1531987&r2=1532001&rev=1532001&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java Mon Oct 14 17:49:13 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.search.suggest
*/
import java.util.Arrays;
+import java.util.Comparator;
import java.util.Iterator;
import org.apache.lucene.util.BytesRef;
@@ -78,4 +79,9 @@ public final class InputArrayIterator im
public boolean hasPayloads() {
return hasPayloads;
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return null;
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Mon Oct 14 17:49:13 2013
@@ -72,12 +72,12 @@ public class LookupBenchmarkTest extends
/**
* Input term/weight pairs.
*/
- private static TermFreqPayload [] dictionaryInput;
+ private static Input [] dictionaryInput;
/**
* Benchmark term/weight pairs (randomized order).
*/
- private static List<TermFreqPayload> benchmarkInput;
+ private static List<Input> benchmarkInput;
/**
* Loads terms and frequencies from Wikipedia (cached).
@@ -85,9 +85,9 @@ public class LookupBenchmarkTest extends
@BeforeClass
public static void setup() throws Exception {
assert false : "disable assertions before running benchmarks!";
- List<TermFreqPayload> input = readTop50KWiki();
+ List<Input> input = readTop50KWiki();
Collections.shuffle(input, random);
- LookupBenchmarkTest.dictionaryInput = input.toArray(new TermFreqPayload [input.size()]);
+ LookupBenchmarkTest.dictionaryInput = input.toArray(new Input [input.size()]);
Collections.shuffle(input, random);
LookupBenchmarkTest.benchmarkInput = input;
}
@@ -97,8 +97,8 @@ public class LookupBenchmarkTest extends
/**
* Collect the multilingual input for benchmarks/ tests.
*/
- public static List<TermFreqPayload> readTop50KWiki() throws Exception {
- List<TermFreqPayload> input = new ArrayList<TermFreqPayload>();
+ public static List<Input> readTop50KWiki() throws Exception {
+ List<Input> input = new ArrayList<Input>();
URL resource = LookupBenchmarkTest.class.getResource("Top50KWiki.utf8");
assert resource != null : "Resource missing: Top50KWiki.utf8";
@@ -109,7 +109,7 @@ public class LookupBenchmarkTest extends
assertTrue("No | separator?: " + line, tab >= 0);
int weight = Integer.parseInt(line.substring(tab + 1));
String key = line.substring(0, tab);
- input.add(new TermFreqPayload(key, weight));
+ input.add(new Input(key, weight));
}
br.close();
return input;
@@ -163,7 +163,7 @@ public class LookupBenchmarkTest extends
/**
* Create {@link Lookup} instance and populate it.
*/
- private Lookup buildLookup(Class<? extends Lookup> cls, TermFreqPayload[] input) throws Exception {
+ private Lookup buildLookup(Class<? extends Lookup> cls, Input[] input) throws Exception {
Lookup lookup = null;
try {
lookup = cls.newInstance();
@@ -176,7 +176,7 @@ public class LookupBenchmarkTest extends
lookup = ctor.newInstance(a);
}
}
- lookup.build(new TermFreqPayloadArrayIterator(input));
+ lookup.build(new InputArrayIterator(input));
return lookup;
}
@@ -220,7 +220,7 @@ public class LookupBenchmarkTest extends
final Lookup lookup = buildLookup(cls, dictionaryInput);
final List<String> input = new ArrayList<String>(benchmarkInput.size());
- for (TermFreqPayload tf : benchmarkInput) {
+ for (Input tf : benchmarkInput) {
String s = tf.term.utf8ToString();
String sub = s.substring(0, Math.min(s.length(),
minPrefixLen + random.nextInt(maxPrefixLen - minPrefixLen + 1)));
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java Mon Oct 14 17:49:13 2013
@@ -65,10 +65,10 @@ public class PersistenceTest extends Luc
// Add all input keys.
Lookup lookup = lookupClass.newInstance();
- TermFreqPayload[] keys = new TermFreqPayload[this.keys.length];
+ Input[] keys = new Input[this.keys.length];
for (int i = 0; i < keys.length; i++)
- keys[i] = new TermFreqPayload(this.keys[i], i);
- lookup.build(new TermFreqPayloadArrayIterator(keys));
+ keys[i] = new Input(this.keys[i], i);
+ lookup.build(new InputArrayIterator(keys));
// Store the suggester.
File storeDir = TEMP_DIR;
@@ -81,7 +81,7 @@ public class PersistenceTest extends Luc
// Assert validity.
Random random = random();
long previous = Long.MIN_VALUE;
- for (TermFreqPayload k : keys) {
+ for (Input k : keys) {
List<LookupResult> list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1);
assertEquals(1, list.size());
LookupResult lookupResult = list.get(0);
Copied: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java (from r1531987, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java?p2=lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java&r1=1531987&r2=1532001&rev=1532001&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java Mon Oct 14 17:49:13 2013
@@ -46,8 +46,8 @@ public class TestInputIterator extends L
int num = atLeast(10000);
Comparator<BytesRef> comparator = random.nextBoolean() ? BytesRef.getUTF8SortedAsUnicodeComparator() : BytesRef.getUTF8SortedAsUTF16Comparator();
- TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> sorted = new TreeMap<>(comparator);
- TreeMap<BytesRef, Long> sortedWithoutPayload = new TreeMap<>(comparator);
+ TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> sorted = new TreeMap<BytesRef,SimpleEntry<Long,BytesRef>>(comparator);
+ TreeMap<BytesRef, Long> sortedWithoutPayload = new TreeMap<BytesRef,Long>(comparator);
Input[] unsorted = new Input[num];
Input[] unsortedWithoutPayload = new Input[num];
@@ -60,7 +60,7 @@ public class TestInputIterator extends L
} while (sorted.containsKey(key));
long value = random.nextLong();
sortedWithoutPayload.put(key, value);
- sorted.put(key, new SimpleEntry<>(value, payload));
+ sorted.put(key, new SimpleEntry<Long,BytesRef>(value, payload));
unsorted[i] = new Input(key, value, payload);
unsortedWithoutPayload[i] = new Input(key, value);
}
@@ -79,12 +79,12 @@ public class TestInputIterator extends L
// test the unsorted iterator wrapper with payloads
wrapper = new UnsortedInputIterator(new InputArrayIterator(unsorted));
- TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> actual = new TreeMap<>();
+ TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> actual = new TreeMap<BytesRef,SimpleEntry<Long,BytesRef>>();
BytesRef key;
while ((key = wrapper.next()) != null) {
long value = wrapper.weight();
BytesRef payload = wrapper.payload();
- actual.put(BytesRef.deepCopyOf(key), new SimpleEntry<>(value, BytesRef.deepCopyOf(payload)));
+ actual.put(BytesRef.deepCopyOf(key), new SimpleEntry<Long,BytesRef>(value, BytesRef.deepCopyOf(payload)));
}
assertEquals(sorted, actual);
@@ -102,7 +102,7 @@ public class TestInputIterator extends L
// test the unsorted iterator wrapper without payloads
wrapperWithoutPayload = new UnsortedInputIterator(new InputArrayIterator(unsortedWithoutPayload));
- TreeMap<BytesRef, Long> actualWithoutPayload = new TreeMap<>();
+ TreeMap<BytesRef, Long> actualWithoutPayload = new TreeMap<BytesRef,Long>();
while ((key = wrapperWithoutPayload.next()) != null) {
long value = wrapperWithoutPayload.weight();
assertNull(wrapperWithoutPayload.payload());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Mon Oct 14 17:49:13 2013
@@ -34,8 +34,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -48,9 +48,9 @@ import org.apache.lucene.util._TestUtil;
public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
public void testBasic() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -62,7 +62,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(2, results.size());
@@ -96,9 +96,9 @@ public class AnalyzingInfixSuggesterTest
}
public void testAfterLoad() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -110,7 +110,7 @@ public class AnalyzingInfixSuggesterTest
return newFSDirectory(path);
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
suggester.close();
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@@ -151,8 +151,8 @@ public class AnalyzingInfixSuggesterTest
@SuppressWarnings("unchecked")
public void testHighlightAsObject() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -208,7 +208,7 @@ public class AnalyzingInfixSuggesterTest
}
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(1, results.size());
@@ -234,9 +234,9 @@ public class AnalyzingInfixSuggesterTest
}
public void testRandomMinPrefixLength() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -249,7 +249,7 @@ public class AnalyzingInfixSuggesterTest
return newFSDirectory(path);
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
for(int i=0;i<2;i++) {
for(int j=0;j<2;j++) {
@@ -316,8 +316,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testHighlight() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -329,7 +329,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>penn</b>y saved is a <b>penn</b>y earned", results.get(0).key);
@@ -337,8 +337,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testHighlightCaseChange() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a Penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a Penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -350,7 +350,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>Penn</b>y saved is a <b>penn</b>y earned", results.get(0).key);
@@ -371,7 +371,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>Penny</b> saved is a <b>penny</b> earned", results.get(0).key);
@@ -379,8 +379,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testDoubleClose() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -392,7 +392,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
suggester.close();
suggester.close();
}
@@ -426,11 +426,11 @@ public class AnalyzingInfixSuggesterTest
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a bob for apples", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a bob for apples", 10, new BytesRef("foobaz")),
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("a", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a bob for <b>a</b>pples", results.get(0).key);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Mon Oct 14 17:49:13 2013
@@ -52,8 +52,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -63,18 +63,18 @@ public class AnalyzingSuggesterTest exte
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo", 50),
- new TermFreqPayload("bar", 10),
- new TermFreqPayload("barbar", 10),
- new TermFreqPayload("barbar", 12),
- new TermFreqPayload("barbara", 6),
- new TermFreqPayload("bar", 5),
- new TermFreqPayload("barbara", 1)
+ Iterable<Input> keys = shuffle(
+ new Input("foo", 50),
+ new Input("bar", 10),
+ new Input("barbar", 10),
+ new Input("barbar", 12),
+ new Input("barbara", 6),
+ new Input("bar", 5),
+ new Input("barbara", 1)
);
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
@@ -109,16 +109,16 @@ public class AnalyzingSuggesterTest exte
}
public void testKeywordWithPayloads() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo", 50, new BytesRef("hello")),
- new TermFreqPayload("bar", 10, new BytesRef("goodbye")),
- new TermFreqPayload("barbar", 12, new BytesRef("thank you")),
- new TermFreqPayload("bar", 9, new BytesRef("should be deduplicated")),
- new TermFreqPayload("bar", 8, new BytesRef("should also be deduplicated")),
- new TermFreqPayload("barbara", 6, new BytesRef("for all the fish")));
+ Iterable<Input> keys = shuffle(
+ new Input("foo", 50, new BytesRef("hello")),
+ new Input("bar", 10, new BytesRef("goodbye")),
+ new Input("barbar", 12, new BytesRef("thank you")),
+ new Input("bar", 9, new BytesRef("should be deduplicated")),
+ new Input("bar", 8, new BytesRef("should also be deduplicated")),
+ new Input("barbara", 6, new BytesRef("for all the fish")));
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
for (int i = 0; i < 2; i++) {
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
@@ -163,14 +163,14 @@ public class AnalyzingSuggesterTest exte
public void testRandomRealisticKeys() throws IOException {
LineFileDocs lineFile = new LineFileDocs(random());
Map<String, Long> mapping = new HashMap<String, Long>();
- List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
+ List<Input> keys = new ArrayList<Input>();
int howMany = atLeast(100); // this might bring up duplicates
for (int i = 0; i < howMany; i++) {
Document nextDoc = lineFile.nextDoc();
String title = nextDoc.getField("title").stringValue();
int randomWeight = random().nextInt(100);
- keys.add(new TermFreqPayload(title, randomWeight));
+ keys.add(new Input(title, randomWeight));
if (!mapping.containsKey(title) || mapping.get(title) < randomWeight) {
mapping.put(title, Long.valueOf(randomWeight));
}
@@ -180,16 +180,16 @@ public class AnalyzingSuggesterTest exte
analyzingSuggester.setPreservePositionIncrements(random().nextBoolean());
boolean doPayloads = random().nextBoolean();
if (doPayloads) {
- List<TermFreqPayload> keysAndPayloads = new ArrayList<TermFreqPayload>();
- for (TermFreqPayload termFreq : keys) {
- keysAndPayloads.add(new TermFreqPayload(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
+ List<Input> keysAndPayloads = new ArrayList<Input>();
+ for (Input termFreq : keys) {
+ keysAndPayloads.add(new Input(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
}
- analyzingSuggester.build(new TermFreqPayloadArrayIterator(keysAndPayloads));
+ analyzingSuggester.build(new InputArrayIterator(keysAndPayloads));
} else {
- analyzingSuggester.build(new TermFreqPayloadArrayIterator(keys));
+ analyzingSuggester.build(new InputArrayIterator(keys));
}
- for (TermFreqPayload termFreq : keys) {
+ for (Input termFreq : keys) {
List<LookupResult> lookup = analyzingSuggester.lookup(termFreq.term.utf8ToString(), false, keys.size());
for (LookupResult lookupResult : lookup) {
assertEquals(mapping.get(lookupResult.key), Long.valueOf(lookupResult.value));
@@ -209,14 +209,14 @@ public class AnalyzingSuggesterTest exte
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("the ghost of christmas past", 50),
+ Input keys[] = new Input[] {
+ new Input("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -239,23 +239,23 @@ public class AnalyzingSuggesterTest exte
public void testEmpty() throws Exception {
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
+ suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}
public void testNoSeps() throws Exception {
- TermFreqPayload[] keys = new TermFreqPayload[] {
- new TermFreqPayload("ab cd", 0),
- new TermFreqPayload("abcd", 1),
+ Input[] keys = new Input[] {
+ new Input("ab cd", 0),
+ new Input("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -316,13 +316,13 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("wifi network is slow", 50),
- new TermFreqPayload("wi fi network is fast", 10),
+ Input keys[] = new Input[] {
+ new Input("wifi network is slow", 50),
+ new Input("wi fi network is fast", 10),
};
//AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
System.out.println("Results: " + results);
@@ -382,12 +382,12 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("ab xc", 50),
- new TermFreqPayload("ba xd", 50),
+ Input keys[] = new Input[] {
+ new Input("ab xc", 50),
+ new Input("ba xd", 50),
};
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -460,11 +460,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -500,11 +500,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -655,12 +655,12 @@ public class AnalyzingSuggesterTest exte
boolean doPayloads = random().nextBoolean();
- TermFreqPayload[] keys = null;
- TermFreqPayload[] payloadKeys = null;
+ Input[] keys = null;
+ Input[] payloadKeys = null;
if (doPayloads) {
- payloadKeys = new TermFreqPayload[numQueries];
+ payloadKeys = new Input[numQueries];
} else {
- keys = new TermFreqPayload[numQueries];
+ keys = new Input[numQueries];
}
boolean preserveSep = random().nextBoolean();
@@ -731,9 +731,9 @@ public class AnalyzingSuggesterTest exte
byte[] bytes = new byte[random().nextInt(10)];
random().nextBytes(bytes);
payload = new BytesRef(bytes);
- payloadKeys[i] = new TermFreqPayload(key, weight, payload);
+ payloadKeys[i] = new Input(key, weight, payload);
} else {
- keys[i] = new TermFreqPayload(key, weight);
+ keys[i] = new Input(key, weight);
payload = null;
}
@@ -754,9 +754,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1);
if (doPayloads) {
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(payloadKeys)));
+ suggester.build(new InputArrayIterator(shuffle(payloadKeys)));
} else {
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(keys)));
+ suggester.build(new InputArrayIterator(shuffle(keys)));
}
for (String prefix : allPrefixes) {
@@ -874,8 +874,8 @@ public class AnalyzingSuggesterTest exte
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1);
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(new TermFreqPayload("a", 40),
- new TermFreqPayload("a ", 50), new TermFreqPayload(" a", 60))));
+ suggester.build(new InputArrayIterator(shuffle(new Input("a", 40),
+ new Input("a ", 50), new Input(" a", 60))));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -889,11 +889,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 2),
- new TermFreqPayload("a b c", 3),
- new TermFreqPayload("a c a", 1),
- new TermFreqPayload("a c b", 1),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 2),
+ new Input("a b c", 3),
+ new Input("a c a", 1),
+ new Input("a c b", 1),
}));
suggester.lookup("a", false, 4);
@@ -905,10 +905,10 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 5),
- new TermFreqPayload("a b", 3),
- new TermFreqPayload("a c", 4),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 5),
+ new Input("a b", 3),
+ new Input("a c", 4),
}));
List<LookupResult> results = suggester.lookup("a", false, 3);
@@ -970,9 +970,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(
- new TermFreqPayload("hambone", 6),
- new TermFreqPayload("nellie", 5))));
+ suggester.build(new InputArrayIterator(shuffle(
+ new Input("hambone", 6),
+ new Input("nellie", 5))));
List<LookupResult> results = suggester.lookup("nellie", false, 2);
assertEquals(2, results.size());
@@ -1039,9 +1039,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 6),
- new TermFreqPayload("b", 5),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 6),
+ new Input("b", 5),
}));
List<LookupResult> results = suggester.lookup("a", false, 2);
@@ -1112,21 +1112,21 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a a", 50),
- new TermFreqPayload("a b", 50),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a a", 50),
+ new Input("a b", 50),
}));
}
public void testDupSurfaceFormsMissingResults3() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a a", 7),
- new TermFreqPayload("a a", 7),
- new TermFreqPayload("a c", 6),
- new TermFreqPayload("a c", 3),
- new TermFreqPayload("a b", 5),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a a", 7),
+ new Input("a a", 7),
+ new Input("a c", 6),
+ new Input("a c", 3),
+ new Input("a b", 5),
}));
assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString());
}
@@ -1134,9 +1134,9 @@ public class AnalyzingSuggesterTest exte
public void testEndingSpace() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("i love lucy", 7),
- new TermFreqPayload("isla de muerta", 8),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("i love lucy", 7),
+ new Input("isla de muerta", 8),
}));
assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
@@ -1167,15 +1167,15 @@ public class AnalyzingSuggesterTest exte
};
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {new TermFreqPayload("a", 1)}));
+ suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)}));
assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
}
public void testIllegalLookupArgument() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("а где ÐÑÑи?", 7),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("а где ÐÑÑи?", 7),
}));
try {
suggester.lookup("а\u001E", false, 3);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Mon Oct 14 17:49:13 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -54,16 +54,16 @@ import org.apache.lucene.util.fst.Util;
public class FuzzySuggesterTest extends LuceneTestCase {
public void testRandomEdits() throws IOException {
- List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
+ List<Input> keys = new ArrayList<Input>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreqPayload("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new Input("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreqPayload("foo bar boo far", 12));
+ keys.add(new Input("foo bar boo far", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("foo bar boo", FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX);
@@ -75,16 +75,16 @@ public class FuzzySuggesterTest extends
}
public void testNonLatinRandomEdits() throws IOException {
- List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
+ List<Input> keys = new ArrayList<Input>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreqPayload("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new Input("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreqPayload("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
+ keys.add(new Input("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ", 0);
@@ -97,15 +97,15 @@ public class FuzzySuggesterTest extends
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("foo", 50),
- new TermFreqPayload("bar", 10),
- new TermFreqPayload("barbar", 12),
- new TermFreqPayload("barbara", 6)
+ Input keys[] = new Input[] {
+ new Input("foo", 50),
+ new Input("bar", 10),
+ new Input("barbar", 12),
+ new Input("barbara", 6)
};
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("bariar", random()), false, 2);
assertEquals(2, results.size());
@@ -172,14 +172,14 @@ public class FuzzySuggesterTest extends
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("the ghost of christmas past", 50),
+ Input keys[] = new Input[] {
+ new Input("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
FuzzySuggester suggester = new FuzzySuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -200,16 +200,16 @@ public class FuzzySuggesterTest extends
}
public void testNoSeps() throws Exception {
- TermFreqPayload[] keys = new TermFreqPayload[] {
- new TermFreqPayload("ab cd", 0),
- new TermFreqPayload("abcd", 1),
+ Input[] keys = new Input[] {
+ new Input("ab cd", 0),
+ new Input("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -270,12 +270,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("wifi network is slow", 50),
- new TermFreqPayload("wi fi network is fast", 10),
+ Input keys[] = new Input[] {
+ new Input("wifi network is slow", 50),
+ new Input("wi fi network is fast", 10),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
@@ -290,7 +290,7 @@ public class FuzzySuggesterTest extends
public void testEmpty() throws Exception {
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
+ suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
@@ -344,12 +344,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("ab xc", 50),
- new TermFreqPayload("ba xd", 50),
+ Input keys[] = new Input[] {
+ new Input("ab xc", 50),
+ new Input("ba xd", 50),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -418,11 +418,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -458,11 +458,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -600,7 +600,7 @@ public class FuzzySuggesterTest extends
final TreeSet<String> allPrefixes = new TreeSet<String>();
final Set<String> seen = new HashSet<String>();
- TermFreqPayload[] keys = new TermFreqPayload[numQueries];
+ Input[] keys = new Input[numQueries];
boolean preserveSep = random().nextBoolean();
boolean unicodeAware = random().nextBoolean();
@@ -666,7 +666,7 @@ public class FuzzySuggesterTest extends
}
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
- keys[i] = new TermFreqPayload(key, weight);
+ keys[i] = new Input(key, weight);
slowCompletor.add(new TermFreqPayload2(key, analyzedKey, weight));
}
@@ -684,7 +684,7 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
FuzzySuggester suggester = new FuzzySuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, 1, false, 1, 3, unicodeAware);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
for (String prefix : allPrefixes) {
@@ -825,14 +825,14 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, 1, true, 1, 3, false);
- List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
- new TermFreqPayload("a", 40),
- new TermFreqPayload("a ", 50),
- new TermFreqPayload(" a", 60),
+ List<Input> keys = Arrays.asList(new Input[] {
+ new Input("a", 40),
+ new Input("a ", 50),
+ new Input(" a", 60),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -846,15 +846,15 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, 2, true, 1, 3, false);
- List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
- new TermFreqPayload("foo bar", 40),
- new TermFreqPayload("foo bar baz", 50),
- new TermFreqPayload("barbaz", 60),
- new TermFreqPayload("barbazfoo", 10),
+ List<Input> keys = Arrays.asList(new Input[] {
+ new Input("foo bar", 40),
+ new Input("foo bar baz", 50),
+ new Input("barbaz", 60),
+ new Input("barbazfoo", 10),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
assertEquals("[foo bar baz/50, foo bar/40]", suggester.lookup("foobar", false, 5).toString());
assertEquals("[foo bar baz/50]", suggester.lookup("foobarbaz", false, 5).toString());
@@ -929,25 +929,25 @@ public class FuzzySuggesterTest extends
public void testRandom2() throws Throwable {
final int NUM = atLeast(200);
- final List<TermFreqPayload> answers = new ArrayList<TermFreqPayload>();
+ final List<Input> answers = new ArrayList<Input>();
final Set<String> seen = new HashSet<String>();
for(int i=0;i<NUM;i++) {
final String s = randomSimpleString(8);
if (!seen.contains(s)) {
- answers.add(new TermFreqPayload(s, random().nextInt(1000)));
+ answers.add(new Input(s, random().nextInt(1000)));
seen.add(s);
}
}
- Collections.sort(answers, new Comparator<TermFreqPayload>() {
+ Collections.sort(answers, new Comparator<Input>() {
@Override
- public int compare(TermFreqPayload a, TermFreqPayload b) {
+ public int compare(Input a, Input b) {
return a.term.compareTo(b.term);
}
});
if (VERBOSE) {
System.out.println("\nTEST: targets");
- for(TermFreqPayload tf : answers) {
+ for(Input tf : answers) {
System.out.println(" " + tf.term.utf8ToString() + " freq=" + tf.v);
}
}
@@ -965,7 +965,7 @@ public class FuzzySuggesterTest extends
}
Collections.shuffle(answers, random());
- suggest.build(new TermFreqPayloadArrayIterator(answers.toArray(new TermFreqPayload[answers.size()])));
+ suggest.build(new InputArrayIterator(answers.toArray(new Input[answers.size()])));
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
@@ -1004,10 +1004,10 @@ public class FuzzySuggesterTest extends
}
}
- private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<TermFreqPayload> answers, String frag) {
+ private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<Input> answers, String frag) {
final List<LookupResult> results = new ArrayList<LookupResult>();
final int fragLen = frag.length();
- for(TermFreqPayload tf : answers) {
+ for(Input tf : answers) {
//System.out.println(" check s=" + tf.term.utf8ToString());
boolean prefixMatches = true;
for(int i=0;i<prefixLen;i++) {