You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/14 19:34:31 UTC
svn commit: r1531987 [1/2] - in /lucene/dev/trunk/lucene: ./
suggest/src/java/org/apache/lucene/search/spell/
suggest/src/java/org/apache/lucene/search/suggest/
suggest/src/java/org/apache/lucene/search/suggest/analyzing/
suggest/src/java/org/apache/lu...
Author: mikemccand
Date: Mon Oct 14 17:34:30 2013
New Revision: 1531987
URL: http://svn.apache.org/r1531987
Log:
LUCENE-5280: rename TermFreqPayloadIterator -> InputIterator
Added:
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java
- copied, changed from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java
Removed:
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Oct 14 17:34:30 2013
@@ -159,6 +159,10 @@ API Changes:
suggesters that can't support payloads will throw an exception if
hasPayloads() is true. (Areek Zillur via Mike McCandless)
+* LUCENE-5280: Rename TermFreqPayloadIterator -> InputIterator, along
+ with associated suggest/spell classes. (Areek Zillur via Mike
+ McCandless)
+
Optimizations
* LUCENE-5225: The ToParentBlockJoinQuery only keeps tracks of the the child
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Mon Oct 14 17:34:30 2013
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.BytesRef;
@@ -59,7 +60,7 @@ public class HighFrequencyDictionary imp
return new HighFrequencyIterator();
}
- final class HighFrequencyIterator implements TermFreqPayloadIterator {
+ final class HighFrequencyIterator implements InputIterator {
private final BytesRef spare = new BytesRef();
private final TermsEnum termsEnum;
private int minNumDocs;
Copied: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqPayloadIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferedInputIterator.java Mon Oct 14 17:34:30 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.search.suggest
import java.io.IOException;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
@@ -28,7 +27,7 @@ import org.apache.lucene.util.Counter;
* This wrapper buffers incoming elements.
* @lucene.experimental
*/
-public class BufferingTermFreqPayloadIteratorWrapper implements TermFreqPayloadIterator {
+public class BufferedInputIterator implements InputIterator {
// TODO keep this for now
/** buffered term entries */
protected BytesRefArray entries = new BytesRefArray(Counter.newCounter());
@@ -43,7 +42,7 @@ public class BufferingTermFreqPayloadIte
private final boolean hasPayloads;
/** Creates a new iterator, buffering entries from the specified iterator */
- public BufferingTermFreqPayloadIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
+ public BufferedInputIterator(InputIterator source) throws IOException {
BytesRef spare;
int freqIndex = 0;
hasPayloads = source.hasPayloads();
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java Mon Oct 14 17:34:30 2013
@@ -27,12 +27,6 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.index.StorableField;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
-import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; // javadoc
-import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; // javadoc
-import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup; // javadoc
-import org.apache.lucene.search.suggest.jaspell.JaspellLookup; // javadoc
-import org.apache.lucene.search.suggest.tst.TSTLookup; // javadoc
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
@@ -83,10 +77,10 @@ public class DocumentDictionary implemen
@Override
public BytesRefIterator getWordsIterator() throws IOException {
- return new TermWeightPayloadIterator(payloadField!=null);
+ return new DocumentInputIterator(payloadField!=null);
}
- final class TermWeightPayloadIterator implements TermFreqPayloadIterator {
+ final class DocumentInputIterator implements InputIterator {
private final int docCount;
private final Set<String> relevantFields;
private final boolean hasPayloads;
@@ -100,7 +94,7 @@ public class DocumentDictionary implemen
* index. setting <code>withPayload</code> to false, implies an iterator
* over only term and weight.
*/
- public TermWeightPayloadIterator(boolean hasPayloads) throws IOException {
+ public DocumentInputIterator(boolean hasPayloads) throws IOException {
docCount = reader.maxDoc() - 1;
this.hasPayloads = hasPayloads;
currentPayload = null;
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Mon Oct 14 17:34:30 2013
@@ -21,7 +21,6 @@ package org.apache.lucene.search.suggest
import java.io.*;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -57,11 +56,11 @@ public class FileDictionary implements D
}
@Override
- public TermFreqPayloadIterator getWordsIterator() {
+ public InputIterator getWordsIterator() {
return new FileIterator();
}
- final class FileIterator implements TermFreqPayloadIterator {
+ final class FileIterator implements InputIterator {
private long curFreq;
private final BytesRef spare = new BytesRef();
Copied: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java Mon Oct 14 17:34:30 2013
@@ -1,4 +1,4 @@
-package org.apache.lucene.search.spell;
+package org.apache.lucene.search.suggest;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,11 +27,11 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
/**
- * Interface for enumerating term,weight,payload triples;
+ * Interface for enumerating term,weight,payload triples for suggester consumption;
* currently only {@link AnalyzingSuggester}, {@link
* FuzzySuggester} and {@link AnalyzingInfixSuggester} support payloads.
*/
-public interface TermFreqPayloadIterator extends BytesRefIterator {
+public interface InputIterator extends BytesRefIterator {
/** A term's weight, higher numbers mean better suggestions. */
public long weight();
@@ -45,10 +45,10 @@ public interface TermFreqPayloadIterator
public boolean hasPayloads();
/**
- * Wraps a BytesRefIterator as a TermFreqPayloadIterator, with all weights
+ * Wraps a BytesRefIterator as a suggester InputIterator, with all weights
* set to <code>1</code> and carries no payload
*/
- public static class TermFreqPayloadIteratorWrapper implements TermFreqPayloadIterator {
+ public static class InputIteratorWrapper implements InputIterator {
private final BytesRefIterator wrapped;
/**
@@ -56,7 +56,7 @@ public interface TermFreqPayloadIterator
* specifying a weight value of <code>1</code> for all terms
* and nullifies associated payloads.
*/
- public TermFreqPayloadIteratorWrapper(BytesRefIterator wrapped) {
+ public InputIteratorWrapper(BytesRefIterator wrapped) {
this.wrapped = wrapped;
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java Mon Oct 14 17:34:30 2013
@@ -24,7 +24,6 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.PriorityQueue;
@@ -154,25 +153,25 @@ public abstract class Lookup {
/** Build lookup from a dictionary. Some implementations may require sorted
* or unsorted keys from the dictionary's iterator - use
- * {@link SortedTermFreqPayloadIteratorWrapper} or
- * {@link UnsortedTermFreqPayloadIteratorWrapper} in such case.
+ * {@link SortedInputIterator} or
+ * {@link UnsortedInputIterator} in such case.
*/
public void build(Dictionary dict) throws IOException {
BytesRefIterator it = dict.getWordsIterator();
- TermFreqPayloadIterator tfit;
- if (it instanceof TermFreqPayloadIterator) {
- tfit = (TermFreqPayloadIterator)it;
+ InputIterator tfit;
+ if (it instanceof InputIterator) {
+ tfit = (InputIterator)it;
} else {
- tfit = new TermFreqPayloadIterator.TermFreqPayloadIteratorWrapper(it);
+ tfit = new InputIterator.InputIteratorWrapper(it);
}
build(tfit);
}
/**
- * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqPayloadIterator}.
+ * Builds up a new internal {@link Lookup} representation based on the given {@link InputIterator}.
* The implementation might re-sort the data internally.
*/
- public abstract void build(TermFreqPayloadIterator tfit) throws IOException;
+ public abstract void build(InputIterator tfit) throws IOException;
/**
* Look up a key and return possible completion for this key.
Copied: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqPayloadIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedInputIterator.java Mon Oct 14 17:34:30 2013
@@ -21,7 +21,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Comparator;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Sort.ByteSequencesReader;
import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -34,9 +33,9 @@ import org.apache.lucene.util.IOUtils;
* This wrapper buffers incoming elements and makes sure they are sorted based on given comparator.
* @lucene.experimental
*/
-public class SortedTermFreqPayloadIteratorWrapper implements TermFreqPayloadIterator {
+public class SortedInputIterator implements InputIterator {
- private final TermFreqPayloadIterator source;
+ private final InputIterator source;
private File tempInput;
private File tempSorted;
private final ByteSequencesReader reader;
@@ -52,7 +51,7 @@ public class SortedTermFreqPayloadIterat
* Creates a new sorted wrapper, using {@link
* BytesRef#getUTF8SortedAsUnicodeComparator} for
* sorting. */
- public SortedTermFreqPayloadIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
+ public SortedInputIterator(InputIterator source) throws IOException {
this(source, BytesRef.getUTF8SortedAsUnicodeComparator());
}
@@ -60,7 +59,7 @@ public class SortedTermFreqPayloadIterat
* Creates a new sorted wrapper, sorting by BytesRef
* (ascending) then cost (ascending).
*/
- public SortedTermFreqPayloadIteratorWrapper(TermFreqPayloadIterator source, Comparator<BytesRef> comparator) throws IOException {
+ public SortedInputIterator(InputIterator source, Comparator<BytesRef> comparator) throws IOException {
this.hasPayloads = source.hasPayloads();
this.source = source;
this.comparator = comparator;
Copied: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqPayloadIteratorWrapper.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedInputIterator.java Mon Oct 14 17:34:30 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.search.suggest
import java.io.IOException;
import java.util.Random;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
/**
@@ -28,7 +27,7 @@ import org.apache.lucene.util.BytesRef;
* random order.
* @lucene.experimental
*/
-public class UnsortedTermFreqPayloadIteratorWrapper extends BufferingTermFreqPayloadIteratorWrapper {
+public class UnsortedInputIterator extends BufferedInputIterator {
// TODO keep this for now
private final int[] ords;
private int currentOrd = -1;
@@ -38,7 +37,7 @@ public class UnsortedTermFreqPayloadIter
* Creates a new iterator, wrapping the specified iterator and
* returning elements in a random order.
*/
- public UnsortedTermFreqPayloadIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
+ public UnsortedInputIterator(InputIterator source) throws IOException {
super(source);
ords = new int[entries.size()];
Random random = new Random();
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Mon Oct 14 17:34:30 2013
@@ -65,8 +65,8 @@ import org.apache.lucene.search.ScoreDoc
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -175,7 +175,7 @@ public class AnalyzingInfixSuggester ext
}
@Override
- public void build(TermFreqPayloadIterator iter) throws IOException {
+ public void build(InputIterator iter) throws IOException {
if (searcher != null) {
searcher.getIndexReader().close();
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Mon Oct 14 17:34:30 2013
@@ -31,7 +31,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenStreamToAutomaton;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -380,7 +380,7 @@ public class AnalyzingSuggester extends
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
String prefix = getClass().getSimpleName();
File directory = Sort.defaultTempDir();
File tempInput = File.createTempFile(prefix, ".input", directory);
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Mon Oct 14 17:34:30 2013
@@ -54,7 +54,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -273,14 +273,14 @@ public class FreeTextSuggester extends L
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
}
/** Build the suggest index, using up to the specified
* amount of temporary RAM while building. Note that
* the weights for the suggestions are ignored. */
- public void build(TermFreqPayloadIterator iterator, double ramBufferSizeMB) throws IOException {
+ public void build(InputIterator iterator, double ramBufferSizeMB) throws IOException {
if (iterator.hasPayloads()) {
throw new IllegalArgumentException("payloads are not supported");
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Mon Oct 14 17:34:30 2013
@@ -24,7 +24,7 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.SortInfo;
import org.apache.lucene.search.suggest.Sort;
@@ -42,7 +42,7 @@ import org.apache.lucene.util.fst.NoOutp
* An adapter from {@link Lookup} API to {@link FSTCompletion}.
*
* <p>This adapter differs from {@link FSTCompletion} in that it attempts
- * to discretize any "weights" as passed from in {@link TermFreqPayloadIterator#weight()}
+ * to discretize any "weights" as passed from in {@link InputIterator#weight()}
* to match the number of buckets. For the rationale for bucketing, see
* {@link FSTCompletion}.
*
@@ -95,7 +95,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqPayloadIterator)} method. The number of weight
+ * {@link #build(InputIterator)} method. The number of weight
* discretization buckets is set to {@link FSTCompletion#DEFAULT_BUCKETS} and
* exact matches are promoted to the top of the suggestions list.
*/
@@ -105,7 +105,7 @@ public class FSTCompletionLookup extends
/**
* This constructor prepares for creating a suggested FST using the
- * {@link #build(TermFreqPayloadIterator)} method.
+ * {@link #build(InputIterator)} method.
*
* @param buckets
* The number of weight discretization buckets (see
@@ -140,7 +140,7 @@ public class FSTCompletionLookup extends
}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Mon Oct 14 17:34:30 2013
@@ -25,10 +25,10 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
-import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.InputStreamDataInput;
@@ -92,12 +92,12 @@ public class WFSTCompletionLookup extend
}
@Override
- public void build(TermFreqPayloadIterator iterator) throws IOException {
+ public void build(InputIterator iterator) throws IOException {
if (iterator.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
BytesRef scratch = new BytesRef();
- TermFreqPayloadIterator iter = new WFSTTermFreqIteratorWrapper(iterator);
+ InputIterator iter = new WFSTInputIterator(iterator);
IntsRef scratchInts = new IntsRef();
BytesRef previous = null;
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton();
@@ -254,9 +254,9 @@ public class WFSTCompletionLookup extend
return Integer.MAX_VALUE - (int)value;
}
- private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqPayloadIteratorWrapper {
+ private final class WFSTInputIterator extends SortedInputIterator {
- WFSTTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException {
+ WFSTInputIterator(InputIterator source) throws IOException {
super(source);
assert source.hasPayloads() == false;
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Mon Oct 14 17:34:30 2013
@@ -25,7 +25,7 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
import org.apache.lucene.util.BytesRef;
@@ -46,12 +46,12 @@ public class JaspellLookup extends Looku
/**
* Creates a new empty trie
- * @see #build(TermFreqPayloadIterator)
+ * @see #build(InputIterator)
* */
public JaspellLookup() {}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Mon Oct 14 17:34:30 2013
@@ -25,9 +25,9 @@ import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.SortedTermFreqPayloadIteratorWrapper;
+import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
@@ -45,19 +45,19 @@ public class TSTLookup extends Lookup {
/**
* Creates a new TSTLookup with an empty Ternary Search Tree.
- * @see #build(TermFreqPayloadIterator)
+ * @see #build(InputIterator)
*/
public TSTLookup() {}
@Override
- public void build(TermFreqPayloadIterator tfit) throws IOException {
+ public void build(InputIterator tfit) throws IOException {
if (tfit.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
root = new TernaryTreeNode();
// make sure it's sorted and the comparator uses UTF16 sort order
- tfit = new SortedTermFreqPayloadIteratorWrapper(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
+ tfit = new SortedInputIterator(tfit, BytesRef.getUTF8SortedAsUTF16Comparator());
ArrayList<String> tokens = new ArrayList<String>();
ArrayList<Number> vals = new ArrayList<Number>();
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java Mon Oct 14 17:34:30 2013
@@ -19,7 +19,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spell.Dictionary;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.DocumentDictionary;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -78,7 +77,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -105,7 +104,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -153,7 +152,7 @@ public class DocumentDictionaryTest exte
IndexReader ir = DirectoryReader.open(dir);
assertEquals(ir.numDocs(), docs.size());
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- TermFreqPayloadIterator tfp = (TermFreqPayloadIterator) dictionary.getWordsIterator();
+ InputIterator tfp = (InputIterator) dictionary.getWordsIterator();
BytesRef f;
while((f = tfp.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
Copied: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java?p2=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayload.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/Input.java Mon Oct 14 17:34:30 2013
@@ -19,29 +19,30 @@ package org.apache.lucene.search.suggest
import org.apache.lucene.util.BytesRef;
-public final class TermFreqPayload {
+/** corresponds to {@link InputIterator}'s entries */
+public final class Input {
public final BytesRef term;
public final long v;
public final BytesRef payload;
public final boolean hasPayloads;
- public TermFreqPayload(BytesRef term, long v, BytesRef payload) {
+ public Input(BytesRef term, long v, BytesRef payload) {
this(term, v, payload, true);
}
- public TermFreqPayload(String term, long v, BytesRef payload) {
+ public Input(String term, long v, BytesRef payload) {
this(new BytesRef(term), v, payload, true);
}
- public TermFreqPayload(BytesRef term, long v) {
+ public Input(BytesRef term, long v) {
this(term, v, null, false);
}
- public TermFreqPayload(String term, long v) {
+ public Input(String term, long v) {
this(new BytesRef(term), v, null, false);
}
- public TermFreqPayload(BytesRef term, long v, BytesRef payload, boolean hasPayloads) {
+ public Input(BytesRef term, long v, BytesRef payload, boolean hasPayloads) {
this.term = term;
this.v = v;
this.payload = payload;
Copied: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/InputArrayIterator.java Mon Oct 14 17:34:30 2013
@@ -20,20 +20,19 @@ package org.apache.lucene.search.suggest
import java.util.Arrays;
import java.util.Iterator;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.util.BytesRef;
/**
- * A {@link TermFreqPayloadIterator} over a sequence of {@link TermFreqPayload}s.
+ * A {@link InputIterator} over a sequence of {@link Input}s.
*/
-public final class TermFreqPayloadArrayIterator implements TermFreqPayloadIterator {
- private final Iterator<TermFreqPayload> i;
+public final class InputArrayIterator implements InputIterator {
+ private final Iterator<Input> i;
private final boolean hasPayloads;
private boolean first;
- private TermFreqPayload current;
+ private Input current;
private final BytesRef spare = new BytesRef();
- public TermFreqPayloadArrayIterator(Iterator<TermFreqPayload> i) {
+ public InputArrayIterator(Iterator<Input> i) {
this.i = i;
if (i.hasNext()) {
current = i.next();
@@ -44,10 +43,10 @@ public final class TermFreqPayloadArrayI
}
}
- public TermFreqPayloadArrayIterator(TermFreqPayload[] i) {
+ public InputArrayIterator(Input[] i) {
this(Arrays.asList(i));
}
- public TermFreqPayloadArrayIterator(Iterable<TermFreqPayload> i) {
+ public InputArrayIterator(Iterable<Input> i) {
this(i.iterator());
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Mon Oct 14 17:34:30 2013
@@ -72,12 +72,12 @@ public class LookupBenchmarkTest extends
/**
* Input term/weight pairs.
*/
- private static TermFreqPayload [] dictionaryInput;
+ private static Input [] dictionaryInput;
/**
* Benchmark term/weight pairs (randomized order).
*/
- private static List<TermFreqPayload> benchmarkInput;
+ private static List<Input> benchmarkInput;
/**
* Loads terms and frequencies from Wikipedia (cached).
@@ -85,9 +85,9 @@ public class LookupBenchmarkTest extends
@BeforeClass
public static void setup() throws Exception {
assert false : "disable assertions before running benchmarks!";
- List<TermFreqPayload> input = readTop50KWiki();
+ List<Input> input = readTop50KWiki();
Collections.shuffle(input, random);
- LookupBenchmarkTest.dictionaryInput = input.toArray(new TermFreqPayload [input.size()]);
+ LookupBenchmarkTest.dictionaryInput = input.toArray(new Input [input.size()]);
Collections.shuffle(input, random);
LookupBenchmarkTest.benchmarkInput = input;
}
@@ -97,8 +97,8 @@ public class LookupBenchmarkTest extends
/**
* Collect the multilingual input for benchmarks/ tests.
*/
- public static List<TermFreqPayload> readTop50KWiki() throws Exception {
- List<TermFreqPayload> input = new ArrayList<TermFreqPayload>();
+ public static List<Input> readTop50KWiki() throws Exception {
+ List<Input> input = new ArrayList<Input>();
URL resource = LookupBenchmarkTest.class.getResource("Top50KWiki.utf8");
assert resource != null : "Resource missing: Top50KWiki.utf8";
@@ -109,7 +109,7 @@ public class LookupBenchmarkTest extends
assertTrue("No | separator?: " + line, tab >= 0);
int weight = Integer.parseInt(line.substring(tab + 1));
String key = line.substring(0, tab);
- input.add(new TermFreqPayload(key, weight));
+ input.add(new Input(key, weight));
}
br.close();
return input;
@@ -163,7 +163,7 @@ public class LookupBenchmarkTest extends
/**
* Create {@link Lookup} instance and populate it.
*/
- private Lookup buildLookup(Class<? extends Lookup> cls, TermFreqPayload[] input) throws Exception {
+ private Lookup buildLookup(Class<? extends Lookup> cls, Input[] input) throws Exception {
Lookup lookup = null;
try {
lookup = cls.newInstance();
@@ -176,7 +176,7 @@ public class LookupBenchmarkTest extends
lookup = ctor.newInstance(a);
}
}
- lookup.build(new TermFreqPayloadArrayIterator(input));
+ lookup.build(new InputArrayIterator(input));
return lookup;
}
@@ -220,7 +220,7 @@ public class LookupBenchmarkTest extends
final Lookup lookup = buildLookup(cls, dictionaryInput);
final List<String> input = new ArrayList<String>(benchmarkInput.size());
- for (TermFreqPayload tf : benchmarkInput) {
+ for (Input tf : benchmarkInput) {
String s = tf.term.utf8ToString();
String sub = s.substring(0, Math.min(s.length(),
minPrefixLen + random.nextInt(maxPrefixLen - minPrefixLen + 1)));
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java Mon Oct 14 17:34:30 2013
@@ -65,10 +65,10 @@ public class PersistenceTest extends Luc
// Add all input keys.
Lookup lookup = lookupClass.newInstance();
- TermFreqPayload[] keys = new TermFreqPayload[this.keys.length];
+ Input[] keys = new Input[this.keys.length];
for (int i = 0; i < keys.length; i++)
- keys[i] = new TermFreqPayload(this.keys[i], i);
- lookup.build(new TermFreqPayloadArrayIterator(keys));
+ keys[i] = new Input(this.keys[i], i);
+ lookup.build(new InputArrayIterator(keys));
// Store the suggester.
File storeDir = TEMP_DIR;
@@ -81,7 +81,7 @@ public class PersistenceTest extends Luc
// Assert validity.
Random random = random();
long previous = Long.MIN_VALUE;
- for (TermFreqPayload k : keys) {
+ for (Input k : keys) {
List<LookupResult> list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1);
assertEquals(1, list.size());
LookupResult lookupResult = list.get(0);
Copied: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java (from r1531951, lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java?p2=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java&p1=lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java&r1=1531951&r2=1531987&rev=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqPayloadIterator.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestInputIterator.java Mon Oct 14 17:34:30 2013
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -32,13 +31,13 @@ import org.apache.lucene.util.BytesRefHa
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-public class TestTermFreqPayloadIterator extends LuceneTestCase {
+public class TestInputIterator extends LuceneTestCase {
public void testEmpty() throws Exception {
- TermFreqPayloadArrayIterator iterator = new TermFreqPayloadArrayIterator(new TermFreqPayload[0]);
- TermFreqPayloadIterator wrapper = new SortedTermFreqPayloadIteratorWrapper(iterator, BytesRef.getUTF8SortedAsUnicodeComparator());
+ InputArrayIterator iterator = new InputArrayIterator(new Input[0]);
+ InputIterator wrapper = new SortedInputIterator(iterator, BytesRef.getUTF8SortedAsUnicodeComparator());
assertNull(wrapper.next());
- wrapper = new UnsortedTermFreqPayloadIteratorWrapper(iterator);
+ wrapper = new UnsortedInputIterator(iterator);
assertNull(wrapper.next());
}
@@ -49,8 +48,8 @@ public class TestTermFreqPayloadIterator
Comparator<BytesRef> comparator = random.nextBoolean() ? BytesRef.getUTF8SortedAsUnicodeComparator() : BytesRef.getUTF8SortedAsUTF16Comparator();
TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> sorted = new TreeMap<>(comparator);
TreeMap<BytesRef, Long> sortedWithoutPayload = new TreeMap<>(comparator);
- TermFreqPayload[] unsorted = new TermFreqPayload[num];
- TermFreqPayload[] unsortedWithoutPayload = new TermFreqPayload[num];
+ Input[] unsorted = new Input[num];
+ Input[] unsortedWithoutPayload = new Input[num];
for (int i = 0; i < num; i++) {
BytesRef key;
@@ -62,12 +61,12 @@ public class TestTermFreqPayloadIterator
long value = random.nextLong();
sortedWithoutPayload.put(key, value);
sorted.put(key, new SimpleEntry<>(value, payload));
- unsorted[i] = new TermFreqPayload(key, value, payload);
- unsortedWithoutPayload[i] = new TermFreqPayload(key, value);
+ unsorted[i] = new Input(key, value, payload);
+ unsortedWithoutPayload[i] = new Input(key, value);
}
// test the sorted iterator wrapper with payloads
- TermFreqPayloadIterator wrapper = new SortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsorted), comparator);
+ InputIterator wrapper = new SortedInputIterator(new InputArrayIterator(unsorted), comparator);
Iterator<Map.Entry<BytesRef, SimpleEntry<Long, BytesRef>>> expected = sorted.entrySet().iterator();
while (expected.hasNext()) {
Map.Entry<BytesRef,SimpleEntry<Long, BytesRef>> entry = expected.next();
@@ -79,7 +78,7 @@ public class TestTermFreqPayloadIterator
assertNull(wrapper.next());
// test the unsorted iterator wrapper with payloads
- wrapper = new UnsortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsorted));
+ wrapper = new UnsortedInputIterator(new InputArrayIterator(unsorted));
TreeMap<BytesRef, SimpleEntry<Long, BytesRef>> actual = new TreeMap<>();
BytesRef key;
while ((key = wrapper.next()) != null) {
@@ -90,7 +89,7 @@ public class TestTermFreqPayloadIterator
assertEquals(sorted, actual);
// test the sorted iterator wrapper without payloads
- TermFreqPayloadIterator wrapperWithoutPayload = new SortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsortedWithoutPayload), comparator);
+ InputIterator wrapperWithoutPayload = new SortedInputIterator(new InputArrayIterator(unsortedWithoutPayload), comparator);
Iterator<Map.Entry<BytesRef, Long>> expectedWithoutPayload = sortedWithoutPayload.entrySet().iterator();
while (expectedWithoutPayload.hasNext()) {
Map.Entry<BytesRef, Long> entry = expectedWithoutPayload.next();
@@ -102,7 +101,7 @@ public class TestTermFreqPayloadIterator
assertNull(wrapperWithoutPayload.next());
// test the unsorted iterator wrapper without payloads
- wrapperWithoutPayload = new UnsortedTermFreqPayloadIteratorWrapper(new TermFreqPayloadArrayIterator(unsortedWithoutPayload));
+ wrapperWithoutPayload = new UnsortedInputIterator(new InputArrayIterator(unsortedWithoutPayload));
TreeMap<BytesRef, Long> actualWithoutPayload = new TreeMap<>();
while ((key = wrapperWithoutPayload.next()) != null) {
long value = wrapperWithoutPayload.weight();
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Mon Oct 14 17:34:30 2013
@@ -34,8 +34,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -47,9 +47,9 @@ import org.apache.lucene.util._TestUtil;
public class AnalyzingInfixSuggesterTest extends LuceneTestCase {
public void testBasic() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -61,7 +61,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(2, results.size());
@@ -95,9 +95,9 @@ public class AnalyzingInfixSuggesterTest
}
public void testAfterLoad() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -109,7 +109,7 @@ public class AnalyzingInfixSuggesterTest
return newFSDirectory(path);
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
suggester.close();
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@@ -150,8 +150,8 @@ public class AnalyzingInfixSuggesterTest
@SuppressWarnings("unchecked")
public void testHighlightAsObject() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -204,7 +204,7 @@ public class AnalyzingInfixSuggesterTest
}
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(1, results.size());
@@ -230,9 +230,9 @@ public class AnalyzingInfixSuggesterTest
}
public void testRandomMinPrefixLength() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("lend me your ear", 8, new BytesRef("foobar")),
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -245,7 +245,7 @@ public class AnalyzingInfixSuggesterTest
return newFSDirectory(path);
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
for(int i=0;i<2;i++) {
for(int j=0;j<2;j++) {
@@ -312,8 +312,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testHighlight() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -325,7 +325,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>penn</b>y saved is a <b>penn</b>y earned", results.get(0).key);
@@ -333,8 +333,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testHighlightCaseChange() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a Penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a Penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -346,7 +346,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>Penn</b>y saved is a <b>penn</b>y earned", results.get(0).key);
@@ -367,7 +367,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
results = suggester.lookup(_TestUtil.stringToCharSequence("penn", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a <b>Penny</b> saved is a <b>penny</b> earned", results.get(0).key);
@@ -375,8 +375,8 @@ public class AnalyzingInfixSuggesterTest
}
public void testDoubleClose() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
};
File tempDir = _TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
@@ -388,7 +388,7 @@ public class AnalyzingInfixSuggesterTest
return newDirectory();
}
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
suggester.close();
suggester.close();
}
@@ -422,11 +422,11 @@ public class AnalyzingInfixSuggesterTest
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("a bob for apples", 10, new BytesRef("foobaz")),
+ Input keys[] = new Input[] {
+ new Input("a bob for apples", 10, new BytesRef("foobaz")),
};
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("a", random()), 10, true, true);
assertEquals(1, results.size());
assertEquals("a bob for <b>a</b>pples", results.get(0).key);
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1531987&r1=1531986&r2=1531987&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Mon Oct 14 17:34:30 2013
@@ -52,8 +52,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -63,18 +63,18 @@ public class AnalyzingSuggesterTest exte
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo", 50),
- new TermFreqPayload("bar", 10),
- new TermFreqPayload("barbar", 10),
- new TermFreqPayload("barbar", 12),
- new TermFreqPayload("barbara", 6),
- new TermFreqPayload("bar", 5),
- new TermFreqPayload("barbara", 1)
+ Iterable<Input> keys = shuffle(
+ new Input("foo", 50),
+ new Input("bar", 10),
+ new Input("barbar", 10),
+ new Input("barbar", 12),
+ new Input("barbara", 6),
+ new Input("bar", 5),
+ new Input("barbara", 1)
);
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
@@ -109,16 +109,16 @@ public class AnalyzingSuggesterTest exte
}
public void testKeywordWithPayloads() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo", 50, new BytesRef("hello")),
- new TermFreqPayload("bar", 10, new BytesRef("goodbye")),
- new TermFreqPayload("barbar", 12, new BytesRef("thank you")),
- new TermFreqPayload("bar", 9, new BytesRef("should be deduplicated")),
- new TermFreqPayload("bar", 8, new BytesRef("should also be deduplicated")),
- new TermFreqPayload("barbara", 6, new BytesRef("for all the fish")));
+ Iterable<Input> keys = shuffle(
+ new Input("foo", 50, new BytesRef("hello")),
+ new Input("bar", 10, new BytesRef("goodbye")),
+ new Input("barbar", 12, new BytesRef("thank you")),
+ new Input("bar", 9, new BytesRef("should be deduplicated")),
+ new Input("bar", 8, new BytesRef("should also be deduplicated")),
+ new Input("barbara", 6, new BytesRef("for all the fish")));
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
for (int i = 0; i < 2; i++) {
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
@@ -163,14 +163,14 @@ public class AnalyzingSuggesterTest exte
public void testRandomRealisticKeys() throws IOException {
LineFileDocs lineFile = new LineFileDocs(random());
Map<String, Long> mapping = new HashMap<>();
- List<TermFreqPayload> keys = new ArrayList<>();
+ List<Input> keys = new ArrayList<>();
int howMany = atLeast(100); // this might bring up duplicates
for (int i = 0; i < howMany; i++) {
Document nextDoc = lineFile.nextDoc();
String title = nextDoc.getField("title").stringValue();
int randomWeight = random().nextInt(100);
- keys.add(new TermFreqPayload(title, randomWeight));
+ keys.add(new Input(title, randomWeight));
if (!mapping.containsKey(title) || mapping.get(title) < randomWeight) {
mapping.put(title, Long.valueOf(randomWeight));
}
@@ -180,16 +180,16 @@ public class AnalyzingSuggesterTest exte
analyzingSuggester.setPreservePositionIncrements(random().nextBoolean());
boolean doPayloads = random().nextBoolean();
if (doPayloads) {
- List<TermFreqPayload> keysAndPayloads = new ArrayList<>();
- for (TermFreqPayload termFreq : keys) {
- keysAndPayloads.add(new TermFreqPayload(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
+ List<Input> keysAndPayloads = new ArrayList<>();
+ for (Input termFreq : keys) {
+ keysAndPayloads.add(new Input(termFreq.term, termFreq.v, new BytesRef(Long.toString(termFreq.v))));
}
- analyzingSuggester.build(new TermFreqPayloadArrayIterator(keysAndPayloads));
+ analyzingSuggester.build(new InputArrayIterator(keysAndPayloads));
} else {
- analyzingSuggester.build(new TermFreqPayloadArrayIterator(keys));
+ analyzingSuggester.build(new InputArrayIterator(keys));
}
- for (TermFreqPayload termFreq : keys) {
+ for (Input termFreq : keys) {
List<LookupResult> lookup = analyzingSuggester.lookup(termFreq.term.utf8ToString(), false, keys.size());
for (LookupResult lookupResult : lookup) {
assertEquals(mapping.get(lookupResult.key), Long.valueOf(lookupResult.value));
@@ -209,14 +209,14 @@ public class AnalyzingSuggesterTest exte
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("the ghost of christmas past", 50),
+ Input keys[] = new Input[] {
+ new Input("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -239,23 +239,23 @@ public class AnalyzingSuggesterTest exte
public void testEmpty() throws Exception {
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
AnalyzingSuggester suggester = new AnalyzingSuggester(standard);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
+ suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}
public void testNoSeps() throws Exception {
- TermFreqPayload[] keys = new TermFreqPayload[] {
- new TermFreqPayload("ab cd", 0),
- new TermFreqPayload("abcd", 1),
+ Input[] keys = new Input[] {
+ new Input("ab cd", 0),
+ new Input("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, options, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -316,13 +316,13 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("wifi network is slow", 50),
- new TermFreqPayload("wi fi network is fast", 10),
+ Input keys[] = new Input[] {
+ new Input("wifi network is slow", 50),
+ new Input("wi fi network is fast", 10),
};
//AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
System.out.println("Results: " + results);
@@ -382,12 +382,12 @@ public class AnalyzingSuggesterTest exte
}
};
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("ab xc", 50),
- new TermFreqPayload("ba xd", 50),
+ Input keys[] = new Input[] {
+ new Input("ab xc", 50),
+ new Input("ba xd", 50),
};
AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -460,11 +460,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -500,11 +500,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = getUnusualAnalyzer();
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 1),
- new TermFreqPayload("x y z", 3),
- new TermFreqPayload("x", 2),
- new TermFreqPayload("z z z", 20),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 1),
+ new Input("x y z", 3),
+ new Input("x", 2),
+ new Input("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -655,12 +655,12 @@ public class AnalyzingSuggesterTest exte
boolean doPayloads = random().nextBoolean();
- TermFreqPayload[] keys = null;
- TermFreqPayload[] payloadKeys = null;
+ Input[] keys = null;
+ Input[] payloadKeys = null;
if (doPayloads) {
- payloadKeys = new TermFreqPayload[numQueries];
+ payloadKeys = new Input[numQueries];
} else {
- keys = new TermFreqPayload[numQueries];
+ keys = new Input[numQueries];
}
boolean preserveSep = random().nextBoolean();
@@ -731,9 +731,9 @@ public class AnalyzingSuggesterTest exte
byte[] bytes = new byte[random().nextInt(10)];
random().nextBytes(bytes);
payload = new BytesRef(bytes);
- payloadKeys[i] = new TermFreqPayload(key, weight, payload);
+ payloadKeys[i] = new Input(key, weight, payload);
} else {
- keys[i] = new TermFreqPayload(key, weight);
+ keys[i] = new Input(key, weight);
payload = null;
}
@@ -754,9 +754,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1);
if (doPayloads) {
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(payloadKeys)));
+ suggester.build(new InputArrayIterator(shuffle(payloadKeys)));
} else {
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(keys)));
+ suggester.build(new InputArrayIterator(shuffle(keys)));
}
for (String prefix : allPrefixes) {
@@ -874,8 +874,8 @@ public class AnalyzingSuggesterTest exte
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 2, -1);
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(new TermFreqPayload("a", 40),
- new TermFreqPayload("a ", 50), new TermFreqPayload(" a", 60))));
+ suggester.build(new InputArrayIterator(shuffle(new Input("a", 40),
+ new Input("a ", 50), new Input(" a", 60))));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -889,11 +889,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 2),
- new TermFreqPayload("a b c", 3),
- new TermFreqPayload("a c a", 1),
- new TermFreqPayload("a c b", 1),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 2),
+ new Input("a b c", 3),
+ new Input("a c a", 1),
+ new Input("a c b", 1),
}));
suggester.lookup("a", false, 4);
@@ -905,10 +905,10 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.EXACT_FIRST, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 5),
- new TermFreqPayload("a b", 3),
- new TermFreqPayload("a c", 4),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 5),
+ new Input("a b", 3),
+ new Input("a c", 4),
}));
List<LookupResult> results = suggester.lookup("a", false, 3);
@@ -970,9 +970,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(shuffle(
- new TermFreqPayload("hambone", 6),
- new TermFreqPayload("nellie", 5))));
+ suggester.build(new InputArrayIterator(shuffle(
+ new Input("hambone", 6),
+ new Input("nellie", 5))));
List<LookupResult> results = suggester.lookup("nellie", false, 2);
assertEquals(2, results.size());
@@ -1039,9 +1039,9 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a", 6),
- new TermFreqPayload("b", 5),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a", 6),
+ new Input("b", 5),
}));
List<LookupResult> results = suggester.lookup("a", false, 2);
@@ -1112,21 +1112,21 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a a", 50),
- new TermFreqPayload("a b", 50),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a a", 50),
+ new Input("a b", 50),
}));
}
public void testDupSurfaceFormsMissingResults3() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("a a", 7),
- new TermFreqPayload("a a", 7),
- new TermFreqPayload("a c", 6),
- new TermFreqPayload("a c", 3),
- new TermFreqPayload("a b", 5),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("a a", 7),
+ new Input("a a", 7),
+ new Input("a c", 6),
+ new Input("a c", 3),
+ new Input("a b", 5),
}));
assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString());
}
@@ -1134,9 +1134,9 @@ public class AnalyzingSuggesterTest exte
public void testEndingSpace() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("i love lucy", 7),
- new TermFreqPayload("isla de muerta", 8),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("i love lucy", 7),
+ new Input("isla de muerta", 8),
}));
assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
@@ -1167,15 +1167,15 @@ public class AnalyzingSuggesterTest exte
};
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, 1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {new TermFreqPayload("a", 1)}));
+ suggester.build(new InputArrayIterator(new Input[] {new Input("a", 1)}));
assertEquals("[a/1]", suggester.lookup("a", false, 1).toString());
}
public void testIllegalLookupArgument() throws Exception {
Analyzer a = new MockAnalyzer(random());
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, 0, 256, -1);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("а где ÐÑÑи?", 7),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("а где ÐÑÑи?", 7),
}));
try {
suggester.lookup("а\u001E", false, 3);