You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/13 12:26:23 UTC
svn commit: r1531664 [2/2] - in /lucene/dev/trunk/lucene: ./
suggest/src/java/org/apache/lucene/search/spell/
suggest/src/java/org/apache/lucene/search/suggest/
suggest/src/java/org/apache/lucene/search/suggest/analyzing/
suggest/src/java/org/apache/lu...
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1531664&r1=1531663&r2=1531664&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Sun Oct 13 10:26:22 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreq;
-import org.apache.lucene.search.suggest.TermFreqArrayIterator;
+import org.apache.lucene.search.suggest.TermFreqPayload;
+import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -54,16 +54,16 @@ import org.apache.lucene.util.fst.Util;
public class FuzzySuggesterTest extends LuceneTestCase {
public void testRandomEdits() throws IOException {
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreq("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new TermFreqPayload("boo" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreq("foo bar boo far", 12));
+ keys.add(new TermFreqPayload("foo bar boo far", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("foo bar boo", FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX);
@@ -75,16 +75,16 @@ public class FuzzySuggesterTest extends
}
public void testNonLatinRandomEdits() throws IOException {
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
int numTerms = atLeast(100);
for (int i = 0; i < numTerms; i++) {
- keys.add(new TermFreq("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
+ keys.add(new TermFreqPayload("бÑÑ" + _TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
}
- keys.add(new TermFreq("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
+ keys.add(new TermFreqPayload("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ ÑаÑ", 12));
MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
FuzzySuggester suggester = new FuzzySuggester(analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
int numIters = atLeast(10);
for (int i = 0; i < numIters; i++) {
String addRandomEdit = addRandomEdit("ÑÑÑ Ð±Ð°Ñ Ð±ÑÑ", 0);
@@ -97,15 +97,15 @@ public class FuzzySuggesterTest extends
/** this is basically the WFST test ported to KeywordAnalyzer. so it acts the same */
public void testKeyword() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("foo", 50),
- new TermFreq("bar", 10),
- new TermFreq("barbar", 12),
- new TermFreq("barbara", 6)
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("foo", 50),
+ new TermFreqPayload("bar", 10),
+ new TermFreqPayload("barbar", 12),
+ new TermFreqPayload("barbara", 6)
};
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("bariar", random()), false, 2);
assertEquals(2, results.size());
@@ -172,14 +172,14 @@ public class FuzzySuggesterTest extends
* basic "standardanalyzer" test with stopword removal
*/
public void testStandard() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("the ghost of christmas past", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("the ghost of christmas past", 50),
};
Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
FuzzySuggester suggester = new FuzzySuggester(standard);
suggester.setPreservePositionIncrements(false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
assertEquals(1, results.size());
@@ -200,16 +200,16 @@ public class FuzzySuggesterTest extends
}
public void testNoSeps() throws Exception {
- TermFreq[] keys = new TermFreq[] {
- new TermFreq("ab cd", 0),
- new TermFreq("abcd", 1),
+ TermFreqPayload[] keys = new TermFreqPayload[] {
+ new TermFreqPayload("ab cd", 0),
+ new TermFreqPayload("abcd", 1),
};
int options = 0;
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, options, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
// TODO: would be nice if "ab " would allow the test to
// pass, and more generally if the analyzer can know
// that the user's current query has ended at a word,
@@ -270,12 +270,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("wifi network is slow", 50),
- new TermFreq("wi fi network is fast", 10),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("wifi network is slow", 50),
+ new TermFreqPayload("wi fi network is fast", 10),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("wifi network", false, 10);
if (VERBOSE) {
@@ -290,7 +290,7 @@ public class FuzzySuggesterTest extends
public void testEmpty() throws Exception {
FuzzySuggester suggester = new FuzzySuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
- suggester.build(new TermFreqArrayIterator(new TermFreq[0]));
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
@@ -344,12 +344,12 @@ public class FuzzySuggesterTest extends
}
};
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("ab xc", 50),
- new TermFreq("ba xd", 50),
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("ab xc", 50),
+ new TermFreqPayload("ba xd", 50),
};
FuzzySuggester suggester = new FuzzySuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
assertTrue(results.size() == 1);
}
@@ -418,11 +418,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
//System.out.println("ALL: " + suggester.lookup("x y", false, 6));
@@ -458,11 +458,11 @@ public class FuzzySuggesterTest extends
Analyzer a = getUnusualAnalyzer();
FuzzySuggester suggester = new FuzzySuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, 1, true, 1, 3, false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 1),
- new TermFreq("x y z", 3),
- new TermFreq("x", 2),
- new TermFreq("z z z", 20),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 1),
+ new TermFreqPayload("x y z", 3),
+ new TermFreqPayload("x", 2),
+ new TermFreqPayload("z z z", 20),
}));
for(int topN=1;topN<6;topN++) {
@@ -491,19 +491,19 @@ public class FuzzySuggesterTest extends
}
// Holds surface form separately:
- private static class TermFreq2 implements Comparable<TermFreq2> {
+ private static class TermFreqPayload2 implements Comparable<TermFreqPayload2> {
public final String surfaceForm;
public final String analyzedForm;
public final long weight;
- public TermFreq2(String surfaceForm, String analyzedForm, long weight) {
+ public TermFreqPayload2(String surfaceForm, String analyzedForm, long weight) {
this.surfaceForm = surfaceForm;
this.analyzedForm = analyzedForm;
this.weight = weight;
}
@Override
- public int compareTo(TermFreq2 other) {
+ public int compareTo(TermFreqPayload2 other) {
int cmp = analyzedForm.compareTo(other.analyzedForm);
if (cmp != 0) {
return cmp;
@@ -596,11 +596,11 @@ public class FuzzySuggesterTest extends
int numQueries = atLeast(100);
- final List<TermFreq2> slowCompletor = new ArrayList<TermFreq2>();
+ final List<TermFreqPayload2> slowCompletor = new ArrayList<TermFreqPayload2>();
final TreeSet<String> allPrefixes = new TreeSet<String>();
final Set<String> seen = new HashSet<String>();
- TermFreq[] keys = new TermFreq[numQueries];
+ TermFreqPayload[] keys = new TermFreqPayload[numQueries];
boolean preserveSep = random().nextBoolean();
boolean unicodeAware = random().nextBoolean();
@@ -666,17 +666,17 @@ public class FuzzySuggesterTest extends
}
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
- keys[i] = new TermFreq(key, weight);
+ keys[i] = new TermFreqPayload(key, weight);
- slowCompletor.add(new TermFreq2(key, analyzedKey, weight));
+ slowCompletor.add(new TermFreqPayload2(key, analyzedKey, weight));
}
if (VERBOSE) {
// Don't just sort original list, to avoid VERBOSE
// altering the test:
- List<TermFreq2> sorted = new ArrayList<TermFreq2>(slowCompletor);
+ List<TermFreqPayload2> sorted = new ArrayList<TermFreqPayload2>(slowCompletor);
Collections.sort(sorted);
- for(TermFreq2 ent : sorted) {
+ for(TermFreqPayload2 ent : sorted) {
System.out.println(" surface='" + ent.surfaceForm + " analyzed='" + ent.analyzedForm + "' weight=" + ent.weight);
}
}
@@ -684,7 +684,7 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
FuzzySuggester suggester = new FuzzySuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1, 1, false, 1, 3, unicodeAware);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
for (String prefix : allPrefixes) {
@@ -756,7 +756,7 @@ public class FuzzySuggesterTest extends
assertTrue(automaton.isDeterministic());
// TODO: could be faster... but its slowCompletor for a reason
BytesRef spare = new BytesRef();
- for (TermFreq2 e : slowCompletor) {
+ for (TermFreqPayload2 e : slowCompletor) {
spare.copyChars(e.analyzedForm);
Set<IntsRef> finiteStrings = suggester.toFiniteStrings(spare, tokenStreamToAutomaton);
for (IntsRef intsRef : finiteStrings) {
@@ -825,14 +825,14 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, 0, 2, -1, 1, true, 1, 3, false);
- List<TermFreq> keys = Arrays.asList(new TermFreq[] {
- new TermFreq("a", 40),
- new TermFreq("a ", 50),
- new TermFreq(" a", 60),
+ List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
+ new TermFreqPayload("a", 40),
+ new TermFreqPayload("a ", 50),
+ new TermFreqPayload(" a", 60),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
List<LookupResult> results = suggester.lookup("a", false, 5);
assertEquals(2, results.size());
@@ -846,15 +846,15 @@ public class FuzzySuggesterTest extends
Analyzer a = new MockAnalyzer(random());
FuzzySuggester suggester = new FuzzySuggester(a, a, FuzzySuggester.PRESERVE_SEP, 2, -1, 2, true, 1, 3, false);
- List<TermFreq> keys = Arrays.asList(new TermFreq[] {
- new TermFreq("foo bar", 40),
- new TermFreq("foo bar baz", 50),
- new TermFreq("barbaz", 60),
- new TermFreq("barbazfoo", 10),
+ List<TermFreqPayload> keys = Arrays.asList(new TermFreqPayload[] {
+ new TermFreqPayload("foo bar", 40),
+ new TermFreqPayload("foo bar baz", 50),
+ new TermFreqPayload("barbaz", 60),
+ new TermFreqPayload("barbazfoo", 10),
});
Collections.shuffle(keys, random());
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
assertEquals("[foo bar baz/50, foo bar/40]", suggester.lookup("foobar", false, 5).toString());
assertEquals("[foo bar baz/50]", suggester.lookup("foobarbaz", false, 5).toString());
@@ -929,25 +929,25 @@ public class FuzzySuggesterTest extends
public void testRandom2() throws Throwable {
final int NUM = atLeast(200);
- final List<TermFreq> answers = new ArrayList<TermFreq>();
+ final List<TermFreqPayload> answers = new ArrayList<TermFreqPayload>();
final Set<String> seen = new HashSet<String>();
for(int i=0;i<NUM;i++) {
final String s = randomSimpleString(8);
if (!seen.contains(s)) {
- answers.add(new TermFreq(s, random().nextInt(1000)));
+ answers.add(new TermFreqPayload(s, random().nextInt(1000)));
seen.add(s);
}
}
- Collections.sort(answers, new Comparator<TermFreq>() {
+ Collections.sort(answers, new Comparator<TermFreqPayload>() {
@Override
- public int compare(TermFreq a, TermFreq b) {
+ public int compare(TermFreqPayload a, TermFreqPayload b) {
return a.term.compareTo(b.term);
}
});
if (VERBOSE) {
System.out.println("\nTEST: targets");
- for(TermFreq tf : answers) {
+ for(TermFreqPayload tf : answers) {
System.out.println(" " + tf.term.utf8ToString() + " freq=" + tf.v);
}
}
@@ -965,7 +965,7 @@ public class FuzzySuggesterTest extends
}
Collections.shuffle(answers, random());
- suggest.build(new TermFreqArrayIterator(answers.toArray(new TermFreq[answers.size()])));
+ suggest.build(new TermFreqPayloadArrayIterator(answers.toArray(new TermFreqPayload[answers.size()])));
final int ITERS = atLeast(100);
for(int iter=0;iter<ITERS;iter++) {
@@ -1004,10 +1004,10 @@ public class FuzzySuggesterTest extends
}
}
- private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<TermFreq> answers, String frag) {
+ private List<LookupResult> slowFuzzyMatch(int prefixLen, int maxEdits, boolean allowTransposition, List<TermFreqPayload> answers, String frag) {
final List<LookupResult> results = new ArrayList<LookupResult>();
final int fragLen = frag.length();
- for(TermFreq tf : answers) {
+ for(TermFreqPayload tf : answers) {
//System.out.println(" check s=" + tf.term.utf8ToString());
boolean prefixMatches = true;
for(int i=0;i<prefixLen;i++) {
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java?rev=1531664&r1=1531663&r2=1531664&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java Sun Oct 13 10:26:22 2013
@@ -41,10 +41,10 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.document.Document;
-import org.apache.lucene.search.spell.TermFreqIterator;
+import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreq;
-import org.apache.lucene.search.suggest.TermFreqArrayIterator;
+import org.apache.lucene.search.suggest.TermFreqPayload;
+import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -54,14 +54,14 @@ import org.junit.Ignore;
public class TestFreeTextSuggester extends LuceneTestCase {
public void testBasic() throws Exception {
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo bar baz blah", 50),
- new TermFreq("boo foo bar foo bee", 20)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo bar baz blah", 50),
+ new TermFreqPayload("boo foo bar foo bee", 20)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
for(int i=0;i<2;i++) {
@@ -101,12 +101,12 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringBuild() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo\u001ebar baz", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo\u001ebar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
try {
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
@@ -116,11 +116,11 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringQuery() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo bar baz", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo bar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
try {
sug.lookup("foo\u001eb", 10);
@@ -136,7 +136,7 @@ public class TestFreeTextSuggester exten
// Skip header:
lfd.nextDoc();
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
- sug.build(new TermFreqIterator() {
+ sug.build(new TermFreqPayloadIterator() {
private int count;
@@ -161,6 +161,16 @@ public class TestFreeTextSuggester exten
}
return new BytesRef(doc.get("body"));
}
+
+ @Override
+ public BytesRef payload() {
+ return null;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
});
if (VERBOSE) {
System.out.println(sug.sizeInBytes() + " bytes");
@@ -175,13 +185,13 @@ public class TestFreeTextSuggester exten
// Make sure you can suggest based only on unigram model:
public void testUnigrams() throws Exception {
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo bar baz blah boo foo bar foo bee", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo bar baz blah boo foo bar foo bee", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 1, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
// Sorts first by count, descending, second by term, ascending
assertEquals("bar/0.22 baz/0.11 bee/0.11 blah/0.11 boo/0.11",
toString(sug.lookup("b", 10)));
@@ -189,24 +199,24 @@ public class TestFreeTextSuggester exten
// Make sure the last token is not duplicated
public void testNoDupsAcrossGrams() throws Exception {
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo bar bar bar bar", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
assertEquals("foo bar/1.00",
toString(sug.lookup("foo b", 10)));
}
// Lookup of just empty string produces unicode only matches:
public void testEmptyString() throws Exception {
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("foo bar bar bar bar", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
try {
sug.lookup("", 10);
fail("did not hit exception");
@@ -228,11 +238,11 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("wizard of oz", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("wizard of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
assertEquals("wizard _ oz/1.00",
toString(sug.lookup("wizard of", 10)));
@@ -256,11 +266,11 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<TermFreq> keys = shuffle(
- new TermFreq("wizard of of oz", 50)
+ Iterable<TermFreqPayload> keys = shuffle(
+ new TermFreqPayload("wizard of of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
- sug.build(new TermFreqArrayIterator(keys));
+ sug.build(new TermFreqPayloadArrayIterator(keys));
assertEquals("",
toString(sug.lookup("wizard of of", 10)));
}
@@ -320,7 +330,7 @@ public class TestFreeTextSuggester exten
// Build suggester model:
FreeTextSuggester sug = new FreeTextSuggester(a, a, grams, (byte) 0x20);
- sug.build(new TermFreqIterator() {
+ sug.build(new TermFreqPayloadIterator() {
int upto;
@Override
@@ -342,6 +352,16 @@ public class TestFreeTextSuggester exten
public long weight() {
return random().nextLong();
}
+
+ @Override
+ public BytesRef payload() {
+ return null;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
});
// Build inefficient but hopefully correct model:
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1531664&r1=1531663&r2=1531664&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Sun Oct 13 10:26:22 2013
@@ -28,8 +28,8 @@ import org.apache.lucene.util.*;
* Unit tests for {@link FSTCompletion}.
*/
public class FSTCompletionTest extends LuceneTestCase {
- public static TermFreq tf(String t, int v) {
- return new TermFreq(t, v);
+ public static TermFreqPayload tf(String t, int v) {
+ return new TermFreqPayload(t, v);
}
private FSTCompletion completion;
@@ -40,15 +40,15 @@ public class FSTCompletionTest extends L
super.setUp();
FSTCompletionBuilder builder = new FSTCompletionBuilder();
- for (TermFreq tf : evalKeys()) {
+ for (TermFreqPayload tf : evalKeys()) {
builder.add(tf.term, (int) tf.v);
}
completion = builder.build();
completionAlphabetical = new FSTCompletion(completion.getFST(), false, true);
}
- private TermFreq[] evalKeys() {
- final TermFreq[] keys = new TermFreq[] {
+ private TermFreqPayload[] evalKeys() {
+ final TermFreqPayload[] keys = new TermFreqPayload[] {
tf("one", 0),
tf("oneness", 1),
tf("onerous", 1),
@@ -157,17 +157,17 @@ public class FSTCompletionTest extends L
FSTCompletionLookup lookup = new FSTCompletionLookup(10, true);
Random r = random();
- List<TermFreq> keys = new ArrayList<TermFreq>();
+ List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
for (int i = 0; i < 5000; i++) {
- keys.add(new TermFreq(_TestUtil.randomSimpleString(r), -1));
+ keys.add(new TermFreqPayload(_TestUtil.randomSimpleString(r), -1));
}
- lookup.build(new TermFreqArrayIterator(keys));
+ lookup.build(new TermFreqPayloadArrayIterator(keys));
// All the weights were constant, so all returned buckets must be constant, whatever they
// are.
Long previous = null;
- for (TermFreq tf : keys) {
+ for (TermFreqPayload tf : keys) {
Long current = ((Number)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random()))).longValue();
if (previous != null) {
assertEquals(previous, current);
@@ -177,11 +177,11 @@ public class FSTCompletionTest extends L
}
public void testMultilingualInput() throws Exception {
- List<TermFreq> input = LookupBenchmarkTest.readTop50KWiki();
+ List<TermFreqPayload> input = LookupBenchmarkTest.readTop50KWiki();
FSTCompletionLookup lookup = new FSTCompletionLookup();
- lookup.build(new TermFreqArrayIterator(input));
- for (TermFreq tf : input) {
+ lookup.build(new TermFreqPayloadArrayIterator(input));
+ for (TermFreqPayload tf : input) {
assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random())));
assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random()), true, 1).get(0).key.toString());
}
@@ -198,17 +198,17 @@ public class FSTCompletionTest extends L
}
public void testRandom() throws Exception {
- List<TermFreq> freqs = new ArrayList<TermFreq>();
+ List<TermFreqPayload> freqs = new ArrayList<TermFreqPayload>();
Random rnd = random();
for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
int weight = rnd.nextInt(100);
- freqs.add(new TermFreq("" + rnd.nextLong(), weight));
+ freqs.add(new TermFreqPayload("" + rnd.nextLong(), weight));
}
FSTCompletionLookup lookup = new FSTCompletionLookup();
- lookup.build(new TermFreqArrayIterator(freqs.toArray(new TermFreq[freqs.size()])));
+ lookup.build(new TermFreqPayloadArrayIterator(freqs.toArray(new TermFreqPayload[freqs.size()])));
- for (TermFreq tf : freqs) {
+ for (TermFreqPayload tf : freqs) {
final String term = tf.term.utf8ToString();
for (int i = 1; i < term.length(); i++) {
String prefix = term.substring(0, i);
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1531664&r1=1531663&r2=1531664&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Sun Oct 13 10:26:22 2013
@@ -20,8 +20,8 @@ package org.apache.lucene.search.suggest
import java.util.*;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreq;
-import org.apache.lucene.search.suggest.TermFreqArrayIterator;
+import org.apache.lucene.search.suggest.TermFreqPayload;
+import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -29,16 +29,16 @@ import org.apache.lucene.util._TestUtil;
public class WFSTCompletionTest extends LuceneTestCase {
public void testBasic() throws Exception {
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("foo", 50),
- new TermFreq("bar", 10),
- new TermFreq("barbar", 12),
- new TermFreq("barbara", 6)
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("foo", 50),
+ new TermFreqPayload("bar", 10),
+ new TermFreqPayload("barbar", 12),
+ new TermFreqPayload("barbara", 6)
};
Random random = new Random(random().nextLong());
WFSTCompletionLookup suggester = new WFSTCompletionLookup();
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random), false, 2);
@@ -81,9 +81,9 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(true);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 20),
- new TermFreq("x", 2),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 20),
+ new TermFreqPayload("x", 2),
}));
for(int topN=1;topN<4;topN++) {
@@ -105,9 +105,9 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq("x y", 20),
- new TermFreq("x", 2),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload("x y", 20),
+ new TermFreqPayload("x", 2),
}));
for(int topN=1;topN<4;topN++) {
@@ -131,7 +131,7 @@ public class WFSTCompletionTest extends
final TreeMap<String,Long> slowCompletor = new TreeMap<String,Long>();
final TreeSet<String> allPrefixes = new TreeSet<String>();
- TermFreq[] keys = new TermFreq[numWords];
+ TermFreqPayload[] keys = new TermFreqPayload[numWords];
for (int i = 0; i < numWords; i++) {
String s;
@@ -150,11 +150,11 @@ public class WFSTCompletionTest extends
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
slowCompletor.put(s, (long)weight);
- keys[i] = new TermFreq(s, weight);
+ keys[i] = new TermFreqPayload(s, weight);
}
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqArrayIterator(keys));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
Random random = new Random(random().nextLong());
for (String prefix : allPrefixes) {
@@ -205,16 +205,16 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[] {
- new TermFreq(key1, 50),
- new TermFreq(key2, 50),
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
+ new TermFreqPayload(key1, 50),
+ new TermFreqPayload(key2, 50),
}));
}
public void testEmpty() throws Exception {
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqArrayIterator(new TermFreq[0]));
+ suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}