You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/10/14 19:49:14 UTC
svn commit: r1532001 [2/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/suggest/ lucene/suggest/src/java/org/apache/lucene/search/spell/
lucene/suggest/src/java/org/apache/lucene/search/suggest/
lucene/suggest/src/java/org/apache/lucene/searc...
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java Mon Oct 14 17:49:13 2013
@@ -41,10 +41,10 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.document.Document;
-import org.apache.lucene.search.spell.TermFreqPayloadIterator;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -56,14 +56,14 @@ import org.junit.Ignore;
public class TestFreeTextSuggester extends LuceneTestCase {
public void testBasic() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo bar baz blah", 50),
- new TermFreqPayload("boo foo bar foo bee", 20)
+ Iterable<Input> keys = shuffle(
+ new Input("foo bar baz blah", 50),
+ new Input("boo foo bar foo bee", 20)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
for(int i=0;i<2;i++) {
@@ -103,12 +103,12 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringBuild() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo\u001ebar baz", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("foo\u001ebar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
try {
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
@@ -118,11 +118,11 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringQuery() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo bar baz", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("foo bar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
try {
sug.lookup("foo\u001eb", 10);
@@ -138,7 +138,7 @@ public class TestFreeTextSuggester exten
// Skip header:
lfd.nextDoc();
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
- sug.build(new TermFreqPayloadIterator() {
+ sug.build(new InputIterator() {
private int count;
@@ -192,13 +192,13 @@ public class TestFreeTextSuggester exten
// Make sure you can suggest based only on unigram model:
public void testUnigrams() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo bar baz blah boo foo bar foo bee", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("foo bar baz blah boo foo bar foo bee", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 1, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
// Sorts first by count, descending, second by term, ascending
assertEquals("bar/0.22 baz/0.11 bee/0.11 blah/0.11 boo/0.11",
toString(sug.lookup("b", 10)));
@@ -206,24 +206,24 @@ public class TestFreeTextSuggester exten
// Make sure the last token is not duplicated
public void testNoDupsAcrossGrams() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo bar bar bar bar", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
assertEquals("foo bar/1.00",
toString(sug.lookup("foo b", 10)));
}
// Lookup of just empty string produces unicode only matches:
public void testEmptyString() throws Exception {
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("foo bar bar bar bar", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
try {
sug.lookup("", 10);
fail("did not hit exception");
@@ -245,11 +245,11 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("wizard of oz", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("wizard of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
assertEquals("wizard _ oz/1.00",
toString(sug.lookup("wizard of", 10)));
@@ -273,11 +273,11 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<TermFreqPayload> keys = shuffle(
- new TermFreqPayload("wizard of of oz", 50)
+ Iterable<Input> keys = shuffle(
+ new Input("wizard of of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
- sug.build(new TermFreqPayloadArrayIterator(keys));
+ sug.build(new InputArrayIterator(keys));
assertEquals("",
toString(sug.lookup("wizard of of", 10)));
}
@@ -337,7 +337,7 @@ public class TestFreeTextSuggester exten
// Build suggester model:
FreeTextSuggester sug = new FreeTextSuggester(a, a, grams, (byte) 0x20);
- sug.build(new TermFreqPayloadIterator() {
+ sug.build(new InputIterator() {
int upto;
@Override
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Mon Oct 14 17:49:13 2013
@@ -28,8 +28,8 @@ import org.apache.lucene.util.*;
* Unit tests for {@link FSTCompletion}.
*/
public class FSTCompletionTest extends LuceneTestCase {
- public static TermFreqPayload tf(String t, int v) {
- return new TermFreqPayload(t, v);
+ public static Input tf(String t, int v) {
+ return new Input(t, v);
}
private FSTCompletion completion;
@@ -40,15 +40,15 @@ public class FSTCompletionTest extends L
super.setUp();
FSTCompletionBuilder builder = new FSTCompletionBuilder();
- for (TermFreqPayload tf : evalKeys()) {
+ for (Input tf : evalKeys()) {
builder.add(tf.term, (int) tf.v);
}
completion = builder.build();
completionAlphabetical = new FSTCompletion(completion.getFST(), false, true);
}
- private TermFreqPayload[] evalKeys() {
- final TermFreqPayload[] keys = new TermFreqPayload[] {
+ private Input[] evalKeys() {
+ final Input[] keys = new Input[] {
tf("one", 0),
tf("oneness", 1),
tf("onerous", 1),
@@ -157,17 +157,17 @@ public class FSTCompletionTest extends L
FSTCompletionLookup lookup = new FSTCompletionLookup(10, true);
Random r = random();
- List<TermFreqPayload> keys = new ArrayList<TermFreqPayload>();
+ List<Input> keys = new ArrayList<Input>();
for (int i = 0; i < 5000; i++) {
- keys.add(new TermFreqPayload(_TestUtil.randomSimpleString(r), -1));
+ keys.add(new Input(_TestUtil.randomSimpleString(r), -1));
}
- lookup.build(new TermFreqPayloadArrayIterator(keys));
+ lookup.build(new InputArrayIterator(keys));
// All the weights were constant, so all returned buckets must be constant, whatever they
// are.
Long previous = null;
- for (TermFreqPayload tf : keys) {
+ for (Input tf : keys) {
Long current = ((Number)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random()))).longValue();
if (previous != null) {
assertEquals(previous, current);
@@ -177,11 +177,11 @@ public class FSTCompletionTest extends L
}
public void testMultilingualInput() throws Exception {
- List<TermFreqPayload> input = LookupBenchmarkTest.readTop50KWiki();
+ List<Input> input = LookupBenchmarkTest.readTop50KWiki();
FSTCompletionLookup lookup = new FSTCompletionLookup();
- lookup.build(new TermFreqPayloadArrayIterator(input));
- for (TermFreqPayload tf : input) {
+ lookup.build(new InputArrayIterator(input));
+ for (Input tf : input) {
assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random())));
assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random()), true, 1).get(0).key.toString());
}
@@ -198,17 +198,17 @@ public class FSTCompletionTest extends L
}
public void testRandom() throws Exception {
- List<TermFreqPayload> freqs = new ArrayList<TermFreqPayload>();
+ List<Input> freqs = new ArrayList<Input>();
Random rnd = random();
for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
int weight = rnd.nextInt(100);
- freqs.add(new TermFreqPayload("" + rnd.nextLong(), weight));
+ freqs.add(new Input("" + rnd.nextLong(), weight));
}
FSTCompletionLookup lookup = new FSTCompletionLookup();
- lookup.build(new TermFreqPayloadArrayIterator(freqs.toArray(new TermFreqPayload[freqs.size()])));
+ lookup.build(new InputArrayIterator(freqs.toArray(new Input[freqs.size()])));
- for (TermFreqPayload tf : freqs) {
+ for (Input tf : freqs) {
final String term = tf.term.utf8ToString();
for (int i = 1; i < term.length(); i++) {
String prefix = term.substring(0, i);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1532001&r1=1532000&r2=1532001&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Mon Oct 14 17:49:13 2013
@@ -20,8 +20,8 @@ package org.apache.lucene.search.suggest
import java.util.*;
import org.apache.lucene.search.suggest.Lookup.LookupResult;
-import org.apache.lucene.search.suggest.TermFreqPayload;
-import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
+import org.apache.lucene.search.suggest.Input;
+import org.apache.lucene.search.suggest.InputArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -29,16 +29,16 @@ import org.apache.lucene.util._TestUtil;
public class WFSTCompletionTest extends LuceneTestCase {
public void testBasic() throws Exception {
- TermFreqPayload keys[] = new TermFreqPayload[] {
- new TermFreqPayload("foo", 50),
- new TermFreqPayload("bar", 10),
- new TermFreqPayload("barbar", 12),
- new TermFreqPayload("barbara", 6)
+ Input keys[] = new Input[] {
+ new Input("foo", 50),
+ new Input("bar", 10),
+ new Input("barbar", 12),
+ new Input("barbara", 6)
};
Random random = new Random(random().nextLong());
WFSTCompletionLookup suggester = new WFSTCompletionLookup();
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
// top N of 2, but only foo is available
List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random), false, 2);
@@ -81,9 +81,9 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(true);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 20),
- new TermFreqPayload("x", 2),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 20),
+ new Input("x", 2),
}));
for(int topN=1;topN<4;topN++) {
@@ -105,9 +105,9 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload("x y", 20),
- new TermFreqPayload("x", 2),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input("x y", 20),
+ new Input("x", 2),
}));
for(int topN=1;topN<4;topN++) {
@@ -131,7 +131,7 @@ public class WFSTCompletionTest extends
final TreeMap<String,Long> slowCompletor = new TreeMap<String,Long>();
final TreeSet<String> allPrefixes = new TreeSet<String>();
- TermFreqPayload[] keys = new TermFreqPayload[numWords];
+ Input[] keys = new Input[numWords];
for (int i = 0; i < numWords; i++) {
String s;
@@ -150,11 +150,11 @@ public class WFSTCompletionTest extends
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
slowCompletor.put(s, (long)weight);
- keys[i] = new TermFreqPayload(s, weight);
+ keys[i] = new Input(s, weight);
}
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqPayloadArrayIterator(keys));
+ suggester.build(new InputArrayIterator(keys));
Random random = new Random(random().nextLong());
for (String prefix : allPrefixes) {
@@ -205,16 +205,16 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[] {
- new TermFreqPayload(key1, 50),
- new TermFreqPayload(key2, 50),
+ suggester.build(new InputArrayIterator(new Input[] {
+ new Input(key1, 50),
+ new Input(key2, 50),
}));
}
public void testEmpty() throws Exception {
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
- suggester.build(new TermFreqPayloadArrayIterator(new TermFreqPayload[0]));
+ suggester.build(new InputArrayIterator(new Input[0]));
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}