You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/03/23 16:43:11 UTC
svn commit: r1580517 [2/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/document/
lucene/suggest/ lucene/suggest/src/java/org/apache/lucene/search/spell/
lucene/suggest/src/java/org/apache/lucene/s...
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1580517&r1=1580516&r2=1580517&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Sun Mar 23 15:43:10 2014
@@ -377,8 +377,8 @@ public class AnalyzingInfixSuggesterTest
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, newDirectory(), a, a, 3);
suggester.build(new InputArrayIterator(new Input[0]));
- suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
- suggester.add(new BytesRef("lend me your ear"), 8, new BytesRef("foobar"));
+ suggester.add(new BytesRef("a penny saved is a penny earned"), null, 10, new BytesRef("foobaz"));
+ suggester.add(new BytesRef("lend me your ear"), null, 8, new BytesRef("foobar"));
suggester.refresh();
List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
assertEquals(2, results.size());
@@ -415,7 +415,7 @@ public class AnalyzingInfixSuggesterTest
Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, newDirectory(), a, a, 3);
suggester.build(new InputArrayIterator(new Input[0]));
- suggester.add(new BytesRef("the pen is pretty"), 10, new BytesRef("foobaz"));
+ suggester.add(new BytesRef("the pen is pretty"), null, 10, new BytesRef("foobaz"));
suggester.refresh();
List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("pen p", random()), 10, true, true);
@@ -534,7 +534,7 @@ public class AnalyzingInfixSuggesterTest
if (VERBOSE) {
System.out.println("TEST: iter=" + iter + " update input=" + input.term.utf8ToString() + "/" + weight);
}
- suggester.update(input.term, weight, input.term);
+ suggester.update(input.term, null, weight, input.term);
} else {
// Add a new suggestion
@@ -543,7 +543,7 @@ public class AnalyzingInfixSuggesterTest
System.out.println("TEST: iter=" + iter + " add input=" + text + "/" + weight);
}
BytesRef br = new BytesRef(text);
- suggester.add(br, weight, br);
+ suggester.add(br, null, weight, br);
}
if (random().nextInt(15) == 7) {
@@ -748,7 +748,7 @@ public class AnalyzingInfixSuggesterTest
assertEquals(new BytesRef("foobar"), results.get(0).payload);
// Add a new suggestion:
- suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
+ suggester.add(new BytesRef("a penny saved is a penny earned"), null, 10, new BytesRef("foobaz"));
// Must refresh to see any newly added suggestions:
suggester.refresh();
@@ -782,7 +782,7 @@ public class AnalyzingInfixSuggesterTest
assertEquals(new BytesRef("foobaz"), results.get(0).payload);
// Change the weight:
- suggester.update(new BytesRef("lend me your ear"), 12, new BytesRef("foobox"));
+ suggester.update(new BytesRef("lend me your ear"), null, 12, new BytesRef("foobox"));
// Must refresh to see any newly added suggestions:
suggester.refresh();
@@ -797,4 +797,128 @@ public class AnalyzingInfixSuggesterTest
assertEquals(new BytesRef("foobaz"), results.get(1).payload);
suggester.close();
}
+
+ private Set<BytesRef> asSet(String... values) {
+ HashSet<BytesRef> result = new HashSet<>();
+ for(String value : values) {
+ result.add(new BytesRef(value));
+ }
+
+ return result;
+ }
+
+ // LUCENE-5528
+ public void testBasicContext() throws Exception {
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar"), asSet("foo", "bar")),
+ new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz"), asSet("foo", "baz"))
+ };
+
+ File tempDir = TestUtil.getTempDir("analyzingInfixContext");
+
+ for(int iter=0;iter<2;iter++) {
+ AnalyzingInfixSuggester suggester;
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ if (iter == 0) {
+ suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, newFSDirectory(tempDir), a, a, 3);
+ suggester.build(new InputArrayIterator(keys));
+ } else {
+ // Test again, after close/reopen:
+ suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, newFSDirectory(tempDir), a, a, 3);
+ }
+
+ // No context provided, all results returned
+ List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(2, results.size());
+ LookupResult result = results.get(0);
+ assertEquals("a penny saved is a penny <b>ear</b>ned", result.key);
+ assertEquals(10, result.value);
+ assertEquals(new BytesRef("foobaz"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("baz")));
+
+ result = results.get(1);
+ assertEquals("lend me your <b>ear</b>", result.key);
+ assertEquals(8, result.value);
+ assertEquals(new BytesRef("foobar"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("bar")));
+
+ // Both suggestions have "foo" context:
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), asSet("foo"), 10, true, true);
+ assertEquals(2, results.size());
+
+ result = results.get(0);
+ assertEquals("a penny saved is a penny <b>ear</b>ned", result.key);
+ assertEquals(10, result.value);
+ assertEquals(new BytesRef("foobaz"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("baz")));
+
+ result = results.get(1);
+ assertEquals("lend me your <b>ear</b>", result.key);
+ assertEquals(8, result.value);
+ assertEquals(new BytesRef("foobar"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("bar")));
+
+ // Only one has "bar" context:
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), asSet("bar"), 10, true, true);
+ assertEquals(1, results.size());
+
+ result = results.get(0);
+ assertEquals("lend me your <b>ear</b>", result.key);
+ assertEquals(8, result.value);
+ assertEquals(new BytesRef("foobar"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("bar")));
+
+ // Only one has "baz" context:
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), asSet("baz"), 10, true, true);
+ assertEquals(1, results.size());
+
+ result = results.get(0);
+ assertEquals("a penny saved is a penny <b>ear</b>ned", result.key);
+ assertEquals(10, result.value);
+ assertEquals(new BytesRef("foobaz"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("baz")));
+
+ // Both have foo or bar:
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), asSet("foo", "bar"), 10, true, true);
+ assertEquals(2, results.size());
+
+ result = results.get(0);
+ assertEquals("a penny saved is a penny <b>ear</b>ned", result.key);
+ assertEquals(10, result.value);
+ assertEquals(new BytesRef("foobaz"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("baz")));
+
+ result = results.get(1);
+ assertEquals("lend me your <b>ear</b>", result.key);
+ assertEquals(8, result.value);
+ assertEquals(new BytesRef("foobar"), result.payload);
+ assertNotNull(result.contexts);
+ assertEquals(2, result.contexts.size());
+ assertTrue(result.contexts.contains(new BytesRef("foo")));
+ assertTrue(result.contexts.contains(new BytesRef("bar")));
+
+ suggester.close();
+ }
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1580517&r1=1580516&r2=1580517&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Sun Mar 23 15:43:10 2014
@@ -1192,9 +1192,9 @@ public class AnalyzingSuggesterTest exte
}
}
- public final <T> Iterable<T> shuffle(T...values) {
- final List<T> asList = new ArrayList<>(values.length);
- for (T value : values) {
+ static final Iterable<Input> shuffle(Input...values) {
+ final List<Input> asList = new ArrayList<>(values.length);
+ for (Input value : values) {
asList.add(value);
}
Collections.shuffle(asList, random());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java?rev=1580517&r1=1580516&r2=1580517&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java Sun Mar 23 15:43:10 2014
@@ -136,12 +136,12 @@ public class BlendedInfixSuggesterTest e
// we don't find it for in the 2 first
- assertEquals(2, suggester.lookup("the", 2, true, false).size());
+ assertEquals(2, suggester.lookup("the", null, 2, true, false).size());
long w0 = getInResults(suggester, "the", ret, 2);
assertTrue(w0 < 0);
// but it's there if we search for 3 elements
- assertEquals(3, suggester.lookup("the", 3, true, false).size());
+ assertEquals(3, suggester.lookup("the", null, 3, true, false).size());
long w1 = getInResults(suggester, "the", ret, 3);
assertTrue(w1 > 0);
@@ -185,7 +185,7 @@ public class BlendedInfixSuggesterTest e
suggester.build(new InputArrayIterator(keys));
- List<Lookup.LookupResult> responses = suggester.lookup("the", 4, true, false);
+ List<Lookup.LookupResult> responses = suggester.lookup("the", null, 4, true, false);
for (Lookup.LookupResult response : responses) {
System.out.println(response);
@@ -196,7 +196,7 @@ public class BlendedInfixSuggesterTest e
private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) throws IOException {
- List<Lookup.LookupResult> responses = suggester.lookup(prefix, num, true, false);
+ List<Lookup.LookupResult> responses = suggester.lookup(prefix, null, num, true, false);
for (Lookup.LookupResult response : responses) {
if (response.payload.equals(payload)) {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java?rev=1580517&r1=1580516&r2=1580517&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java Sun Mar 23 15:43:10 2014
@@ -56,7 +56,7 @@ import org.junit.Ignore;
public class TestFreeTextSuggester extends LuceneTestCase {
public void testBasic() throws Exception {
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo bar baz blah", 50),
new Input("boo foo bar foo bee", 20)
);
@@ -105,7 +105,7 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringBuild() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo\u001ebar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
@@ -120,7 +120,7 @@ public class TestFreeTextSuggester exten
public void testIllegalByteDuringQuery() throws Exception {
// Default separator is INFORMATION SEPARATOR TWO
// (0x1e), so no input token is allowed to contain it
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo bar baz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random()));
@@ -181,6 +181,15 @@ public class TestFreeTextSuggester exten
return false;
}
+ @Override
+ public Set<BytesRef> contexts() {
+ return null;
+ }
+
+ @Override
+ public boolean hasContexts() {
+ return false;
+ }
});
if (VERBOSE) {
System.out.println(sug.sizeInBytes() + " bytes");
@@ -195,7 +204,7 @@ public class TestFreeTextSuggester exten
// Make sure you can suggest based only on unigram model:
public void testUnigrams() throws Exception {
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo bar baz blah boo foo bar foo bee", 50)
);
@@ -209,7 +218,7 @@ public class TestFreeTextSuggester exten
// Make sure the last token is not duplicated
public void testNoDupsAcrossGrams() throws Exception {
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
@@ -221,7 +230,7 @@ public class TestFreeTextSuggester exten
// Lookup of just empty string produces unicode only matches:
public void testEmptyString() throws Exception {
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("foo bar bar bar bar", 50)
);
Analyzer a = new MockAnalyzer(random());
@@ -248,7 +257,7 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("wizard of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
@@ -276,7 +285,7 @@ public class TestFreeTextSuggester exten
}
};
- Iterable<Input> keys = shuffle(
+ Iterable<Input> keys = AnalyzingSuggesterTest.shuffle(
new Input("wizard of of oz", 50)
);
FreeTextSuggester sug = new FreeTextSuggester(a, a, 3, (byte) 0x20);
@@ -377,6 +386,16 @@ public class TestFreeTextSuggester exten
public boolean hasPayloads() {
return false;
}
+
+ @Override
+ public Set<BytesRef> contexts() {
+ return null;
+ }
+
+ @Override
+ public boolean hasContexts() {
+ return false;
+ }
});
// Build inefficient but hopefully correct model:
@@ -597,14 +616,5 @@ public class TestFreeTextSuggester exten
}
return b.toString().trim();
}
-
- private final <T> Iterable<T> shuffle(T...values) {
- final List<T> asList = new ArrayList<>(values.length);
- for (T value : values) {
- asList.add(value);
- }
- Collections.shuffle(asList, random());
- return asList;
- }
}