You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2013/07/19 00:29:56 UTC
svn commit: r1504697 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/suggest/
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/
lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/
Author: simonw
Date: Thu Jul 18 22:29:56 2013
New Revision: 1504697
URL: http://svn.apache.org/r1504697
Log:
LUCENE-5120: AnalyzingSuggester might modify it's FST's cached root arc if payloads are used
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1504697&r1=1504696&r2=1504697&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Thu Jul 18 22:29:56 2013
@@ -194,6 +194,10 @@ Bug Fixes
SortedSetDocValuesReaderState and SortedSetDocValuesAccumulator.
(Robert Muir, Mike McCandless)
+* LUCENE-5120: AnalyzingSuggester modifed it's FST's cached root arc if payloads
+ are used and the entire output resided on the root arc on the first access. This
+ caused subsequent suggest calls to fail. (Simon Willnauer)
+
Optimizations
* LUCENE-4936: Improve numeric doc values compression in case all values share
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1504697&r1=1504696&r2=1504697&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Thu Jul 18 22:29:56 2013
@@ -624,9 +624,8 @@ public class AnalyzingSuggester extends
}
assert sepIndex != -1;
spare.grow(sepIndex);
- int payloadLen = output2.length - sepIndex - 1;
- output2.length = sepIndex;
- UnicodeUtil.UTF8toUTF16(output2, spare);
+ final int payloadLen = output2.length - sepIndex - 1;
+ UnicodeUtil.UTF8toUTF16(output2.bytes, output2.offset, sepIndex, spare);
BytesRef payload = new BytesRef(payloadLen);
System.arraycopy(output2.bytes, sepIndex+1, payload.bytes, 0, payloadLen);
payload.length = payloadLen;
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1504697&r1=1504696&r2=1504697&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Thu Jul 18 22:29:56 2013
@@ -112,44 +112,45 @@ public class AnalyzingSuggesterTest exte
AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
suggester.build(new TermFreqPayloadArrayIterator(keys));
-
- // top N of 2, but only foo is available
- List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
- assertEquals(1, results.size());
- assertEquals("foo", results.get(0).key.toString());
- assertEquals(50, results.get(0).value, 0.01F);
- assertEquals(new BytesRef("hello"), results.get(0).payload);
-
- // top N of 1 for 'bar': we return this even though
- // barbar is higher because exactFirst is enabled:
- results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random()), false, 1);
- assertEquals(1, results.size());
- assertEquals("bar", results.get(0).key.toString());
- assertEquals(10, results.get(0).value, 0.01F);
- assertEquals(new BytesRef("goodbye"), results.get(0).payload);
-
- // top N Of 2 for 'b'
- results = suggester.lookup(_TestUtil.stringToCharSequence("b", random()), false, 2);
- assertEquals(2, results.size());
- assertEquals("barbar", results.get(0).key.toString());
- assertEquals(12, results.get(0).value, 0.01F);
- assertEquals(new BytesRef("thank you"), results.get(0).payload);
- assertEquals("bar", results.get(1).key.toString());
- assertEquals(10, results.get(1).value, 0.01F);
- assertEquals(new BytesRef("goodbye"), results.get(1).payload);
-
- // top N of 3 for 'ba'
- results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random()), false, 3);
- assertEquals(3, results.size());
- assertEquals("barbar", results.get(0).key.toString());
- assertEquals(12, results.get(0).value, 0.01F);
- assertEquals(new BytesRef("thank you"), results.get(0).payload);
- assertEquals("bar", results.get(1).key.toString());
- assertEquals(10, results.get(1).value, 0.01F);
- assertEquals(new BytesRef("goodbye"), results.get(1).payload);
- assertEquals("barbara", results.get(2).key.toString());
- assertEquals(6, results.get(2).value, 0.01F);
- assertEquals(new BytesRef("for all the fish"), results.get(2).payload);
+ for (int i = 0; i < 2; i++) {
+ // top N of 2, but only foo is available
+ List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
+ assertEquals(1, results.size());
+ assertEquals("foo", results.get(0).key.toString());
+ assertEquals(50, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("hello"), results.get(0).payload);
+
+ // top N of 1 for 'bar': we return this even though
+ // barbar is higher because exactFirst is enabled:
+ results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random()), false, 1);
+ assertEquals(1, results.size());
+ assertEquals("bar", results.get(0).key.toString());
+ assertEquals(10, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(0).payload);
+
+ // top N Of 2 for 'b'
+ results = suggester.lookup(_TestUtil.stringToCharSequence("b", random()), false, 2);
+ assertEquals(2, results.size());
+ assertEquals("barbar", results.get(0).key.toString());
+ assertEquals(12, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("thank you"), results.get(0).payload);
+ assertEquals("bar", results.get(1).key.toString());
+ assertEquals(10, results.get(1).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(1).payload);
+
+ // top N of 3 for 'ba'
+ results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random()), false, 3);
+ assertEquals(3, results.size());
+ assertEquals("barbar", results.get(0).key.toString());
+ assertEquals(12, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("thank you"), results.get(0).payload);
+ assertEquals("bar", results.get(1).key.toString());
+ assertEquals(10, results.get(1).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(1).payload);
+ assertEquals("barbara", results.get(2).key.toString());
+ assertEquals(6, results.get(2).value, 0.01F);
+ assertEquals(new BytesRef("for all the fish"), results.get(2).payload);
+ }
}
// TODO: more tests