You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/10/11 18:07:28 UTC
svn commit: r1397135 - in /lucene/dev/trunk/lucene/suggest/src:
java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
Author: mikemccand
Date: Thu Oct 11 16:07:27 2012
New Revision: 1397135
URL: http://svn.apache.org/viewvc?rev=1397135&view=rev
Log:
AnalyzingSuggester doesn't have to escape 0xff byte unless preserveSep is true
Modified:
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1397135&r1=1397134&r2=1397135&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Thu Oct 11 16:07:27 2012
@@ -273,7 +273,7 @@ public class AnalyzingSuggester extends
}
}
- /** Just escapes the bytes we steal (0xff, 0x0). */
+ /** Just escapes the 0xff byte (which we still for SEP). */
private static final class EscapingTokenStreamToAutomaton extends TokenStreamToAutomaton {
final BytesRef spare = new BytesRef();
@@ -301,6 +301,16 @@ public class AnalyzingSuggester extends
return spare;
}
}
+
+ private TokenStreamToAutomaton getTokenStreamToAutomaton() {
+ if (preserveSep) {
+ return new EscapingTokenStreamToAutomaton();
+ } else {
+ // When we're not preserving sep, we don't steal 0xff
+ // byte, so we don't need to do any escaping:
+ return new TokenStreamToAutomaton();
+ }
+ }
@Override
public void build(TermFreqIterator iterator) throws IOException {
@@ -313,8 +323,7 @@ public class AnalyzingSuggester extends
Sort.ByteSequencesReader reader = null;
BytesRef scratch = new BytesRef();
- TokenStreamToAutomaton ts2a = new EscapingTokenStreamToAutomaton();
-
+ TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();
// analyzed sequence + 0(byte) + weight(int) + surface + analyzedLength(short)
boolean success = false;
byte buffer[] = new byte[8];
@@ -489,7 +498,7 @@ public class AnalyzingSuggester extends
// TODO: is there a Reader from a CharSequence?
// Turn tokenstream into automaton:
TokenStream ts = queryAnalyzer.tokenStream("", new StringReader(key.toString()));
- Automaton automaton = (new EscapingTokenStreamToAutomaton()).toAutomaton(ts);
+ Automaton automaton = getTokenStreamToAutomaton().toAutomaton(ts);
ts.end();
ts.close();
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1397135&r1=1397134&r2=1397135&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Thu Oct 11 16:07:27 2012
@@ -706,63 +706,67 @@ public class AnalyzingSuggesterTest exte
}
public void testStolenBytes() throws Exception {
-
- final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
-
- // TokenStream stream = new SynonymFilter(tokenizer, map, true);
- // return new TokenStreamComponents(tokenizer, new RemoveDuplicatesTokenFilter(stream));
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),
- }),
- new CannedTokenStream(new Token[] {
- token("a",1,1),
- token("a",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("a",1,1),
- token("a",1,1)
- }),
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),
- })
- };
+ // First time w/ preserveSep, second time without:
+ for(int i=0;i<2;i++) {
+
+ final Analyzer analyzer = new Analyzer() {
@Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+
+ // TokenStream stream = new SynonymFilter(tokenizer, map, true);
+ // return new TokenStreamComponents(tokenizer, new RemoveDuplicatesTokenFilter(stream));
+ return new TokenStreamComponents(tokenizer) {
+ int tokenStreamCounter = 0;
+ final TokenStream[] tokenStreams = new TokenStream[] {
+ new CannedBinaryTokenStream(new BinaryToken[] {
+ token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),
+ }),
+ new CannedTokenStream(new Token[] {
+ token("a",1,1),
+ token("a",1,1)
+ }),
+ new CannedTokenStream(new Token[] {
+ token("a",1,1),
+ token("a",1,1)
+ }),
+ new CannedBinaryTokenStream(new BinaryToken[] {
+ token(new BytesRef(new byte[] {0x61, (byte) 0xff, 0x61})),
+ })
+ };
+
+ @Override
+ public TokenStream getTokenStream() {
+ TokenStream result = tokenStreams[tokenStreamCounter];
+ tokenStreamCounter++;
+ return result;
+ }
- @Override
- protected void setReader(final Reader reader) throws IOException {
+ @Override
+ protected void setReader(final Reader reader) throws IOException {
+ }
+ };
}
};
- }
- };
-
- TermFreq keys[] = new TermFreq[] {
- new TermFreq("a a", 50),
- new TermFreq("a b", 50),
- };
- AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer);
- suggester.build(new TermFreqArrayIterator(keys));
- List<LookupResult> results = suggester.lookup("a a", false, 5);
- assertEquals(1, results.size());
- assertEquals("a b", results.get(0).key);
- assertEquals(50, results.get(0).value);
-
- results = suggester.lookup("a a", false, 5);
- assertEquals(1, results.size());
- assertEquals("a a", results.get(0).key);
- assertEquals(50, results.get(0).value);
+ TermFreq keys[] = new TermFreq[] {
+ new TermFreq("a a", 50),
+ new TermFreq("a b", 50),
+ };
+
+ AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, analyzer, AnalyzingSuggester.EXACT_FIRST | (i==0 ? AnalyzingSuggester.PRESERVE_SEP : 0), 256, -1);
+ suggester.build(new TermFreqArrayIterator(keys));
+ List<LookupResult> results = suggester.lookup("a a", false, 5);
+ assertEquals(1, results.size());
+ assertEquals("a b", results.get(0).key);
+ assertEquals(50, results.get(0).value);
+
+ results = suggester.lookup("a a", false, 5);
+ assertEquals(1, results.size());
+ assertEquals("a a", results.get(0).key);
+ assertEquals(50, results.get(0).value);
+ }
}
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {