You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/09/29 01:55:24 UTC
svn commit: r1391700 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/analysis/ lucene/suggest/
lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/
Author: rmuir
Date: Fri Sep 28 23:55:24 2012
New Revision: 1391700
URL: http://svn.apache.org/viewvc?rev=1391700&view=rev
Log:
LUCENE-4448: speedups for AnalyzingSuggester
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java?rev=1391700&r1=1391699&r2=1391700&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java Fri Sep 28 23:55:24 2012
@@ -83,6 +83,7 @@ public class TokenStreamToAutomaton {
* automaton where arcs are bytes from each term. */
public Automaton toAutomaton(TokenStream in) throws IOException {
final Automaton a = new Automaton();
+ boolean deterministic = true;
final TermToBytesRefAttribute termBytesAtt = in.addAttribute(TermToBytesRefAttribute.class);
final PositionIncrementAttribute posIncAtt = in.addAttribute(PositionIncrementAttribute.class);
@@ -132,6 +133,11 @@ public class TokenStreamToAutomaton {
}
}
positions.freeBefore(pos);
+ } else {
+ // note: this isn't necessarily true. its just that we aren't surely det.
+ // we could optimize this further (e.g. buffer and sort synonyms at a position)
+ // but thats probably overkill. this is cheap and dirty
+ deterministic = false;
}
final int endPos = pos + posLengthAtt.getPositionLength();
@@ -161,7 +167,7 @@ public class TokenStreamToAutomaton {
}
//toDot(a);
-
+ a.setDeterministic(deterministic);
return a;
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1391700&r1=1391699&r2=1391700&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Fri Sep 28 23:55:24 2012
@@ -45,6 +45,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicOperations;
import org.apache.lucene.util.automaton.SpecialOperations;
import org.apache.lucene.util.automaton.State;
import org.apache.lucene.util.automaton.Transition;
@@ -246,6 +247,7 @@ public class AnalyzingSuggester extends
// but because we are going in reverse topo sort
// it will not add any SEP/HOLE transitions:
state.addEpsilon(t.getDest());
+ a.setDeterministic(false);
t = null;
}
} else if (t.getMin() == TokenStreamToAutomaton.HOLE) {
@@ -263,6 +265,7 @@ public class AnalyzingSuggester extends
// but because we are going in reverse topo sort
// it will not add any SEP/HOLE transitions:
state.addEpsilon(t.getDest());
+ a.setDeterministic(false);
t = null;
}
if (t != null) {
@@ -504,7 +507,7 @@ public class AnalyzingSuggester extends
// TODO: we can optimize this somewhat by determinizing
// while we convert
- automaton = Automaton.minimize(automaton);
+ BasicOperations.determinize(automaton);
final CharsRef spare = new CharsRef();