You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/09/29 01:55:24 UTC

svn commit: r1391700 - in /lucene/dev/branches/branch_4x: ./ lucene/ lucene/core/ lucene/core/src/java/org/apache/lucene/analysis/ lucene/suggest/ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/

Author: rmuir
Date: Fri Sep 28 23:55:24 2012
New Revision: 1391700

URL: http://svn.apache.org/viewvc?rev=1391700&view=rev
Log:
LUCENE-4448: speedups for AnalyzingSuggester

Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/   (props changed)
    lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
    lucene/dev/branches/branch_4x/lucene/suggest/   (props changed)
    lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java

Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java?rev=1391700&r1=1391699&r2=1391700&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java Fri Sep 28 23:55:24 2012
@@ -83,6 +83,7 @@ public class TokenStreamToAutomaton {
    *  automaton where arcs are bytes from each term. */
   public Automaton toAutomaton(TokenStream in) throws IOException {
     final Automaton a = new Automaton();
+    boolean deterministic = true;
 
     final TermToBytesRefAttribute termBytesAtt = in.addAttribute(TermToBytesRefAttribute.class);
     final PositionIncrementAttribute posIncAtt = in.addAttribute(PositionIncrementAttribute.class);
@@ -132,6 +133,11 @@ public class TokenStreamToAutomaton {
           }
         }
         positions.freeBefore(pos);
+      } else {
+        // note: this isn't necessarily true. its just that we aren't surely det.
+        // we could optimize this further (e.g. buffer and sort synonyms at a position)
+        // but thats probably overkill. this is cheap and dirty
+        deterministic = false;
       }
 
       final int endPos = pos + posLengthAtt.getPositionLength();
@@ -161,7 +167,7 @@ public class TokenStreamToAutomaton {
     }
 
     //toDot(a);
-
+    a.setDeterministic(deterministic);
     return a;
   }
 

Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1391700&r1=1391699&r2=1391700&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Fri Sep 28 23:55:24 2012
@@ -45,6 +45,7 @@ import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicOperations;
 import org.apache.lucene.util.automaton.SpecialOperations;
 import org.apache.lucene.util.automaton.State;
 import org.apache.lucene.util.automaton.Transition;
@@ -246,6 +247,7 @@ public class AnalyzingSuggester extends 
             // but because we are going in reverse topo sort
             // it will not add any SEP/HOLE transitions:
             state.addEpsilon(t.getDest());
+            a.setDeterministic(false);
             t = null;
           }
         } else if (t.getMin() == TokenStreamToAutomaton.HOLE) {
@@ -263,6 +265,7 @@ public class AnalyzingSuggester extends 
           // but because we are going in reverse topo sort
           // it will not add any SEP/HOLE transitions:
           state.addEpsilon(t.getDest());
+          a.setDeterministic(false);
           t = null;
         }
         if (t != null) {
@@ -504,7 +507,7 @@ public class AnalyzingSuggester extends 
 
       // TODO: we can optimize this somewhat by determinizing
       // while we convert
-      automaton = Automaton.minimize(automaton);
+      BasicOperations.determinize(automaton);
 
       final CharsRef spare = new CharsRef();