svn commit: r1533857 - /lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java

[us...@apache.org]

Author: uschindler
Date: Sun Oct 20 08:20:03 2013
New Revision: 1533857

URL: http://svn.apache.org/r1533857
Log:
LUCENE-4956: Partially fix posIncrAtt to preserve increment of first token. The morphQueue still has a bug, added nocommit!

Modified:
    lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java

Modified: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java?rev=1533857&r1=1533856&r2=1533857&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/ko/KoreanFilter.java Sun Oct 20 08:20:03 2013
@@ -97,7 +97,7 @@ public final class KoreanFilter extends 
   public boolean incrementToken() throws IOException {
     if (!morphQueue.isEmpty()) {
       restoreState(currentState);
-      setAttributesFromQueue();
+      setAttributesFromQueue(false);
       return true;
     }
 
@@ -115,7 +115,7 @@ public final class KoreanFilter extends 
   
       if (!morphQueue.isEmpty()) {
         // no need to restore state!
-        setAttributesFromQueue();
+        setAttributesFromQueue(true);
         return true;
       }
     }
@@ -124,14 +124,27 @@ public final class KoreanFilter extends 
   }
   
 
-  private void setAttributesFromQueue() {
+  private void setAttributesFromQueue(boolean isFirst) {
     final IndexWord iw = morphQueue.removeFirst();
     final String word = iw.getWord();
     final int ofs = iw.getOffset();
     
     termAtt.setEmpty().append(word);
     offsetAtt.setOffset(ofs, ofs + word.length());
-    posIncrAtt.setPositionIncrement(iw.getIncrement());      
+    if (isFirst) {
+      // on the first token preserve the position increment as given by the previous filter:
+      int posIncr = iw.getIncrement();
+      // nocommit: assert posIncr > 0 : "the first token in the morphQueue cannot have 0";
+      // this is just a workaround for the assertion failure above:
+      posIncr = Math.min(1, posIncr);
+      // -- end fix
+      posIncr += posIncrAtt.getPositionIncrement() - 1;
+      posIncrAtt.setPositionIncrement(posIncr);
+    } else {
+      posIncrAtt.setPositionIncrement(iw.getIncrement());
+    }
+    
+    // TODO: How to handle PositionLengthAttribute correctly?
   }
   
   /**