You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/06/18 09:21:17 UTC
[5/6] lucene-solr:branch_7_4: LUCENE-8360: NGram filters should set
posIncAtt in end()
LUCENE-8360: NGram filters should set posIncAtt in end()
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b03b527a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b03b527a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b03b527a
Branch: refs/heads/branch_7_4
Commit: b03b527a31206665a4d814f03d4299c416bbec31
Parents: 3aa6086
Author: Alan Woodward <ro...@apache.org>
Authored: Sun Jun 17 21:19:14 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Jun 18 09:58:39 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../analysis/ngram/EdgeNGramTokenFilter.java | 6 ++++++
.../lucene/analysis/ngram/NGramTokenFilter.java | 6 ++++++
.../ngram/EdgeNGramTokenFilterTest.java | 10 ++++++++++
.../analysis/ngram/NGramTokenFilterTest.java | 21 +++++++++++++++-----
5 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b03b527a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7182bb1..28bb128 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -179,6 +179,9 @@ Bug Fixes
* LUCENE-8357: FunctionScoreQuery.boostByQuery() and boostByValue() were
producing truncated Explanations (Markus Jelsma, Alan Woodward)
+* LUCENE-8360: NGramTokenFilter and EdgeNGramTokenFilter did not correctly
+ set position increments in end() (Alan Woodward)
+
Other
* LUCENE-8301: Update randomizedtesting to 2.6.0. (Dawid Weiss)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b03b527a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
index 154f075..6ad6343 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
@@ -175,4 +175,10 @@ public final class EdgeNGramTokenFilter extends TokenFilter {
curTermBuffer = null;
curPosIncr = 0;
}
+
+ @Override
+ public void end() throws IOException {
+ super.end();
+ posIncrAtt.setPositionIncrement(curPosIncr);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b03b527a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
index 8e1a7e4..cf0a6f2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
@@ -203,4 +203,10 @@ public final class NGramTokenFilter extends TokenFilter {
curTermBuffer = null;
curPosIncr = 0;
}
+
+ @Override
+ public void end() throws IOException {
+ super.end();
+ posIncrAtt.setPositionIncrement(curPosIncr);
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b03b527a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
index fd1949a..8fcf7a1 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
@@ -269,4 +269,14 @@ public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase {
tk.close();
}
}
+
+ public void testEndPositionIncrement() throws IOException {
+ TokenStream source = whitespaceMockTokenizer("seventeen one two three four");
+ TokenStream input = new EdgeNGramTokenFilter(source, 8, 8, false);
+ PositionIncrementAttribute posIncAtt = input.addAttribute(PositionIncrementAttribute.class);
+ input.reset();
+ while (input.incrementToken()) {}
+ input.end();
+ assertEquals(4, posIncAtt.getPositionIncrement());
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b03b527a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
index 2a47396..c3ef231 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
@@ -17,23 +17,24 @@
package org.apache.lucene.analysis.ngram;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Random;
+
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.TestUtil;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.Random;
-
/**
* Tests {@link NGramTokenFilter} for correctness.
*/
@@ -169,6 +170,16 @@ public class NGramTokenFilterTest extends BaseTokenStreamTestCase {
new int[] { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 });
analyzer.close();
}
+
+ public void testEndPositionIncrement() throws IOException {
+ TokenStream source = whitespaceMockTokenizer("seventeen one two three four");
+ TokenStream input = new NGramTokenFilter(source, 8, 8, false);
+ PositionIncrementAttribute posIncAtt = input.addAttribute(PositionIncrementAttribute.class);
+ input.reset();
+ while (input.incrementToken()) {}
+ input.end();
+ assertEquals(4, posIncAtt.getPositionIncrement());
+ }
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {