You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/07/19 08:41:14 UTC
[13/50] [abbrv] lucene-solr:jira/solr-12509: LUCENE-8395:
WordDelimiterGraphFilter can incorrectly add holes
LUCENE-8395: WordDelimiterGraphFilter can incorrectly add holes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8997d413
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8997d413
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8997d413
Branch: refs/heads/jira/solr-12509
Commit: 8997d41357ab95eaa88702b93c75de4b1813457b
Parents: b68829e
Author: Alan Woodward <ro...@apache.org>
Authored: Thu Jul 12 13:12:23 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Jul 12 13:13:04 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 ++++
.../miscellaneous/WordDelimiterGraphFilter.java | 1 +
.../miscellaneous/TestWordDelimiterGraphFilter.java | 15 ++++++++++++++-
3 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8997d413/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 42a5405..01b32f6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -154,6 +154,10 @@ Bug Fixes:
if the last token in the stream was subsequently dropped; FixedShingleFilter did
not set position increment in end() (Alan Woodward)
+* LUCENE-8395: WordDelimiterGraphFilter would incorrectly insert a hole into a
+ TokenStream if a token consisting entirely of delimiter characters was
+ encountered, but preserve_original was set. (Alan Woodward)
+
Changes in Runtime Behavior:
* LUCENE-7976: TieredMergePolicy now respects maxSegmentSizeMB by default when executing
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8997d413/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
index 7d021c5..a438213 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterGraphFilter.java
@@ -368,6 +368,7 @@ public final class WordDelimiterGraphFilter extends TokenFilter {
if (has(PRESERVE_ORIGINAL) == false) {
continue;
} else {
+ accumPosInc = 0;
return true;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8997d413/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
index b5dc8a3..65d3b02 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterGraphFilter.java
@@ -213,6 +213,19 @@ public class TestWordDelimiterGraphFilter extends BaseTokenStreamTestCase {
}
public void testPositionIncrements() throws Exception {
+
+ Analyzer a4 = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ final int flags = SPLIT_ON_NUMERICS | GENERATE_WORD_PARTS | PRESERVE_ORIGINAL | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE;
+ return new TokenStreamComponents(tokenizer, new WordDelimiterGraphFilter(tokenizer, flags, CharArraySet.EMPTY_SET));
+ }
+ };
+ assertAnalyzesTo(a4, "SAL_S8371 - SAL",
+ new String[]{ "SAL_S8371", "SAL", "S", "8371", "-", "SAL"},
+ new int[]{ 1, 0, 1, 1, 1, 1});
+
final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
final CharArraySet protWords = new CharArraySet(new HashSet<>(Arrays.asList("NUTCH")), false);
@@ -327,7 +340,7 @@ public class TestWordDelimiterGraphFilter extends BaseTokenStreamTestCase {
null,
false);
- IOUtils.close(a, a2, a3);
+ IOUtils.close(a, a2, a3, a4);
}
public void testKeywordFilter() throws Exception {