You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/09/08 16:59:15 UTC
svn commit: r1166728 - in /lucene/dev/trunk: lucene/contrib/
modules/analysis/common/src/java/org/apache/lucene/analysis/compound/
modules/analysis/common/src/test/org/apache/lucene/analysis/compound/
Author: rmuir
Date: Thu Sep 8 14:59:15 2011
New Revision: 1166728
URL: http://svn.apache.org/viewvc?rev=1166728&view=rev
Log:
LUCENE-3417: DictionaryCompoundWordFilter did not properly add tokens from the end compound word
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1166728&r1=1166727&r2=1166728&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Thu Sep 8 14:59:15 2011
@@ -83,6 +83,11 @@ New Features
SimpleBoundaryScanner and BreakIteratorBoundaryScanner, so that FVH's FragmentsBuilder
can find "natural" boundary to make snippets. (Robert Muir, Koji Sekiguchi)
+Bug Fixes
+
+ * LUCENE-3417: DictionaryCompoundWordFilter did not properly add tokens from the
+ end compound word. (Njal Karevoll via Robert Muir)
+
======================= Lucene 3.4.0 ================
New Features
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java?rev=1166728&r1=1166727&r2=1166728&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java Thu Sep 8 14:59:15 2011
@@ -136,9 +136,9 @@ public class DictionaryCompoundWordToken
char[] lowerCaseTermBuffer=makeLowerCaseCopy(token.buffer());
- for (int i=0;i<token.length()-this.minSubwordSize;++i) {
+ for (int i=0;i<=token.length()-this.minSubwordSize;++i) {
Token longestMatchToken=null;
- for (int j=this.minSubwordSize-1;j<this.maxSubwordSize;++j) {
+ for (int j=this.minSubwordSize;j<=this.maxSubwordSize;++j) {
if(i+j>token.length()) {
break;
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=1166728&r1=1166727&r2=1166728&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Thu Sep 8 14:59:15 2011
@@ -161,6 +161,49 @@ public class TestCompoundWordTokenFilter
14, 20 }, new int[] { 26, 3, 14, 14, 20, 26 }, new int[] { 1, 0, 0, 0,
0, 0 });
}
+
+ public void testTokenEndingWithWordComponentOfMinimumLength() throws Exception {
+ String[] dict = {"ab", "cd", "ef"};
+
+ DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT,
+ new StringReader(
+ "abcdef")
+ ),
+ dict,
+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false);
+
+ assertTokenStreamContents(tf,
+ new String[] { "abcdef", "ab", "cd", "ef" },
+ new int[] { 0, 0, 2, 4},
+ new int[] { 6, 2, 4, 6},
+ new int[] { 1, 0, 0, 0}
+ );
+ }
+
+ public void testWordComponentWithLessThanMinimumLength() throws Exception {
+ String[] dict = {"abc", "d", "efg"};
+
+ DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT,
+ new StringReader(
+ "abcdefg")
+ ),
+ dict,
+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false);
+
+ // since "d" is shorter than the minimum subword size, it should not be added to the token stream
+ assertTokenStreamContents(tf,
+ new String[] { "abcdefg", "abc", "efg" },
+ new int[] { 0, 0, 4},
+ new int[] { 7, 3, 7},
+ new int[] { 1, 0, 0}
+ );
+ }
public void testReset() throws Exception {
String[] dict = { "Rind", "Fleisch", "Draht", "Schere", "Gesetz",