You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/09/08 17:52:10 UTC
svn commit: r1166766 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ solr/ solr/core/src/java/org/apache/solr/analysis/
solr/core/src/test/org/apache/solr/analysis/
Author: rmuir
Date: Thu Sep 8 15:52:10 2011
New Revision: 1166766
URL: http://svn.apache.org/viewvc?rev=1166766&view=rev
Log:
LUCENE-3410: Deprecated multi-int constructors in WordDelimiterFilter. Now uses int bitfield
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/CHANGES.txt
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilter.java
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java
lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterIterator.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java
Modified: lucene/dev/branches/branch_3x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/CHANGES.txt?rev=1166766&r1=1166765&r2=1166766&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/CHANGES.txt Thu Sep 8 15:52:10 2011
@@ -23,7 +23,12 @@ on how to get started.
================== 3.5.0 ==================
-(No Changes)
+ Other Changes
+----------------------
+
+* LUCENE-3410: Deprecated the WordDelimiterFilter constructors accepting multiple
+ ints masquerading as booleans. Preferred constructor now accepts a single int
+ bitfield (Chris Male)
================== 3.4.0 ==================
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilter.java?rev=1166766&r1=1166765&r2=1166766&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilter.java Thu Sep 8 15:52:10 2011
@@ -73,52 +73,73 @@ final class WordDelimiterFilter extends
public static final int ALPHANUM = 0x07;
/**
- * If true, causes parts of words to be generated:
+ * Causes parts of words to be generated:
* <p/>
* "PowerShot" => "Power" "Shot"
*/
- final boolean generateWordParts;
+ public static final int GENERATE_WORD_PARTS = 1;
/**
- * If true, causes number subwords to be generated:
+ * Causes number subwords to be generated:
* <p/>
* "500-42" => "500" "42"
*/
- final boolean generateNumberParts;
+ public static final int GENERATE_NUMBER_PARTS = 2;
/**
- * If true, causes maximum runs of word parts to be catenated:
+ * Causes maximum runs of word parts to be catenated:
* <p/>
* "wi-fi" => "wifi"
*/
- final boolean catenateWords;
+ public static final int CATENATE_WORDS = 4;
/**
- * If true, causes maximum runs of number parts to be catenated:
+ * Causes maximum runs of word parts to be catenated:
* <p/>
- * "500-42" => "50042"
+ * "wi-fi" => "wifi"
*/
- final boolean catenateNumbers;
+ public static final int CATENATE_NUMBERS = 8;
/**
- * If true, causes all subword parts to be catenated:
+ * Causes all subword parts to be catenated:
* <p/>
* "wi-fi-4000" => "wifi4000"
*/
- final boolean catenateAll;
+ public static final int CATENATE_ALL = 16;
/**
- * If true, original words are preserved and added to the subword list (Defaults to false)
+ * Causes original words are preserved and added to the subword list (Defaults to false)
* <p/>
* "500-42" => "500" "42" "500-42"
*/
- final boolean preserveOriginal;
+ public static final int PRESERVE_ORIGINAL = 32;
+
+ /**
+ * If not set, causes case changes to be ignored (subwords will only be generated
+ * given SUBWORD_DELIM tokens)
+ */
+ public static final int SPLIT_ON_CASE_CHANGE = 64;
+
+ /**
+ * If not set, causes numeric changes to be ignored (subwords will only be generated
+ * given SUBWORD_DELIM tokens).
+ */
+ public static final int SPLIT_ON_NUMERICS = 128;
+
+ /**
+ * Causes trailing "'s" to be removed for each subword
+ * <p/>
+ * "O'Neil's" => "O", "Neil"
+ */
+ public static final int STEM_ENGLISH_POSSESSIVE = 256;
/**
* If not null is the set of tokens to protect from being delimited
*
*/
final CharArraySet protWords;
+
+ private final int flags;
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
@@ -155,8 +176,36 @@ final class WordDelimiterFilter extends
private boolean hasOutputFollowingOriginal = false;
/**
+ * Creates a new WordDelimiterFilter
+ *
+ * @param in TokenStream to be filtered
+ * @param charTypeTable table containing character types
+ * @param configurationFlags Flags configuring the filter
+ * @param protWords If not null is the set of tokens to protect from being delimited
+ */
+ public WordDelimiterFilter(TokenStream in, byte[] charTypeTable, int configurationFlags, CharArraySet protWords) {
+ super(in);
+ this.flags = configurationFlags;
+ this.protWords = protWords;
+ this.iterator = new WordDelimiterIterator(
+ charTypeTable, has(SPLIT_ON_CASE_CHANGE), has(SPLIT_ON_NUMERICS), has(STEM_ENGLISH_POSSESSIVE));
+ }
+
+ /**
+ * Creates a new WordDelimiterFilter using {@link WordDelimiterIterator#DEFAULT_WORD_DELIM_TABLE}
+ * as its charTypeTable
+ *
+ * @param in TokenStream to be filtered
+ * @param configurationFlags Flags configuring the filter
+ * @param protWords If not null is the set of tokens to protect from being delimited
+ */
+ public WordDelimiterFilter(TokenStream in, int configurationFlags, CharArraySet protWords) {
+ this(in, WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, configurationFlags, protWords);
+ }
+
+ /**
* @param in Token stream to be filtered.
- * @param charTypeTable
+ * @param charTypeTable table containing character types
* @param generateWordParts If 1, causes parts of words to be generated: "PowerShot" => "Power" "Shot"
* @param generateNumberParts If 1, causes number subwords to be generated: "500-42" => "500" "42"
* @param catenateWords 1, causes maximum runs of word parts to be catenated: "wi-fi" => "wifi"
@@ -167,7 +216,9 @@ final class WordDelimiterFilter extends
* @param splitOnNumerics 1, causes "j2se" to be three tokens; "j" "2" "se"
* @param stemEnglishPossessive If 1, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil"
* @param protWords If not null is the set of tokens to protect from being delimited
+ * @deprecated Use {@link #WordDelimiterFilter(TokenStream, byte[], int, CharArraySet)}
*/
+ @Deprecated
public WordDelimiterFilter(TokenStream in,
byte[] charTypeTable,
int generateWordParts,
@@ -181,14 +232,38 @@ final class WordDelimiterFilter extends
int stemEnglishPossessive,
CharArraySet protWords) {
super(in);
- this.generateWordParts = generateWordParts != 0;
- this.generateNumberParts = generateNumberParts != 0;
- this.catenateWords = catenateWords != 0;
- this.catenateNumbers = catenateNumbers != 0;
- this.catenateAll = catenateAll != 0;
- this.preserveOriginal = preserveOriginal != 0;
+
+ int flags = 0;
+ if (generateWordParts != 0) {
+ flags |= GENERATE_WORD_PARTS;
+ }
+ if (generateNumberParts != 0) {
+ flags |= GENERATE_NUMBER_PARTS;
+ }
+ if (catenateWords != 0) {
+ flags |= CATENATE_WORDS;
+ }
+ if (catenateNumbers != 0) {
+ flags |= CATENATE_NUMBERS;
+ }
+ if (catenateAll != 0) {
+ flags |= CATENATE_ALL;
+ }
+ if (preserveOriginal != 0) {
+ flags |= PRESERVE_ORIGINAL;
+ }
+ if (splitOnCaseChange != 0) {
+ flags |= SPLIT_ON_CASE_CHANGE;
+ }
+ if (splitOnNumerics != 0) {
+ flags |= SPLIT_ON_NUMERICS;
+ }
+ if (stemEnglishPossessive != 0) {
+ flags |= STEM_ENGLISH_POSSESSIVE;
+ }
this.protWords = protWords;
this.iterator = new WordDelimiterIterator(charTypeTable, splitOnCaseChange != 0, splitOnNumerics != 0, stemEnglishPossessive != 0);
+ this.flags = flags;
}
/**
@@ -245,7 +320,9 @@ final class WordDelimiterFilter extends
* @param splitOnNumerics 1, causes "j2se" to be three tokens; "j" "2" "se"
* @param stemEnglishPossessive If 1, causes trailing "'s" to be removed for each subword: "O'Neil's" => "O", "Neil"
* @param protWords If not null is the set of tokens to protect from being delimited
+ * @deprecated Use {@link #WordDelimiterFilter(TokenStream, int, CharArraySet)}
*/
+ @Deprecated
public WordDelimiterFilter(TokenStream in,
int generateWordParts,
int generateNumberParts,
@@ -356,7 +433,7 @@ final class WordDelimiterFilter extends
}
// word of simply delimiters
- if (iterator.end == WordDelimiterIterator.DONE && !preserveOriginal) {
+ if (iterator.end == WordDelimiterIterator.DONE && !has(PRESERVE_ORIGINAL)) {
// if the posInc is 1, simply ignore it in the accumulation
if (posIncAttribute.getPositionIncrement() == 1) {
accumPosInc--;
@@ -367,10 +444,10 @@ final class WordDelimiterFilter extends
saveState();
hasOutputToken = false;
- hasOutputFollowingOriginal = !preserveOriginal;
+ hasOutputFollowingOriginal = !has(PRESERVE_ORIGINAL);
lastConcatCount = 0;
- if (preserveOriginal) {
+ if (has(PRESERVE_ORIGINAL)) {
posIncAttribute.setPositionIncrement(accumPosInc);
accumPosInc = 0;
return true;
@@ -426,7 +503,7 @@ final class WordDelimiterFilter extends
}
// add all subwords (catenateAll)
- if (catenateAll) {
+ if (has(CATENATE_ALL)) {
concatenate(concatAll);
}
@@ -499,7 +576,7 @@ final class WordDelimiterFilter extends
* @return {@code true} if concatenation should occur, {@code false} otherwise
*/
private boolean shouldConcatenate(int wordType) {
- return (catenateWords && isAlpha(wordType)) || (catenateNumbers && isDigit(wordType));
+ return (has(CATENATE_WORDS) && isAlpha(wordType)) || (has(CATENATE_NUMBERS) && isDigit(wordType));
}
/**
@@ -509,7 +586,7 @@ final class WordDelimiterFilter extends
* @return {@code true} if a word/number part should be generated, {@code false} otherwise
*/
private boolean shouldGenerateParts(int wordType) {
- return (generateWordParts && isAlpha(wordType)) || (generateNumberParts && isDigit(wordType));
+ return (has(GENERATE_WORD_PARTS) && isAlpha(wordType)) || (has(GENERATE_NUMBER_PARTS) && isDigit(wordType));
}
/**
@@ -608,6 +685,16 @@ final class WordDelimiterFilter extends
return (type & UPPER) != 0;
}
+ /**
+ * Determines whether the given flag is set
+ *
+ * @param flag Flag to see if set
+ * @return {@code} true if flag is set
+ */
+ private boolean has(int flag) {
+ return (flags & flag) != 0;
+ }
+
// ================================================= Inner Classes =================================================
/**
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java?rev=1166766&r1=1166765&r2=1166766&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterFilterFactory.java Thu Sep 8 15:52:10 2011
@@ -33,6 +33,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.io.IOException;
+import static org.apache.solr.analysis.WordDelimiterFilter.*;
+
/**
* Factory for WordDelimiterFilter.
@@ -78,38 +80,44 @@ public class WordDelimiterFilterFactory
}
private CharArraySet protectedWords = null;
-
- int generateWordParts=0;
- int generateNumberParts=0;
- int catenateWords=0;
- int catenateNumbers=0;
- int catenateAll=0;
- int splitOnCaseChange=0;
- int splitOnNumerics=0;
- int preserveOriginal=0;
- int stemEnglishPossessive=0;
+ private int flags;
byte[] typeTable = null;
@Override
public void init(Map<String, String> args) {
super.init(args);
- generateWordParts = getInt("generateWordParts", 1);
- generateNumberParts = getInt("generateNumberParts", 1);
- catenateWords = getInt("catenateWords", 0);
- catenateNumbers = getInt("catenateNumbers", 0);
- catenateAll = getInt("catenateAll", 0);
- splitOnCaseChange = getInt("splitOnCaseChange", 1);
- splitOnNumerics = getInt("splitOnNumerics", 1);
- preserveOriginal = getInt("preserveOriginal", 0);
- stemEnglishPossessive = getInt("stemEnglishPossessive", 1);
+ if (getInt("generateWordParts", 1) != 0) {
+ flags |= GENERATE_WORD_PARTS;
+ }
+ if (getInt("generateNumberParts", 1) != 0) {
+ flags |= GENERATE_NUMBER_PARTS;
+ }
+ if (getInt("catenateWords", 0) != 0) {
+ flags |= CATENATE_WORDS;
+ }
+ if (getInt("catenateNumbers", 0) != 0) {
+ flags |= CATENATE_NUMBERS;
+ }
+ if (getInt("catenateAll", 0) != 0) {
+ flags |= CATENATE_ALL;
+ }
+ if (getInt("splitOnCaseChange", 1) != 0) {
+ flags |= SPLIT_ON_CASE_CHANGE;
+ }
+ if (getInt("splitOnNumerics", 1) != 0) {
+ flags |= SPLIT_ON_NUMERICS;
+ }
+ if (getInt("preserveOriginal", 0) != 0) {
+ flags |= PRESERVE_ORIGINAL;
+ }
+ if (getInt("stemEnglishPossessive", 1) != 0) {
+ flags |= STEM_ENGLISH_POSSESSIVE;
+ }
}
public WordDelimiterFilter create(TokenStream input) {
return new WordDelimiterFilter(input, typeTable == null ? WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE : typeTable,
- generateWordParts, generateNumberParts,
- catenateWords, catenateNumbers, catenateAll,
- splitOnCaseChange, preserveOriginal,
- splitOnNumerics, stemEnglishPossessive, protectedWords);
+ flags, protectedWords);
}
// source => type
@@ -142,17 +150,17 @@ public class WordDelimiterFilterFactory
private Byte parseType(String s) {
if (s.equals("LOWER"))
- return WordDelimiterFilter.LOWER;
+ return LOWER;
else if (s.equals("UPPER"))
- return WordDelimiterFilter.UPPER;
+ return UPPER;
else if (s.equals("ALPHA"))
- return WordDelimiterFilter.ALPHA;
+ return ALPHA;
else if (s.equals("DIGIT"))
- return WordDelimiterFilter.DIGIT;
+ return DIGIT;
else if (s.equals("ALPHANUM"))
- return WordDelimiterFilter.ALPHANUM;
+ return ALPHANUM;
else if (s.equals("SUBWORD_DELIM"))
- return WordDelimiterFilter.SUBWORD_DELIM;
+ return SUBWORD_DELIM;
else
return null;
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterIterator.java?rev=1166766&r1=1166765&r2=1166766&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterIterator.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/java/org/apache/solr/analysis/WordDelimiterIterator.java Thu Sep 8 15:52:10 2011
@@ -64,7 +64,7 @@ public final class WordDelimiterIterator
* "O'Neil's" => "O", "Neil"
*/
final boolean stemEnglishPossessive;
-
+
private final byte[] charTypeTable;
/** if true, need to skip over a possessive found in the last call to next() */
Modified: lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java?rev=1166766&r1=1166765&r2=1166766&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilter.java Thu Sep 8 15:52:10 2011
@@ -47,6 +47,9 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import static org.apache.solr.analysis.WordDelimiterFilter.*;
+import static org.apache.solr.analysis.WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE;
+
/**
* New WordDelimiterFilter tests... most of the tests are in ConvertedLegacyTest
*/
@@ -160,21 +163,17 @@ public class TestWordDelimiterFilter ext
@Test
public void testOffsets() throws IOException {
-
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
// test that subwords and catenated subwords have
// the correct offsets.
- WordDelimiterFilter wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("foo-bar", 5, 12)),
- 1,1,0,0,1,1,0);
+ WordDelimiterFilter wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("foo-bar", 5, 12)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar" },
new int[] { 5, 9, 5 },
new int[] { 8, 12, 12 });
- wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("foo-bar", 5, 6)),
- 1,1,0,0,1,1,0);
+ wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("foo-bar", 5, 6)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar" },
@@ -183,12 +182,9 @@ public class TestWordDelimiterFilter ext
}
@Test
- public void testOffsetChange() throws Exception
- {
- WordDelimiterFilter wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("übelkeit)", 7, 16)),
- 1,1,0,0,1,1,0
- );
+ public void testOffsetChange() throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("übelkeit)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "übelkeit" },
@@ -197,12 +193,9 @@ public class TestWordDelimiterFilter ext
}
@Test
- public void testOffsetChange2() throws Exception
- {
- WordDelimiterFilter wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("(übelkeit", 7, 17)),
- 1,1,0,0,1,1,0
- );
+ public void testOffsetChange2() throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("(übelkeit", 7, 17)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "übelkeit" },
@@ -211,12 +204,9 @@ public class TestWordDelimiterFilter ext
}
@Test
- public void testOffsetChange3() throws Exception
- {
- WordDelimiterFilter wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("(übelkeit", 7, 16)),
- 1,1,0,0,1,1,0
- );
+ public void testOffsetChange3() throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("(übelkeit", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "übelkeit" },
@@ -225,12 +215,9 @@ public class TestWordDelimiterFilter ext
}
@Test
- public void testOffsetChange4() throws Exception
- {
- WordDelimiterFilter wdf = new WordDelimiterFilter(
- new SingleTokenTokenStream(new Token("(foo,bar)", 7, 16)),
- 1,1,0,0,1,1,0
- );
+ public void testOffsetChange4() throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(new SingleTokenTokenStream(new Token("(foo,bar)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf,
new String[] { "foo", "bar", "foobar"},
@@ -338,8 +325,9 @@ public class TestWordDelimiterFilter ext
}
public void doSplit(final String input, String... output) throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
WordDelimiterFilter wdf = new WordDelimiterFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.KEYWORD, false), WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, 1, 1, 0, 0, 0, 1, 0, 1, 1, null);
+ new StringReader(input), MockTokenizer.KEYWORD, false), WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf, output);
}
@@ -380,8 +368,10 @@ public class TestWordDelimiterFilter ext
}
public void doSplitPossessive(int stemPossessive, final String input, final String... output) throws Exception {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+ flags |= (stemPossessive == 1) ? STEM_ENGLISH_POSSESSIVE : 0;
WordDelimiterFilter wdf = new WordDelimiterFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.KEYWORD, false), 1,1,0,0,0,1,0,1,stemPossessive, null);
+ new StringReader(input), MockTokenizer.KEYWORD, false), flags, null);
assertTokenStreamContents(wdf, output);
}
@@ -420,7 +410,8 @@ public class TestWordDelimiterFilter ext
@Test
public void testPositionIncrements() throws Exception {
- final CharArraySet protWords = new CharArraySet(DEFAULT_VERSION, new HashSet<String>(Arrays.asList("NUTCH")), false);
+ final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ final CharArraySet protWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<String>(Arrays.asList("NUTCH")), false);
/* analyzer that uses whitespace + wdf */
Analyzer a = new Analyzer() {
@@ -428,7 +419,7 @@ public class TestWordDelimiterFilter ext
public TokenStream tokenStream(String field, Reader reader) {
return new WordDelimiterFilter(
new MockTokenizer(reader, MockTokenizer.WHITESPACE, false),
- 1, 1, 0, 0, 1, 1, 0, 1, 1, protWords);
+ flags, protWords);
}
};
@@ -456,7 +447,7 @@ public class TestWordDelimiterFilter ext
return new WordDelimiterFilter(
new LargePosIncTokenFilter(
new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)),
- 1, 1, 0, 0, 1, 1, 0, 1, 1, protWords);
+ flags, protWords);
}
};
@@ -489,8 +480,7 @@ public class TestWordDelimiterFilter ext
StopFilter filter = new StopFilter(TEST_VERSION_CURRENT,
new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), StandardAnalyzer.STOP_WORDS_SET);
filter.setEnablePositionIncrements(true);
- return new WordDelimiterFilter(filter,
- 1, 1, 0, 0, 1, 1, 0, 1, 1, protWords);
+ return new WordDelimiterFilter(filter, flags, protWords);
}
};