You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/07 13:21:14 UTC
svn commit: r1479862 [4/38] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/
dev-tools/maven/ dev-tools/maven/solr/ dev-tools/maven/solr/core/src/java/
dev-tools/maven/solr/solrj/src/java/ dev-to...
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java Tue May 7 11:20:55 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.misce
*/
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.CapitalizationFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -26,7 +25,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
-import java.util.StringTokenizer;
+import java.util.Set;
/**
* Factory for {@link CapitalizationFilter}.
@@ -44,7 +43,7 @@ import java.util.StringTokenizer;
* "maxWordCount" - if the token contains more then maxWordCount words, the capitalization is
* assumed to be correct.<br/>
*
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_cptlztn" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
@@ -54,7 +53,6 @@ import java.util.StringTokenizer;
* </analyzer>
* </fieldType></pre>
*
- *
* @since solr 1.3
*/
public class CapitalizationFilterFactory extends TokenFilterFactory {
@@ -67,69 +65,42 @@ public class CapitalizationFilterFactory
public static final String ONLY_FIRST_WORD = "onlyFirstWord";
public static final String FORCE_FIRST_LETTER = "forceFirstLetter";
- //Map<String,String> keep = new HashMap<String, String>(); // not synchronized because it is only initialized once
CharArraySet keep;
Collection<char[]> okPrefix = Collections.emptyList(); // for Example: McK
- int minWordLength = 0; // don't modify capitalization for words shorter then this
- int maxWordCount = CapitalizationFilter.DEFAULT_MAX_WORD_COUNT;
- int maxTokenLength = CapitalizationFilter.DEFAULT_MAX_TOKEN_LENGTH;
- boolean onlyFirstWord = true;
- boolean forceFirstLetter = true; // make sure the first letter is capitol even if it is in the keep list
-
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
+ final int minWordLength; // don't modify capitalization for words shorter then this
+ final int maxWordCount;
+ final int maxTokenLength;
+ final boolean onlyFirstWord;
+ final boolean forceFirstLetter; // make sure the first letter is capital even if it is in the keep list
+
+ /** Creates a new CapitalizationFilterFactory */
+ public CapitalizationFilterFactory(Map<String, String> args) {
+ super(args);
assureMatchVersion();
-
- String k = args.get(KEEP);
+ boolean ignoreCase = getBoolean(args, KEEP_IGNORE_CASE, false);
+ Set<String> k = getSet(args, KEEP);
if (k != null) {
- StringTokenizer st = new StringTokenizer(k);
- boolean ignoreCase = false;
- String ignoreStr = args.get(KEEP_IGNORE_CASE);
- if ("true".equalsIgnoreCase(ignoreStr)) {
- ignoreCase = true;
- }
keep = new CharArraySet(luceneMatchVersion, 10, ignoreCase);
- while (st.hasMoreTokens()) {
- k = st.nextToken().trim();
- keep.add(k.toCharArray());
- }
+ keep.addAll(k);
}
- k = args.get(OK_PREFIX);
+ k = getSet(args, OK_PREFIX);
if (k != null) {
okPrefix = new ArrayList<char[]>();
- StringTokenizer st = new StringTokenizer(k);
- while (st.hasMoreTokens()) {
- okPrefix.add(st.nextToken().trim().toCharArray());
+ for (String item : k) {
+ okPrefix.add(item.toCharArray());
}
}
- k = args.get(MIN_WORD_LENGTH);
- if (k != null) {
- minWordLength = Integer.valueOf(k);
- }
-
- k = args.get(MAX_WORD_COUNT);
- if (k != null) {
- maxWordCount = Integer.valueOf(k);
- }
-
- k = args.get(MAX_TOKEN_LENGTH);
- if (k != null) {
- maxTokenLength = Integer.valueOf(k);
- }
-
- k = args.get(ONLY_FIRST_WORD);
- if (k != null) {
- onlyFirstWord = Boolean.valueOf(k);
- }
-
- k = args.get(FORCE_FIRST_LETTER);
- if (k != null) {
- forceFirstLetter = Boolean.valueOf(k);
+ minWordLength = getInt(args, MIN_WORD_LENGTH, 0);
+ maxWordCount = getInt(args, MAX_WORD_COUNT, CapitalizationFilter.DEFAULT_MAX_WORD_COUNT);
+ maxTokenLength = getInt(args, MAX_TOKEN_LENGTH, CapitalizationFilter.DEFAULT_MAX_TOKEN_LENGTH);
+ onlyFirstWord = getBoolean(args, ONLY_FIRST_WORD, true);
+ forceFirstLetter = getBoolean(args, FORCE_FIRST_LETTER, true);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/HyphenatedWordsFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,22 +17,32 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.HyphenatedWordsFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link HyphenatedWordsFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_hyphn" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.HyphenatedWordsFilterFactory"/>
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class HyphenatedWordsFilterFactory extends TokenFilterFactory {
+
+ /** Creates a new HyphenatedWordsFilterFactory */
+ public HyphenatedWordsFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public HyphenatedWordsFilter create(TokenStream input) {
return new HyphenatedWordsFilter(input);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java Tue May 7 11:20:55 2013
@@ -21,6 +21,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.Version;
/**
* A TokenFilter that only keeps tokens with text contained in the
@@ -32,10 +33,16 @@ public final class KeepWordFilter extend
private final CharArraySet words;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- /** The words set passed to this constructor will be directly used by this filter
- * and should not be modified, */
- public KeepWordFilter(boolean enablePositionIncrements, TokenStream in, CharArraySet words) {
- super(enablePositionIncrements, in);
+ /**
+ * Create a new {@link KeepWordFilter}.
+ * <p><b>NOTE</b>: The words set passed to this constructor will be directly
+ * used by this filter and should not be modified.
+ * @param version the Lucene match version
+ * @param in the {@link TokenStream} to consume
+ * @param words the words to keep
+ */
+ public KeepWordFilter(Version version, TokenStream in, CharArraySet words) {
+ super(version, in);
this.words = words;
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,67 +17,48 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
-import org.apache.lucene.analysis.util.*;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.KeepWordFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
import java.util.Map;
-import java.util.Set;
import java.io.IOException;
/**
* Factory for {@link KeepWordFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_keepword" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.KeepWordFilterFactory" words="keepwords.txt" ignoreCase="false" enablePositionIncrements="false"/>
+ * <filter class="solr.KeepWordFilterFactory" words="keepwords.txt" ignoreCase="false"/>
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class KeepWordFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
-
- @Override
- public void init(Map<String,String> args) {
- super.init(args);
+ private final boolean ignoreCase;
+ private final String wordFiles;
+ private CharArraySet words;
+
+ /** Creates a new KeepWordFilterFactory */
+ public KeepWordFilterFactory(Map<String,String> args) {
+ super(args);
assureMatchVersion();
+ wordFiles = get(args, "words");
+ ignoreCase = getBoolean(args, "ignoreCase", false);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@Override
public void inform(ResourceLoader loader) throws IOException {
- String wordFiles = args.get("words");
- ignoreCase = getBoolean("ignoreCase", false);
- enablePositionIncrements = getBoolean("enablePositionIncrements",false);
-
if (wordFiles != null) {
words = getWordSet(loader, wordFiles, ignoreCase);
}
}
- private CharArraySet words;
- private boolean ignoreCase;
- private boolean enablePositionIncrements;
-
- /**
- * Set the keep word list.
- * NOTE: if ignoreCase==true, the words are expected to be lowercase
- */
- public void setWords(Set<String> words) {
- this.words = new CharArraySet(luceneMatchVersion, words, ignoreCase);
- }
-
- public void setIgnoreCase(boolean ignoreCase) {
- if (words != null && this.ignoreCase != ignoreCase) {
- words = new CharArraySet(luceneMatchVersion, words, ignoreCase);
- }
- this.ignoreCase = ignoreCase;
- }
-
- public boolean isEnablePositionIncrements() {
- return enablePositionIncrements;
- }
-
public boolean isIgnoreCase() {
return ignoreCase;
}
@@ -89,6 +70,11 @@ public class KeepWordFilterFactory exten
@Override
public TokenStream create(TokenStream input) {
// if the set is null, it means it was empty
- return words == null ? input : new KeepWordFilter(enablePositionIncrements, input, words);
+ if (words == null) {
+ return input;
+ } else {
+ final TokenStream filter = new KeepWordFilter(luceneMatchVersion, input, words);
+ return filter;
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordMarkerFilterFactory.java Tue May 7 11:20:55 2013
@@ -18,42 +18,53 @@ package org.apache.lucene.analysis.misce
*/
import java.io.IOException;
+import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
-import org.apache.lucene.analysis.util.*;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link KeywordMarkerFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_keyword" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.KeywordMarkerFilterFactory" protected="protectedkeyword.txt" pattern="^.+er$" ignoreCase="false"/>
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class KeywordMarkerFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
public static final String PROTECTED_TOKENS = "protected";
public static final String PATTERN = "pattern";
- private CharArraySet protectedWords;
- private boolean ignoreCase;
+ private final String wordFiles;
+ private final String stringPattern;
+ private final boolean ignoreCase;
private Pattern pattern;
+ private CharArraySet protectedWords;
+
+ /** Creates a new KeywordMarkerFilterFactory */
+ public KeywordMarkerFilterFactory(Map<String,String> args) {
+ super(args);
+ wordFiles = get(args, PROTECTED_TOKENS);
+ stringPattern = get(args, PATTERN);
+ ignoreCase = getBoolean(args, "ignoreCase", false);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
@Override
public void inform(ResourceLoader loader) throws IOException {
- String wordFiles = args.get(PROTECTED_TOKENS);
- String stringPattern = args.get(PATTERN);
- ignoreCase = getBoolean("ignoreCase", false);
if (wordFiles != null) {
protectedWords = getWordSet(loader, wordFiles, ignoreCase);
}
if (stringPattern != null) {
pattern = ignoreCase ? Pattern.compile(stringPattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE) : Pattern.compile(stringPattern);
}
-
}
public boolean isIgnoreCase() {
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordRepeatFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordRepeatFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordRepeatFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeywordRepeatFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,6 +17,8 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -28,6 +30,15 @@ import org.apache.lucene.analysis.util.T
* {@link RemoveDuplicatesTokenFilterFactory} later in the analysis chain.
*/
public final class KeywordRepeatFilterFactory extends TokenFilterFactory {
+
+ /** Creates a new KeywordRepeatFilterFactory */
+ public KeywordRepeatFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public TokenStream create(TokenStream input) {
return new KeywordRepeatFilter(input);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java Tue May 7 11:20:55 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.misce
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
/**
* Removes words that are too long or too short from the stream.
@@ -35,15 +36,20 @@ public final class LengthFilter extends
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/**
- * Build a filter that removes words that are too long or too
- * short from the text.
+ * Create a new {@link LengthFilter}. This will filter out tokens whose
+ * {@link CharTermAttribute} is either too short ({@link CharTermAttribute#length()}
+ * < min) or too long ({@link CharTermAttribute#length()} > max).
+ * @param version the Lucene match version
+ * @param in the {@link TokenStream} to consume
+ * @param min the minimum length
+ * @param max the maximum length
*/
- public LengthFilter(boolean enablePositionIncrements, TokenStream in, int min, int max) {
- super(enablePositionIncrements, in);
+ public LengthFilter(Version version, TokenStream in, int min, int max) {
+ super(version, in);
this.min = min;
this.max = max;
}
-
+
@Override
public boolean accept() {
final int len = termAtt.length();
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,44 +17,40 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.LengthFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Map;
-
/**
* Factory for {@link LengthFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_lngth" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.LengthFilterFactory" min="0" max="1" enablePositionIncrements="false"/>
+ * <filter class="solr.LengthFilterFactory" min="0" max="1" />
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class LengthFilterFactory extends TokenFilterFactory {
- int min,max;
- boolean enablePositionIncrements;
+ final int min;
+ final int max;
public static final String MIN_KEY = "min";
public static final String MAX_KEY = "max";
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- String minKey = args.get(MIN_KEY);
- String maxKey = args.get(MAX_KEY);
- if (minKey == null || maxKey == null) {
- throw new IllegalArgumentException("Both " + MIN_KEY + " and " + MAX_KEY + " are mandatory");
+ /** Creates a new LengthFilterFactory */
+ public LengthFilterFactory(Map<String, String> args) {
+ super(args);
+ min = requireInt(args, MIN_KEY);
+ max = requireInt(args, MAX_KEY);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
}
- min=Integer.parseInt(minKey);
- max=Integer.parseInt(maxKey);
- enablePositionIncrements = getBoolean("enablePositionIncrements",false);
}
@Override
public LengthFilter create(TokenStream input) {
- return new LengthFilter(enablePositionIncrements, input,min,max);
+ final LengthFilter filter = new LengthFilter(luceneMatchVersion, input,min,max);
+ return filter;
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java Tue May 7 11:20:55 2013
@@ -20,33 +20,36 @@ package org.apache.lucene.analysis.misce
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.LimitTokenCountFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link LimitTokenCountFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_lngthcnt" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10" consumeAllTokens="false" />
* </analyzer>
- * </fieldType></pre>
+ * </fieldType></pre>
* <p>
- * The {@code consumeAllTokens} property is optional and defaults to {@code false}. See {@link LimitTokenCountFilter} for an explanation of it's use.
+ * The {@code consumeAllTokens} property is optional and defaults to {@code false}.
+ * See {@link LimitTokenCountFilter} for an explanation of it's use.
*/
public class LimitTokenCountFilterFactory extends TokenFilterFactory {
public static final String MAX_TOKEN_COUNT_KEY = "maxTokenCount";
public static final String CONSUME_ALL_TOKENS_KEY = "consumeAllTokens";
- int maxTokenCount;
- boolean consumeAllTokens;
+ final int maxTokenCount;
+ final boolean consumeAllTokens;
- @Override
- public void init(Map<String, String> args) {
- super.init( args );
- maxTokenCount = getInt(MAX_TOKEN_COUNT_KEY);
- consumeAllTokens = getBoolean(CONSUME_ALL_TOKENS_KEY, false);
+ /** Creates a new LimitTokenCountFilterFactory */
+ public LimitTokenCountFilterFactory(Map<String, String> args) {
+ super(args);
+ maxTokenCount = requireInt(args, MAX_TOKEN_COUNT_KEY);
+ consumeAllTokens = getBoolean(args, CONSUME_ALL_TOKENS_KEY, false);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenPositionFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenPositionFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenPositionFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenPositionFilterFactory.java Tue May 7 11:20:55 2013
@@ -23,13 +23,13 @@ import org.apache.lucene.analysis.util.T
/**
* Factory for {@link LimitTokenPositionFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_limit_pos" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.LimitTokenPositionFilterFactory" maxTokenPosition="3" consumeAllTokens="false" />
* </analyzer>
- * </fieldType></pre>
+ * </fieldType></pre>
* <p>
* The {@code consumeAllTokens} property is optional and defaults to {@code false}.
* See {@link LimitTokenPositionFilter} for an explanation of its use.
@@ -38,14 +38,17 @@ public class LimitTokenPositionFilterFac
public static final String MAX_TOKEN_POSITION_KEY = "maxTokenPosition";
public static final String CONSUME_ALL_TOKENS_KEY = "consumeAllTokens";
- int maxTokenPosition;
- boolean consumeAllTokens;
+ final int maxTokenPosition;
+ final boolean consumeAllTokens;
- @Override
- public void init(Map<String,String> args) {
- super.init(args);
- maxTokenPosition = getInt(MAX_TOKEN_POSITION_KEY);
- consumeAllTokens = getBoolean(CONSUME_ALL_TOKENS_KEY, false);
+ /** Creates a new LimitTokenPositionFilterFactory */
+ public LimitTokenPositionFilterFactory(Map<String,String> args) {
+ super(args);
+ maxTokenPosition = requireInt(args, MAX_TOKEN_POSITION_KEY);
+ consumeAllTokens = getBoolean(args, CONSUME_ALL_TOKENS_KEY, false);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/RemoveDuplicatesTokenFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,22 +17,32 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.miscellaneous.RemoveDuplicatesTokenFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link RemoveDuplicatesTokenFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_rmdup" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
* </analyzer>
* </fieldType></pre>
- *
*/
public class RemoveDuplicatesTokenFilterFactory extends TokenFilterFactory {
+
+ /** Creates a new RemoveDuplicatesTokenFilterFactory */
+ public RemoveDuplicatesTokenFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public RemoveDuplicatesTokenFilter create(TokenStream input) {
return new RemoveDuplicatesTokenFilter(input);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java Tue May 7 11:20:55 2013
@@ -18,22 +18,34 @@ package org.apache.lucene.analysis.misce
*/
import java.io.IOException;
+import java.util.ArrayList;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.util.CharArrayMap;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.fst.ByteSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.FST.Arc;
+import org.apache.lucene.util.fst.FST.BytesReader;
/**
* Provides the ability to override any {@link KeywordAttribute} aware stemmer
* with custom dictionary-based stemming.
*/
public final class StemmerOverrideFilter extends TokenFilter {
- private final CharArrayMap<String> dictionary;
+ private final StemmerOverrideMap stemmerOverrideMap;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
+ private final BytesReader fstReader;
+ private final Arc<BytesRef> scratchArc = new FST.Arc<BytesRef>();
+ private final CharsRef spare = new CharsRef();
/**
* Create a new StemmerOverrideFilter, performing dictionary-based stemming
@@ -43,19 +55,28 @@ public final class StemmerOverrideFilter
* so that they will not be stemmed with stemmers down the chain.
* </p>
*/
- public StemmerOverrideFilter(TokenStream input,
- CharArrayMap<String> dictionary) {
+ public StemmerOverrideFilter(final TokenStream input, final StemmerOverrideMap stemmerOverrideMap) {
super(input);
- this.dictionary = dictionary;
+ this.stemmerOverrideMap = stemmerOverrideMap;
+ fstReader = stemmerOverrideMap.getBytesReader();
}
-
+
@Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
+ if (fstReader == null) {
+ // No overrides
+ return true;
+ }
if (!keywordAtt.isKeyword()) { // don't muck with already-keyworded terms
- String stem = dictionary.get(termAtt.buffer(), 0, termAtt.length());
+ final BytesRef stem = stemmerOverrideMap.get(termAtt.buffer(), termAtt.length(), scratchArc, fstReader);
if (stem != null) {
- termAtt.setEmpty().append(stem);
+ final char[] buffer = spare.chars = termAtt.buffer();
+ UnicodeUtil.UTF8toUTF16(stem.bytes, stem.offset, stem.length, spare);
+ if (spare.chars != buffer) {
+ termAtt.copyBuffer(spare.chars, spare.offset, spare.length);
+ }
+ termAtt.setLength(spare.length);
keywordAtt.setKeyword(true);
}
}
@@ -64,4 +85,134 @@ public final class StemmerOverrideFilter
return false;
}
}
+
+ /**
+ * A read-only 4-byte FST backed map that allows fast case-insensitive key
+ * value lookups for {@link StemmerOverrideFilter}
+ */
+ // TODO maybe we can generalize this and reuse this map somehow?
+ public final static class StemmerOverrideMap {
+ private final FST<BytesRef> fst;
+ private final boolean ignoreCase;
+
+ /**
+ * Creates a new {@link StemmerOverrideMap}
+ * @param fst the fst to lookup the overrides
+ * @param ignoreCase if the keys case should be ingored
+ */
+ StemmerOverrideMap(FST<BytesRef> fst, boolean ignoreCase) {
+ this.fst = fst;
+ this.ignoreCase = ignoreCase;
+ }
+
+ /**
+ * Returns a {@link BytesReader} to pass to the {@link #get(char[], int, Arc, BytesReader)} method.
+ */
+ BytesReader getBytesReader() {
+ if (fst == null) {
+ return null;
+ } else {
+ return fst.getBytesReader();
+ }
+ }
+
+ /**
+ * Returns the value mapped to the given key or <code>null</code> if the key is not in the FST dictionary.
+ */
+ BytesRef get(char[] buffer, int bufferLen, Arc<BytesRef> scratchArc, BytesReader fstReader) throws IOException {
+ BytesRef pendingOutput = fst.outputs.getNoOutput();
+ BytesRef matchOutput = null;
+ int bufUpto = 0;
+ fst.getFirstArc(scratchArc);
+ while (bufUpto < bufferLen) {
+ final int codePoint = Character.codePointAt(buffer, bufUpto, bufferLen);
+ if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc, fstReader) == null) {
+ return null;
+ }
+ pendingOutput = fst.outputs.add(pendingOutput, scratchArc.output);
+ bufUpto += Character.charCount(codePoint);
+ }
+ if (scratchArc.isFinal()) {
+ matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput);
+ }
+ return matchOutput;
+ }
+
+ }
+ /**
+ * This builder builds an {@link FST} for the {@link StemmerOverrideFilter}
+ */
+ public static class Builder {
+ private final BytesRefHash hash = new BytesRefHash();
+ private final BytesRef spare = new BytesRef();
+ private final ArrayList<CharSequence> outputValues = new ArrayList<CharSequence>();
+ private final boolean ignoreCase;
+ private final CharsRef charsSpare = new CharsRef();
+
+ /**
+ * Creates a new {@link Builder} with ignoreCase set to <code>false</code>
+ */
+ public Builder() {
+ this(false);
+ }
+
+ /**
+ * Creates a new {@link Builder}
+ * @param ignoreCase if the input case should be ignored.
+ */
+ public Builder(boolean ignoreCase) {
+ this.ignoreCase = ignoreCase;
+ }
+
+ /**
+ * Adds an input string and it's stemmer override output to this builder.
+ *
+ * @param input the input char sequence
+ * @param output the stemmer override output char sequence
+ * @return <code>false</code> iff the input has already been added to this builder otherwise <code>true</code>.
+ */
+ public boolean add(CharSequence input, CharSequence output) {
+ final int length = input.length();
+ if (ignoreCase) {
+ // convert on the fly to lowercase
+ charsSpare.grow(length);
+ final char[] buffer = charsSpare.chars;
+ for (int i = 0; i < length; ) {
+ i += Character.toChars(
+ Character.toLowerCase(
+ Character.codePointAt(input, i)), buffer, i);
+ }
+ UnicodeUtil.UTF16toUTF8(buffer, 0, length, spare);
+ } else {
+ UnicodeUtil.UTF16toUTF8(input, 0, length, spare);
+ }
+ if (hash.add(spare) >= 0) {
+ outputValues.add(output);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Returns an {@link StemmerOverrideMap} to be used with the {@link StemmerOverrideFilter}
+ * @return an {@link StemmerOverrideMap} to be used with the {@link StemmerOverrideFilter}
+ * @throws IOException if an {@link IOException} occurs;
+ */
+ public StemmerOverrideMap build() throws IOException {
+ ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ org.apache.lucene.util.fst.Builder<BytesRef> builder = new org.apache.lucene.util.fst.Builder<BytesRef>(
+ FST.INPUT_TYPE.BYTE4, outputs);
+ final int[] sort = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
+ IntsRef intsSpare = new IntsRef();
+ final int size = hash.size();
+ for (int i = 0; i < size; i++) {
+ int id = sort[i];
+ BytesRef bytesRef = hash.get(id, spare);
+ UnicodeUtil.UTF8toUTF32(bytesRef, intsSpare);
+ builder.add(intsSpare, new BytesRef(outputValues.get(id)));
+ }
+ return new StemmerOverrideMap(builder.finish(), ignoreCase);
+ }
+
+ }
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilterFactory.java Tue May 7 11:20:55 2013
@@ -19,43 +19,54 @@ package org.apache.lucene.analysis.misce
import java.io.IOException;
import java.util.List;
+import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter;
-import org.apache.lucene.analysis.util.*;
+import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter.StemmerOverrideMap;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link StemmerOverrideFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_dicstem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.StemmerOverrideFilterFactory" dictionary="dictionary.txt" ignoreCase="false"/>
* </analyzer>
* </fieldType></pre>
- *
*/
public class StemmerOverrideFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
- private CharArrayMap<String> dictionary = null;
- private boolean ignoreCase;
+ private StemmerOverrideMap dictionary;
+ private final String dictionaryFiles;
+ private final boolean ignoreCase;
+
+ /** Creates a new StemmerOverrideFilterFactory */
+ public StemmerOverrideFilterFactory(Map<String,String> args) {
+ super(args);
+ dictionaryFiles = get(args, "dictionary");
+ ignoreCase = getBoolean(args, "ignoreCase", false);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
@Override
public void inform(ResourceLoader loader) throws IOException {
- String dictionaryFiles = args.get("dictionary");
- ignoreCase = getBoolean("ignoreCase", false);
if (dictionaryFiles != null) {
assureMatchVersion();
List<String> files = splitFileNames(dictionaryFiles);
if (files.size() > 0) {
- dictionary = new CharArrayMap<String>(luceneMatchVersion,
- files.size() * 10, ignoreCase);
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(ignoreCase);
for (String file : files) {
List<String> list = getLines(loader, file.trim());
for (String line : list) {
String[] mapping = line.split("\t", 2);
- dictionary.put(mapping[0], mapping[1]);
+ builder.add(mapping[0], mapping[1]);
}
}
+ dictionary = builder.build();
}
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java Tue May 7 11:20:55 2013
@@ -21,22 +21,26 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util.Version;
import java.io.IOException;
/**
* Trims leading and trailing whitespace from Tokens in the stream.
+ * <p>As of Lucene 4.4, this filter does not support updateOffsets=true anymore
+ * as it can lead to broken token streams.
*/
public final class TrimFilter extends TokenFilter {
- final boolean updateOffsets;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
-
- public TrimFilter(TokenStream in, boolean updateOffsets) {
+ /**
+ * Create a new {@link TrimFilter}.
+ * @param version the Lucene match version
+ * @param in the stream to consume
+ */
+ public TrimFilter(Version version, TokenStream in) {
super(in);
- this.updateOffsets = updateOffsets;
}
@Override
@@ -52,15 +56,12 @@ public final class TrimFilter extends To
}
int start = 0;
int end = 0;
- int endOff = 0;
// eat the first characters
- //QUESTION: Should we use Character.isWhitespace() instead?
- for (start = 0; start < len && termBuffer[start] <= ' '; start++) {
+ for (start = 0; start < len && Character.isWhitespace(termBuffer[start]); start++) {
}
// eat the end characters
- for (end = len; end >= start && termBuffer[end - 1] <= ' '; end--) {
- endOff++;
+ for (end = len; end >= start && Character.isWhitespace(termBuffer[end - 1]); end--) {
}
if (start > 0 || end < len) {
if (start < end) {
@@ -68,11 +69,6 @@ public final class TrimFilter extends To
} else {
termAtt.setEmpty();
}
- if (updateOffsets && len == offsetAtt.endOffset() - offsetAtt.startOffset()) {
- int newStart = offsetAtt.startOffset()+start;
- int newEnd = offsetAtt.endOffset() - (start<end ? endOff:0);
- offsetAtt.setOffset(newStart, newEnd);
- }
}
return true;
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java Tue May 7 11:20:55 2013
@@ -25,11 +25,11 @@ import org.apache.lucene.analysis.util.T
/**
* Factory for {@link TrimFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_trm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.NGramTokenizerFactory"/>
- * <filter class="solr.TrimFilterFactory" updateOffsets="false"/>
+ * <filter class="solr.TrimFilterFactory" />
* </analyzer>
* </fieldType></pre>
*
@@ -37,20 +37,17 @@ import org.apache.lucene.analysis.util.T
*/
public class TrimFilterFactory extends TokenFilterFactory {
- protected boolean updateOffsets = false;
-
- @Override
- public void init(Map<String,String> args) {
- super.init( args );
-
- String v = args.get( "updateOffsets" );
- if (v != null) {
- updateOffsets = Boolean.valueOf( v );
+ /** Creates a new TrimFilterFactory */
+ public TrimFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
@Override
public TrimFilter create(TokenStream input) {
- return new TrimFilter(input, updateOffsets);
+ final TrimFilter filter = new TrimFilter(luceneMatchVersion, input);
+ return filter;
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java Tue May 7 11:20:55 2013
@@ -18,9 +18,10 @@ package org.apache.lucene.analysis.misce
*/
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter;
-import org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator;
-import org.apache.lucene.analysis.util.*;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
import java.util.ArrayList;
import java.util.List;
@@ -33,10 +34,9 @@ import java.io.IOException;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.*;
-
/**
* Factory for {@link WordDelimiterFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_wd" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
@@ -46,65 +46,71 @@ import static org.apache.lucene.analysis
* generateWordParts="1" generateNumberParts="1" stemEnglishPossessive="1"
* types="wdfftypes.txt" />
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class WordDelimiterFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
public static final String PROTECTED_TOKENS = "protected";
public static final String TYPES = "types";
-
- @Override
- public void inform(ResourceLoader loader) throws IOException {
- String wordFiles = args.get(PROTECTED_TOKENS);
- if (wordFiles != null) {
- protectedWords = getWordSet(loader, wordFiles, false);
- }
- String types = args.get(TYPES);
- if (types != null) {
- List<String> files = splitFileNames( types );
- List<String> wlist = new ArrayList<String>();
- for( String file : files ){
- List<String> lines = getLines(loader, file.trim());
- wlist.addAll( lines );
- }
- typeTable = parseTypes(wlist);
- }
- }
- private CharArraySet protectedWords = null;
- private int flags;
+ private final String wordFiles;
+ private final String types;
+ private final int flags;
byte[] typeTable = null;
-
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- if (getInt("generateWordParts", 1) != 0) {
+ private CharArraySet protectedWords = null;
+
+ /** Creates a new WordDelimiterFilterFactory */
+ public WordDelimiterFilterFactory(Map<String, String> args) {
+ super(args);
+ int flags = 0;
+ if (getInt(args, "generateWordParts", 1) != 0) {
flags |= GENERATE_WORD_PARTS;
}
- if (getInt("generateNumberParts", 1) != 0) {
+ if (getInt(args, "generateNumberParts", 1) != 0) {
flags |= GENERATE_NUMBER_PARTS;
}
- if (getInt("catenateWords", 0) != 0) {
+ if (getInt(args, "catenateWords", 0) != 0) {
flags |= CATENATE_WORDS;
}
- if (getInt("catenateNumbers", 0) != 0) {
+ if (getInt(args, "catenateNumbers", 0) != 0) {
flags |= CATENATE_NUMBERS;
}
- if (getInt("catenateAll", 0) != 0) {
+ if (getInt(args, "catenateAll", 0) != 0) {
flags |= CATENATE_ALL;
}
- if (getInt("splitOnCaseChange", 1) != 0) {
+ if (getInt(args, "splitOnCaseChange", 1) != 0) {
flags |= SPLIT_ON_CASE_CHANGE;
}
- if (getInt("splitOnNumerics", 1) != 0) {
+ if (getInt(args, "splitOnNumerics", 1) != 0) {
flags |= SPLIT_ON_NUMERICS;
}
- if (getInt("preserveOriginal", 0) != 0) {
+ if (getInt(args, "preserveOriginal", 0) != 0) {
flags |= PRESERVE_ORIGINAL;
}
- if (getInt("stemEnglishPossessive", 1) != 0) {
+ if (getInt(args, "stemEnglishPossessive", 1) != 0) {
flags |= STEM_ENGLISH_POSSESSIVE;
}
+ wordFiles = get(args, PROTECTED_TOKENS);
+ types = get(args, TYPES);
+ this.flags = flags;
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ @Override
+ public void inform(ResourceLoader loader) throws IOException {
+ if (wordFiles != null) {
+ protectedWords = getWordSet(loader, wordFiles, false);
+ }
+ if (types != null) {
+ List<String> files = splitFileNames( types );
+ List<String> wlist = new ArrayList<String>();
+ for( String file : files ){
+ List<String> lines = getLines(loader, file.trim());
+ wlist.addAll( lines );
+ }
+ typeTable = parseTypes(wlist);
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramFilterFactory.java Tue May 7 11:20:55 2013
@@ -19,41 +19,31 @@ package org.apache.lucene.analysis.ngram
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Creates new instances of {@link EdgeNGramTokenFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_edgngrm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.EdgeNGramFilterFactory" side="front" minGramSize="1" maxGramSize="1"/>
* </analyzer>
* </fieldType></pre>
- *
*/
public class EdgeNGramFilterFactory extends TokenFilterFactory {
- private int maxGramSize = 0;
-
- private int minGramSize = 0;
-
- private String side;
-
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- String maxArg = args.get("maxGramSize");
- maxGramSize = (maxArg != null ? Integer.parseInt(maxArg)
- : EdgeNGramTokenFilter.DEFAULT_MAX_GRAM_SIZE);
-
- String minArg = args.get("minGramSize");
- minGramSize = (minArg != null ? Integer.parseInt(minArg)
- : EdgeNGramTokenFilter.DEFAULT_MIN_GRAM_SIZE);
-
- side = args.get("side");
- if (side == null) {
- side = EdgeNGramTokenFilter.Side.FRONT.getLabel();
+ private final int maxGramSize;
+ private final int minGramSize;
+ private final String side;
+
+ /** Creates a new EdgeNGramFilterFactory */
+ public EdgeNGramFilterFactory(Map<String, String> args) {
+ super(args);
+ minGramSize = getInt(args, "minGramSize", EdgeNGramTokenFilter.DEFAULT_MIN_GRAM_SIZE);
+ maxGramSize = getInt(args, "maxGramSize", EdgeNGramTokenFilter.DEFAULT_MAX_GRAM_SIZE);
+ side = get(args, "side", EdgeNGramTokenFilter.Side.FRONT.getLabel());
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Tue May 7 11:20:55 2013
@@ -75,6 +75,7 @@ public final class EdgeNGramTokenFilter
private int tokEnd; // only used if the length changed before this filter
private boolean hasIllegalOffsets; // only if the length changed before this filter
private int savePosIncr;
+ private boolean isFirstToken = true;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -138,9 +139,8 @@ public final class EdgeNGramTokenFilter
savePosIncr = posIncrAtt.getPositionIncrement();
}
}
- if (curGramSize <= maxGram) {
- if (! (curGramSize > curTermLength // if the remaining input is too short, we can't generate any n-grams
- || curGramSize > maxGram)) { // if we have hit the end of our n-gram size range, quit
+ if (curGramSize <= maxGram) { // if we have hit the end of our n-gram size range, quit
+ if (curGramSize <= curTermLength) { // if the remaining input is too short, we can't generate any n-grams
// grab gramSize chars from front or back
int start = side == Side.FRONT ? 0 : curTermLength - curGramSize;
int end = start + curGramSize;
@@ -152,12 +152,16 @@ public final class EdgeNGramTokenFilter
}
// first ngram gets increment, others don't
if (curGramSize == minGram) {
- posIncrAtt.setPositionIncrement(savePosIncr);
+ // Leave the first token position increment at the cleared-attribute value of 1
+ if ( ! isFirstToken) {
+ posIncrAtt.setPositionIncrement(savePosIncr);
+ }
} else {
posIncrAtt.setPositionIncrement(0);
}
termAtt.copyBuffer(curTermBuffer, start, curGramSize);
curGramSize++;
+ isFirstToken = false;
return true;
}
}
@@ -169,5 +173,6 @@ public final class EdgeNGramTokenFilter
public void reset() throws IOException {
super.reset();
curTermBuffer = null;
+ isFirstToken = true;
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java Tue May 7 11:20:55 2013
@@ -25,33 +25,26 @@ import java.util.Map;
/**
* Creates new instances of {@link EdgeNGramTokenizer}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_edgngrm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.EdgeNGramTokenizerFactory" side="front" minGramSize="1" maxGramSize="1"/>
* </analyzer>
* </fieldType></pre>
- *
*/
public class EdgeNGramTokenizerFactory extends TokenizerFactory {
- private int maxGramSize = 0;
-
- private int minGramSize = 0;
-
- private String side;
-
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- String maxArg = args.get("maxGramSize");
- maxGramSize = (maxArg != null ? Integer.parseInt(maxArg) : EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
-
- String minArg = args.get("minGramSize");
- minGramSize = (minArg != null ? Integer.parseInt(minArg) : EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE);
-
- side = args.get("side");
- if (side == null) {
- side = EdgeNGramTokenizer.Side.FRONT.getLabel();
+ private final int maxGramSize;
+ private final int minGramSize;
+ private final String side;
+
+ /** Creates a new EdgeNGramTokenizerFactory */
+ public EdgeNGramTokenizerFactory(Map<String, String> args) {
+ super(args);
+ minGramSize = getInt(args, "minGramSize", EdgeNGramTokenizer.DEFAULT_MIN_GRAM_SIZE);
+ maxGramSize = getInt(args, "maxGramSize", EdgeNGramTokenizer.DEFAULT_MAX_GRAM_SIZE);
+ side = get(args, "side", EdgeNGramTokenFilter.Side.FRONT.getLabel());
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java Tue May 7 11:20:55 2013
@@ -19,40 +19,34 @@ package org.apache.lucene.analysis.ngram
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.ngram.NGramTokenFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link NGramTokenFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_ngrm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.NGramFilterFactory" minGramSize="1" maxGramSize="2"/>
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class NGramFilterFactory extends TokenFilterFactory {
- private int maxGramSize = 0;
+ private final int maxGramSize;
+ private final int minGramSize;
- private int minGramSize = 0;
-
- /** Initialize the n-gram min and max sizes and the side from which one should start tokenizing. */
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- String maxArg = args.get("maxGramSize");
- maxGramSize = (maxArg != null ? Integer.parseInt(maxArg)
- : NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE);
-
- String minArg = args.get("minGramSize");
- minGramSize = (minArg != null ? Integer.parseInt(minArg)
- : NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
+ /** Creates a new NGramFilterFactory */
+ public NGramFilterFactory(Map<String, String> args) {
+ super(args);
+ minGramSize = getInt(args, "minGramSize", NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
+ maxGramSize = getInt(args, "maxGramSize", NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
@Override
public NGramTokenFilter create(TokenStream input) {
- return new NGramTokenFilter(input, minGramSize, maxGramSize);
+ return new NGramTokenFilter(luceneMatchVersion, input, minGramSize, maxGramSize);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Tue May 7 11:20:55 2013
@@ -21,37 +21,60 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.miscellaneous.LengthFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.Version;
/**
* Tokenizes the input into n-grams of the given size(s).
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version} compatibility when
+ * creating a {@link NGramTokenFilter}. As of Lucene 4.4, this token filters:<ul>
+ * <li>emits all n-grams for the same token at the same position,</li>
+ * <li>does not modify offsets,</li>
+ * <li>sorts n-grams by their offset in the original token first, then
+ * increasing length (meaning that "abc" will give "a", "ab", "abc", "b", "bc",
+ * "c").</li></ul>
+ * <p>You can make this filter use the old behavior by providing a version <
+ * {@link Version#LUCENE_44} in the constructor but this is not recommended as
+ * it will lead to broken {@link TokenStream}s that will cause highlighting
+ * bugs.
*/
public final class NGramTokenFilter extends TokenFilter {
public static final int DEFAULT_MIN_NGRAM_SIZE = 1;
public static final int DEFAULT_MAX_NGRAM_SIZE = 2;
- private int minGram, maxGram;
-
+ private final int minGram, maxGram;
+
private char[] curTermBuffer;
private int curTermLength;
private int curGramSize;
private int curPos;
+ private int curPosInc, curPosLen;
private int tokStart;
- private int tokEnd; // only used if the length changed before this filter
+ private int tokEnd;
private boolean hasIllegalOffsets; // only if the length changed before this filter
-
+
+ private final Version version;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncAtt;
+ private final PositionLengthAttribute posLenAtt;
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates NGramTokenFilter with given min and max n-grams.
+ * @param version Lucene version to enable correct position increments.
+ * See <a href="#version">above</a> for details.
* @param input {@link TokenStream} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public NGramTokenFilter(TokenStream input, int minGram, int maxGram) {
- super(input);
+ public NGramTokenFilter(Version version, TokenStream input, int minGram, int maxGram) {
+ super(new LengthFilter(version, input, minGram, Integer.MAX_VALUE));
+ this.version = version;
if (minGram < 1) {
throw new IllegalArgumentException("minGram must be greater than zero");
}
@@ -60,14 +83,37 @@ public final class NGramTokenFilter exte
}
this.minGram = minGram;
this.maxGram = maxGram;
+ if (version.onOrAfter(Version.LUCENE_44)) {
+ posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ posLenAtt = addAttribute(PositionLengthAttribute.class);
+ } else {
+ posIncAtt = new PositionIncrementAttribute() {
+ @Override
+ public void setPositionIncrement(int positionIncrement) {}
+ @Override
+ public int getPositionIncrement() {
+ return 0;
+ }
+ };
+ posLenAtt = new PositionLengthAttribute() {
+ @Override
+ public void setPositionLength(int positionLength) {}
+ @Override
+ public int getPositionLength() {
+ return 0;
+ }
+ };
+ }
}
/**
* Creates NGramTokenFilter with default min and max n-grams.
+ * @param version Lucene version to enable correct position increments.
+ * See <a href="#version">above</a> for details.
* @param input {@link TokenStream} holding the input to be tokenized
*/
- public NGramTokenFilter(TokenStream input) {
- this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
+ public NGramTokenFilter(Version version, TokenStream input) {
+ this(version, input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
}
/** Returns the next token in the stream, or null at EOS. */
@@ -82,6 +128,8 @@ public final class NGramTokenFilter exte
curTermLength = termAtt.length();
curGramSize = minGram;
curPos = 0;
+ curPosInc = posIncAtt.getPositionIncrement();
+ curPosLen = posLenAtt.getPositionLength();
tokStart = offsetAtt.startOffset();
tokEnd = offsetAtt.endOffset();
// if length by start + end offsets doesn't match the term text then assume
@@ -89,20 +137,37 @@ public final class NGramTokenFilter exte
hasIllegalOffsets = (tokStart + curTermLength) != tokEnd;
}
}
- while (curGramSize <= maxGram) {
- while (curPos+curGramSize <= curTermLength) { // while there is input
+ if (version.onOrAfter(Version.LUCENE_44)) {
+ if (curGramSize > maxGram || curPos + curGramSize > curTermLength) {
+ ++curPos;
+ curGramSize = minGram;
+ }
+ if (curPos + curGramSize <= curTermLength) {
clearAttributes();
termAtt.copyBuffer(curTermBuffer, curPos, curGramSize);
- if (hasIllegalOffsets) {
- offsetAtt.setOffset(tokStart, tokEnd);
- } else {
- offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
- }
- curPos++;
+ posIncAtt.setPositionIncrement(curPosInc);
+ curPosInc = 0;
+ posLenAtt.setPositionLength(curPosLen);
+ offsetAtt.setOffset(tokStart, tokEnd);
+ curGramSize++;
return true;
}
- curGramSize++; // increase n-gram size
- curPos = 0;
+ } else {
+ while (curGramSize <= maxGram) {
+ while (curPos+curGramSize <= curTermLength) { // while there is input
+ clearAttributes();
+ termAtt.copyBuffer(curTermBuffer, curPos, curGramSize);
+ if (hasIllegalOffsets) {
+ offsetAtt.setOffset(tokStart, tokEnd);
+ } else {
+ offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
+ }
+ curPos++;
+ return true;
+ }
+ curGramSize++; // increase n-gram size
+ curPos = 0;
+ }
}
curTermBuffer = null;
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Tue May 7 11:20:55 2013
@@ -17,64 +17,90 @@ package org.apache.lucene.analysis.ngram
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.util.AttributeSource;
-
-import java.io.IOException;
-import java.io.Reader;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.util.Version;
/**
* Tokenizes the input into n-grams of the given size(s).
+ * <p>On the contrary to {@link NGramTokenFilter}, this class sets offsets so
+ * that characters between startOffset and endOffset in the original stream are
+ * the same as the term chars.
+ * <p>For example, "abcde" would be tokenized as (minGram=2, maxGram=3):
+ * <table>
+ * <tr><th>Term</th><td>ab</td><td>abc</td><td>bc</td><td>bcd</td><td>cd</td><td>cde</td><td>de</td></tr>
+ * <tr><th>Position increment</th><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td></tr>
+ * <tr><th>Position length</th><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td><td>1</td></tr>
+ * <tr><th>Offsets</th><td>[0,2[</td><td>[0,3[</td><td>[1,3[</td><td>[1,4[</td><td>[2,4[</td><td>[2,5[</td><td>[3,5[</td></tr>
+ * </table>
+ * <a name="version"/>
+ * <p>Before Lucene 4.4, this class had a different behavior:<ul>
+ * <li>It didn't support more than 1024 chars of input, the rest was trashed.</li>
+ * <li>The last whitespaces of the 1024 chars block were trimmed.</li>
+ * <li>Tokens were emitted in a different order (by increasing lengths).</li></ul>
+ * <p>Although highly discouraged, it is still possible to use the old behavior
+ * through {@link Lucene43NGramTokenizer}.
*/
public final class NGramTokenizer extends Tokenizer {
public static final int DEFAULT_MIN_NGRAM_SIZE = 1;
public static final int DEFAULT_MAX_NGRAM_SIZE = 2;
- private int minGram, maxGram;
+ private char[] buffer;
+ private int bufferStart, bufferEnd; // remaining slice of the buffer
+ private int offset;
private int gramSize;
- private int pos;
- private int inLen; // length of the input AFTER trim()
- private int charsRead; // length of the input
- private String inStr;
- private boolean started;
-
+ private int minGram, maxGram;
+ private boolean exhausted;
+
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Creates NGramTokenizer with given min and max n-grams.
+ * @param version the lucene compatibility <a href="#version">version</a>
* @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public NGramTokenizer(Reader input, int minGram, int maxGram) {
+ public NGramTokenizer(Version version, Reader input, int minGram, int maxGram) {
super(input);
- init(minGram, maxGram);
+ init(version, minGram, maxGram);
}
/**
* Creates NGramTokenizer with given min and max n-grams.
+ * @param version the lucene compatibility <a href="#version">version</a>
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
* @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public NGramTokenizer(AttributeFactory factory, Reader input, int minGram, int maxGram) {
+ public NGramTokenizer(Version version, AttributeFactory factory, Reader input, int minGram, int maxGram) {
super(factory, input);
- init(minGram, maxGram);
+ init(version, minGram, maxGram);
}
/**
* Creates NGramTokenizer with default min and max n-grams.
+ * @param version the lucene compatibility <a href="#version">version</a>
* @param input {@link Reader} holding the input to be tokenized
*/
- public NGramTokenizer(Reader input) {
- this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
+ public NGramTokenizer(Version version, Reader input) {
+ this(version, input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
}
-
- private void init(int minGram, int maxGram) {
+
+ private void init(Version version, int minGram, int maxGram) {
+ if (!version.onOrAfter(Version.LUCENE_44)) {
+ throw new IllegalArgumentException("This class only works with Lucene 4.4+. To emulate the old (broken) behavior of NGramTokenizer, use Lucene43NGramTokenizer");
+ }
if (minGram < 1) {
throw new IllegalArgumentException("minGram must be greater than zero");
}
@@ -83,73 +109,66 @@ public final class NGramTokenizer extend
}
this.minGram = minGram;
this.maxGram = maxGram;
+ buffer = new char[maxGram + 1024];
}
/** Returns the next token in the stream, or null at EOS. */
@Override
public boolean incrementToken() throws IOException {
clearAttributes();
- if (!started) {
- started = true;
- gramSize = minGram;
- char[] chars = new char[1024];
- charsRead = 0;
- // TODO: refactor to a shared readFully somewhere:
- while (charsRead < chars.length) {
- int inc = input.read(chars, charsRead, chars.length-charsRead);
- if (inc == -1) {
- break;
- }
- charsRead += inc;
- }
- inStr = new String(chars, 0, charsRead).trim(); // remove any trailing empty strings
- if (charsRead == chars.length) {
- // Read extra throwaway chars so that on end() we
- // report the correct offset:
- char[] throwaway = new char[1024];
- while(true) {
- final int inc = input.read(throwaway, 0, throwaway.length);
- if (inc == -1) {
+ // compact
+ if (bufferStart >= buffer.length - maxGram) {
+ System.arraycopy(buffer, bufferStart, buffer, 0, bufferEnd - bufferStart);
+ bufferEnd -= bufferStart;
+ bufferStart = 0;
+
+ // fill in remaining space
+ if (!exhausted) {
+ // TODO: refactor to a shared readFully
+ while (bufferEnd < buffer.length) {
+ final int read = input.read(buffer, bufferEnd, buffer.length - bufferEnd);
+ if (read == -1) {
+ exhausted = true;
break;
}
- charsRead += inc;
+ bufferEnd += read;
}
}
+ }
- inLen = inStr.length();
- if (inLen == 0) {
- return false;
- }
+ // should we go to the next offset?
+ if (gramSize > maxGram || bufferStart + gramSize > bufferEnd) {
+ bufferStart++;
+ offset++;
+ gramSize = minGram;
}
- if (pos+gramSize > inLen) { // if we hit the end of the string
- pos = 0; // reset to beginning of string
- gramSize++; // increase n-gram size
- if (gramSize > maxGram) // we are done
- return false;
- if (pos+gramSize > inLen)
- return false;
+ // are there enough chars remaining?
+ if (bufferStart + gramSize > bufferEnd) {
+ return false;
}
- int oldPos = pos;
- pos++;
- termAtt.setEmpty().append(inStr, oldPos, oldPos+gramSize);
- offsetAtt.setOffset(correctOffset(oldPos), correctOffset(oldPos+gramSize));
+ termAtt.copyBuffer(buffer, bufferStart, gramSize);
+ posIncAtt.setPositionIncrement(1);
+ posLenAtt.setPositionLength(1);
+ offsetAtt.setOffset(correctOffset(offset), correctOffset(offset + gramSize));
+ ++gramSize;
return true;
}
-
+
@Override
public void end() {
- // set final offset
- final int finalOffset = correctOffset(charsRead);
- this.offsetAtt.setOffset(finalOffset, finalOffset);
- }
-
+ final int endOffset = correctOffset(offset + bufferEnd - bufferStart);
+ offsetAtt.setOffset(endOffset, endOffset);
+ }
+
@Override
public void reset() throws IOException {
super.reset();
- started = false;
- pos = 0;
+ bufferStart = bufferEnd = buffer.length;
+ offset = 0;
+ gramSize = minGram;
+ exhausted = false;
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java Tue May 7 11:20:55 2013
@@ -18,40 +18,44 @@ package org.apache.lucene.analysis.ngram
*/
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.util.Version;
import java.io.Reader;
import java.util.Map;
/**
* Factory for {@link NGramTokenizer}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_ngrm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.NGramTokenizerFactory" minGramSize="1" maxGramSize="2"/>
* </analyzer>
- * </fieldType></pre>
- *
+ * </fieldType></pre>
*/
public class NGramTokenizerFactory extends TokenizerFactory {
- private int maxGramSize = 0;
- private int minGramSize = 0;
-
- /** Initializes the n-gram min and max sizes and the side from which one should start tokenizing. */
- @Override
- public void init(Map<String, String> args) {
- super.init(args);
- String maxArg = args.get("maxGramSize");
- maxGramSize = (maxArg != null ? Integer.parseInt(maxArg) : NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
-
- String minArg = args.get("minGramSize");
- minGramSize = (minArg != null ? Integer.parseInt(minArg) : NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
+ private final int maxGramSize;
+ private final int minGramSize;
+
+ /** Creates a new NGramTokenizerFactory */
+ public NGramTokenizerFactory(Map<String, String> args) {
+ super(args);
+ minGramSize = getInt(args, "minGramSize", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
+ maxGramSize = getInt(args, "maxGramSize", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
}
/** Creates the {@link TokenStream} of n-grams from the given {@link Reader} and {@link AttributeFactory}. */
@Override
- public NGramTokenizer create(AttributeFactory factory, Reader input) {
- return new NGramTokenizer(factory, input, minGramSize, maxGramSize);
+ public Tokenizer create(AttributeFactory factory, Reader input) {
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_44)) {
+ return new NGramTokenizer(luceneMatchVersion, factory, input, minGramSize, maxGramSize);
+ } else {
+ return new Lucene43NGramTokenizer(factory, input, minGramSize, maxGramSize);
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Tue May 7 11:20:55 2013
@@ -21,6 +21,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter.StemmerOverrideMap;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter;
@@ -30,9 +31,13 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.analysis.standard.StandardAnalyzer; // for javadoc
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.analysis.util.WordlistLoader;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util.fst.FST;
import java.io.IOException;
import java.io.Reader;
@@ -96,7 +101,7 @@ public final class DutchAnalyzer extends
*/
private CharArraySet excltable = CharArraySet.EMPTY_SET;
- private final CharArrayMap<String> stemdict;
+ private final StemmerOverrideMap stemdict;
private final Version matchVersion;
/**
@@ -120,7 +125,24 @@ public final class DutchAnalyzer extends
this.matchVersion = matchVersion;
this.stoptable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stopwords));
this.excltable = CharArraySet.unmodifiableSet(CharArraySet.copy(matchVersion, stemExclusionTable));
- this.stemdict = CharArrayMap.unmodifiableMap(CharArrayMap.copy(matchVersion, stemOverrideDict));
+ if (stemOverrideDict.isEmpty()) {
+ this.stemdict = null;
+ } else {
+ // we don't need to ignore case here since we lowercase in this analyzer anyway
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(false);
+ CharArrayMap<String>.EntryIterator iter = stemOverrideDict.entrySet().iterator();
+ CharsRef spare = new CharsRef();
+ while (iter.hasNext()) {
+ char[] nextKey = iter.nextKey();
+ spare.copyChars(nextKey, 0, nextKey.length);
+ builder.add(spare, iter.currentValue());
+ }
+ try {
+ this.stemdict = builder.build();
+ } catch (IOException ex) {
+ throw new RuntimeException("can not build stem dict", ex);
+ }
+ }
}
/**
@@ -141,7 +163,7 @@ public final class DutchAnalyzer extends
result = new StopFilter(matchVersion, result, stoptable);
if (!excltable.isEmpty())
result = new SetKeywordMarkerFilter(result, excltable);
- if (!stemdict.isEmpty())
+ if (stemdict != null)
result = new StemmerOverrideFilter(result, stemdict);
result = new SnowballFilter(result, new org.tartarus.snowball.ext.DutchStemmer());
return new TokenStreamComponents(source, result);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianLightStemFilterFactory.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianLightStemFilterFactory.java Tue May 7 11:20:55 2013
@@ -17,13 +17,15 @@ package org.apache.lucene.analysis.no;
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.no.NorwegianLightStemFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link NorwegianLightStemFilter}.
- * <pre class="prettyprint" >
+ * <pre class="prettyprint">
* <fieldType name="text_svlgtstem" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.StandardTokenizerFactory"/>
@@ -33,6 +35,15 @@ import org.apache.lucene.analysis.util.T
* </fieldType></pre>
*/
public class NorwegianLightStemFilterFactory extends TokenFilterFactory {
+
+ /** Creates a new NorwegianLightStemFilterFactory */
+ public NorwegianLightStemFilterFactory(Map<String,String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
@Override
public TokenStream create(TokenStream input) {
return new NorwegianLightStemFilter(input);