You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/05/04 20:18:09 UTC
svn commit: r1479148 [1/2] - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscel...
Author: jpountz
Date: Sat May 4 18:18:08 2013
New Revision: 1479148
URL: http://svn.apache.org/r1479148
Log:
LUCENE-4963: Deprecate broken TokenFilter options.
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java
lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/package.html
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
lucene/dev/trunk/lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
lucene/dev/trunk/lucene/queryparser/src/test/org/apache/lucene/queryparser/xml/TestParser.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenFilter.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
lucene/dev/trunk/solr/example/example-DIH/solr/mail/conf/schema.xml
lucene/dev/trunk/solr/example/solr/collection1/conf/schema.xml
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sat May 4 18:18:08 2013
@@ -59,6 +59,16 @@ Changes in backwards compatibility polic
completely refactored to allow for a better implementation of TimSort.
(Adrien Grand, Uwe Schindler, Dawid Weiss)
+* LUCENE-4963: Some TokenFilter options that generate broken TokenStreams have
+ been deprecated: updateOffsets=true on TrimFilter and
+ enablePositionIncrements=false on all classes that inherit from
+ FilteringTokenFilter: JapanesePartOfSpeechStopFilter, KeepWordFilter,
+ LengthFilter, StopFilter and TypeTokenFilter. (Adrien Grand)
+
+* LUCENE-4963: In order not to take position increments into account in
+ suggesters, you now need to call setPreservePositionIncrements(false) instead
+ of configuring the token filters to not increment positions. (Adrien Grand)
+
Bug Fixes
* LUCENE-4935: CustomScoreQuery wrongly applied its query boost twice
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilter.java Sat May 4 18:18:08 2013
@@ -57,7 +57,7 @@ public final class StopFilter extends Fi
* @see #makeStopSet(Version, java.lang.String...)
*/
public StopFilter(Version matchVersion, TokenStream in, CharArraySet stopWords) {
- super(true, in);
+ super(matchVersion, in);
this.stopWords = stopWords;
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java Sat May 4 18:18:08 2013
@@ -51,7 +51,7 @@ public class StopFilterFactory extends T
stopWordFiles = get(args, "words");
format = get(args, "format");
ignoreCase = getBoolean(args, "ignoreCase", false);
- enablePositionIncrements = getBoolean(args, "enablePositionIncrements", false);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", true);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java Sat May 4 18:18:08 2013
@@ -17,12 +17,12 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
+import java.util.Set;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
-
-import java.io.IOException;
-import java.util.Set;
+import org.apache.lucene.util.Version;
/**
* Removes tokens whose types appear in a set of blocked types from a token stream.
@@ -33,14 +33,41 @@ public final class TypeTokenFilter exten
private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class);
private final boolean useWhiteList;
- public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
- super(enablePositionIncrements, input);
+ /** @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4. */
+ @Deprecated
+ public TypeTokenFilter(Version version, boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
+ super(version, enablePositionIncrements, input);
+ this.stopTypes = stopTypes;
+ this.useWhiteList = useWhiteList;
+ }
+
+ /** @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4. */
+ @Deprecated
+ public TypeTokenFilter(Version version, boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes) {
+ this(version, enablePositionIncrements, input, stopTypes, false);
+ }
+
+ /**
+ * Create a new {@link TypeTokenFilter}.
+ * @param version the Lucene match version
+ * @param input the {@link TokenStream} to consume
+ * @param stopTypes the types to filter
+ * @param useWhiteList if true, then tokens whose type is in stopTypes will
+ * be kept, otherwise they will be filtered out
+ */
+ public TypeTokenFilter(Version version, TokenStream input, Set<String> stopTypes, boolean useWhiteList) {
+ super(version, input);
this.stopTypes = stopTypes;
this.useWhiteList = useWhiteList;
}
- public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTypes) {
- this(enablePositionIncrements, input, stopTypes, false);
+ /**
+ * Create a new {@link TypeTokenFilter} that filters tokens out
+ * (useWhiteList=false).
+ * @see #TypeTokenFilter(Version, TokenStream, Set, boolean)
+ */
+ public TypeTokenFilter(Version version, TokenStream input, Set<String> stopTypes) {
+ this(version, input, stopTypes, false);
}
/**
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilterFactory.java Sat May 4 18:18:08 2013
@@ -35,7 +35,7 @@ import java.util.Set;
* <analyzer>
* <tokenizer class="solr.StandardTokenizerFactory"/>
* <filter class="solr.TypeTokenFilterFactory" types="stoptypes.txt"
- * enablePositionIncrements="true" useWhitelist="false"/>
+ * useWhitelist="false"/>
* </analyzer>
* </fieldType></pre>
*/
@@ -49,7 +49,7 @@ public class TypeTokenFilterFactory exte
public TypeTokenFilterFactory(Map<String,String> args) {
super(args);
stopTypesFiles = require(args, "types");
- enablePositionIncrements = getBoolean(args, "enablePositionIncrements", false);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", true);
useWhitelist = getBoolean(args, "useWhitelist", false);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@@ -78,6 +78,8 @@ public class TypeTokenFilterFactory exte
@Override
public TokenStream create(TokenStream input) {
- return new TypeTokenFilter(enablePositionIncrements, input, stopTypes, useWhitelist);
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new TypeTokenFilter(luceneMatchVersion, enablePositionIncrements, input, stopTypes, useWhitelist);
+ return filter;
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java Sat May 4 18:18:08 2013
@@ -138,7 +138,9 @@ public final class IrishAnalyzer extends
final Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
StopFilter s = new StopFilter(matchVersion, result, HYPHENATIONS);
- s.setEnablePositionIncrements(false);
+ if (!matchVersion.onOrAfter(Version.LUCENE_44)) {
+ s.setEnablePositionIncrements(false);
+ }
result = s;
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new IrishLowerCaseFilter(result);
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilter.java Sat May 4 18:18:08 2013
@@ -21,6 +21,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.Version;
/**
* A TokenFilter that only keeps tokens with text contained in the
@@ -32,10 +33,23 @@ public final class KeepWordFilter extend
private final CharArraySet words;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- /** The words set passed to this constructor will be directly used by this filter
- * and should not be modified, */
- public KeepWordFilter(boolean enablePositionIncrements, TokenStream in, CharArraySet words) {
- super(enablePositionIncrements, in);
+ /** @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4. */
+ @Deprecated
+ public KeepWordFilter(Version version, boolean enablePositionIncrements, TokenStream in, CharArraySet words) {
+ super(version, enablePositionIncrements, in);
+ this.words = words;
+ }
+
+ /**
+ * Create a new {@link KeepWordFilter}.
+ * <p><b>NOTE</b>: The words set passed to this constructor will be directly
+ * used by this filter and should not be modified.
+ * @param version the Lucene match version
+ * @param in the {@link TokenStream} to consume
+ * @param words the words to keep
+ */
+ public KeepWordFilter(Version version, TokenStream in, CharArraySet words) {
+ super(version, in);
this.words = words;
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/KeepWordFilterFactory.java Sat May 4 18:18:08 2013
@@ -32,7 +32,7 @@ import java.io.IOException;
* <fieldType name="text_keepword" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.KeepWordFilterFactory" words="keepwords.txt" ignoreCase="false" enablePositionIncrements="false"/>
+ * <filter class="solr.KeepWordFilterFactory" words="keepwords.txt" ignoreCase="false"/>
* </analyzer>
* </fieldType></pre>
*/
@@ -48,7 +48,7 @@ public class KeepWordFilterFactory exten
assureMatchVersion();
wordFiles = get(args, "words");
ignoreCase = getBoolean(args, "ignoreCase", false);
- enablePositionIncrements = getBoolean(args, "enablePositionIncrements", false);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", true);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -76,6 +76,12 @@ public class KeepWordFilterFactory exten
@Override
public TokenStream create(TokenStream input) {
// if the set is null, it means it was empty
- return words == null ? input : new KeepWordFilter(enablePositionIncrements, input, words);
+ if (words == null) {
+ return input;
+ } else {
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new KeepWordFilter(luceneMatchVersion, enablePositionIncrements, input, words);
+ return filter;
+ }
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java Sat May 4 18:18:08 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.misce
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
/**
* Removes words that are too long or too short from the stream.
@@ -34,16 +35,29 @@ public final class LengthFilter extends
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ /** @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4. */
+ @Deprecated
+ public LengthFilter(Version version, boolean enablePositionIncrements, TokenStream in, int min, int max) {
+ super(version, enablePositionIncrements, in);
+ this.min = min;
+ this.max = max;
+ }
+
/**
- * Build a filter that removes words that are too long or too
- * short from the text.
+ * Create a new {@link LengthFilter}. This will filter out tokens whose
+ * {@link CharTermAttribute} is either too short ({@link CharTermAttribute#length()}
+ * < min) or too long ({@link CharTermAttribute#length()} > max).
+ * @param version the Lucene match version
+ * @param in the {@link TokenStream} to consume
+ * @param min the minimum length
+ * @param max the maximum length
*/
- public LengthFilter(boolean enablePositionIncrements, TokenStream in, int min, int max) {
- super(enablePositionIncrements, in);
+ public LengthFilter(Version version, TokenStream in, int min, int max) {
+ super(version, in);
this.min = min;
this.max = max;
}
-
+
@Override
public boolean accept() {
final int len = termAtt.length();
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilterFactory.java Sat May 4 18:18:08 2013
@@ -17,18 +17,18 @@ package org.apache.lucene.analysis.misce
* limitations under the License.
*/
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Map;
-
/**
* Factory for {@link LengthFilter}.
* <pre class="prettyprint">
* <fieldType name="text_lngth" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.LengthFilterFactory" min="0" max="1" enablePositionIncrements="false"/>
+ * <filter class="solr.LengthFilterFactory" min="0" max="1" />
* </analyzer>
* </fieldType></pre>
*/
@@ -44,7 +44,7 @@ public class LengthFilterFactory extends
super(args);
min = requireInt(args, MIN_KEY);
max = requireInt(args, MAX_KEY);
- enablePositionIncrements = getBoolean(args, "enablePositionIncrements", false);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", true);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -52,6 +52,8 @@ public class LengthFilterFactory extends
@Override
public LengthFilter create(TokenStream input) {
- return new LengthFilter(enablePositionIncrements, input,min,max);
+ @SuppressWarnings("deprecation")
+ final LengthFilter filter = new LengthFilter(luceneMatchVersion, enablePositionIncrements, input,min,max);
+ return filter;
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilter.java Sat May 4 18:18:08 2013
@@ -21,11 +21,14 @@ import org.apache.lucene.analysis.TokenF
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util.Version;
import java.io.IOException;
/**
* Trims leading and trailing whitespace from Tokens in the stream.
+ * <p>As of Lucene 4.4, this filter does not support updateOffsets=true anymore
+ * as it can lead to broken token streams.
*/
public final class TrimFilter extends TokenFilter {
@@ -33,12 +36,27 @@ public final class TrimFilter extends To
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
-
- public TrimFilter(TokenStream in, boolean updateOffsets) {
+ /**
+ * Create a new {@link TrimFilter}.
+ * @param version the Lucene match version
+ * @param in the stream to consume
+ * @param updateOffsets whether to update offsets
+ * @deprecated Offset updates are not supported anymore as of Lucene 4.4.
+ */
+ @Deprecated
+ public TrimFilter(Version version, TokenStream in, boolean updateOffsets) {
super(in);
+ if (updateOffsets && version.onOrAfter(Version.LUCENE_44)) {
+ throw new IllegalArgumentException("updateOffsets=true is not supported anymore as of Lucene 4.4");
+ }
this.updateOffsets = updateOffsets;
}
+ /** Create a new {@link TrimFilter} on top of <code>in</code>. */
+ public TrimFilter(Version version, TokenStream in) {
+ this(version, in, false);
+ }
+
@Override
public boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
@@ -55,11 +73,10 @@ public final class TrimFilter extends To
int endOff = 0;
// eat the first characters
- //QUESTION: Should we use Character.isWhitespace() instead?
- for (start = 0; start < len && termBuffer[start] <= ' '; start++) {
+ for (start = 0; start < len && Character.isWhitespace(termBuffer[start]); start++) {
}
// eat the end characters
- for (end = len; end >= start && termBuffer[end - 1] <= ' '; end--) {
+ for (end = len; end >= start && Character.isWhitespace(termBuffer[end - 1]); end--) {
endOff++;
}
if (start > 0 || end < len) {
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java Sat May 4 18:18:08 2013
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.util.T
* <fieldType name="text_trm" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.NGramTokenizerFactory"/>
- * <filter class="solr.TrimFilterFactory" updateOffsets="false"/>
+ * <filter class="solr.TrimFilterFactory" />
* </analyzer>
* </fieldType></pre>
*
@@ -50,6 +50,8 @@ public class TrimFilterFactory extends T
@Override
public TrimFilter create(TokenStream input) {
- return new TrimFilter(input, updateOffsets);
+ @SuppressWarnings("deprecation")
+ final TrimFilter filter = new TrimFilter(luceneMatchVersion, input, updateOffsets);
+ return filter;
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Sat May 4 18:18:08 2013
@@ -73,7 +73,7 @@ public final class NGramTokenFilter exte
* @param maxGram the largest n-gram to generate
*/
public NGramTokenFilter(Version version, TokenStream input, int minGram, int maxGram) {
- super(new LengthFilter(true, input, minGram, Integer.MAX_VALUE));
+ super(new LengthFilter(version, input, minGram, Integer.MAX_VALUE));
this.version = version;
if (minGram < 1) {
throw new IllegalArgumentException("minGram must be greater than zero");
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilteringTokenFilter.java Sat May 4 18:18:08 2013
@@ -22,24 +22,54 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.Version;
/**
* Abstract base class for TokenFilters that may remove tokens.
* You have to implement {@link #accept} and return a boolean if the current
* token should be preserved. {@link #incrementToken} uses this method
* to decide if a token should be passed to the caller.
+ * <p><a name="version" />As of Lucene 4.4, an {@link IllegalArgumentException}
+ * is thrown when trying to disable position increments when filtering terms.
*/
public abstract class FilteringTokenFilter extends TokenFilter {
+ private static void checkPositionIncrement(Version version, boolean enablePositionIncrements) {
+ if (!enablePositionIncrements && version.onOrAfter(Version.LUCENE_44)) {
+ throw new IllegalArgumentException("enablePositionIncrements=false is not supported anymore as of Lucene 4.4 as it can create broken token streams");
+ }
+ }
+
+ protected final Version version;
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private boolean enablePositionIncrements; // no init needed, as ctor enforces setting value!
- private boolean first = true; // only used when not preserving gaps
+ private boolean first = true;
- public FilteringTokenFilter(boolean enablePositionIncrements, TokenStream input){
- super(input);
+ /**
+ * Create a new {@link FilteringTokenFilter}.
+ * @param version the Lucene match <a href="#version">version</a>
+ * @param enablePositionIncrements whether to increment position increments when filtering out terms
+ * @param input the input to consume
+ * @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4
+ */
+ @Deprecated
+ public FilteringTokenFilter(Version version, boolean enablePositionIncrements, TokenStream input){
+ this(version, input);
+ checkPositionIncrement(version, enablePositionIncrements);
this.enablePositionIncrements = enablePositionIncrements;
}
+ /**
+ * Create a new {@link FilteringTokenFilter}.
+ * @param version the Lucene match version
+ * @param in the {@link TokenStream} to consume
+ */
+ public FilteringTokenFilter(Version version, TokenStream in) {
+ super(in);
+ this.version = version;
+ this.enablePositionIncrements = true;
+ }
+
/** Override this method and return if the current input token should be returned by {@link #incrementToken}. */
protected abstract boolean accept() throws IOException;
@@ -102,8 +132,11 @@ public abstract class FilteringTokenFilt
* <p> <b>NOTE</b>: be sure to also
* set org.apache.lucene.queryparser.classic.QueryParser#setEnablePositionIncrements if
* you use QueryParser to create queries.
+ * @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4
*/
+ @Deprecated
public void setEnablePositionIncrements(boolean enable) {
+ checkPositionIncrement(version, enable);
this.enablePositionIncrements = enable;
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Sat May 4 18:18:08 2013
@@ -161,8 +161,6 @@ public class TestRandomChains extends Ba
// startOffset thats > its endOffset
// (see LUCENE-3738 for a list of other offenders here)
// broken!
- Lucene43NGramTokenizer.class,
- // broken!
EdgeNGramTokenizer.class,
// broken!
EdgeNGramTokenFilter.class,
@@ -182,55 +180,6 @@ public class TestRandomChains extends Ba
private static final Map<Constructor<?>,Predicate<Object[]>> brokenOffsetsConstructors = new HashMap<Constructor<?>, Predicate<Object[]>>();
static {
try {
- brokenOffsetsConstructors.put(
- TrimFilter.class.getConstructor(TokenStream.class, boolean.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 2;
- return (Boolean) args[1]; // args are broken if updateOffsets is true
- }
- });
- brokenOffsetsConstructors.put(
- TypeTokenFilter.class.getConstructor(boolean.class, TokenStream.class, Set.class, boolean.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 4;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- TypeTokenFilter.class.getConstructor(boolean.class, TokenStream.class, Set.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 3;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- LengthFilter.class.getConstructor(boolean.class, TokenStream.class, int.class, int.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 4;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
- brokenOffsetsConstructors.put(
- KeepWordFilter.class.getConstructor(boolean.class, TokenStream.class, CharArraySet.class),
- new Predicate<Object[]>() {
- @Override
- public boolean apply(Object[] args) {
- assert args.length == 3;
- // LUCENE-4065: only if you pass 'false' to enablePositionIncrements!
- return !(Boolean) args[0];
- }
- });
for (Class<?> c : Arrays.<Class<?>>asList(
ReversePathHierarchyTokenizer.class,
PathHierarchyTokenizer.class,
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java Sat May 4 18:18:08 2013
@@ -75,7 +75,7 @@ public class TestStopFilter extends Base
doTestStopPositons(stpf,true);
// without increments
reader = new StringReader(sb.toString());
- stpf = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
+ stpf = new StopFilter(Version.LUCENE_43, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
doTestStopPositons(stpf,false);
// with increments, concatenating two stop filters
ArrayList<String> a0 = new ArrayList<String>();
@@ -166,7 +166,7 @@ public class TestStopFilter extends Base
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenFilter filter = new MockSynonymFilter(tokenizer);
- StopFilter stopfilter = new StopFilter(TEST_VERSION_CURRENT, filter, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+ StopFilter stopfilter = new StopFilter(Version.LUCENE_43, filter, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
stopfilter.setEnablePositionIncrements(false);
return new TokenStreamComponents(tokenizer, stopfilter);
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java Sat May 4 18:18:08 2013
@@ -24,6 +24,7 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.English;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.StringReader;
@@ -36,7 +37,7 @@ public class TestTypeTokenFilter extends
public void testTypeFilter() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = asSet("<NUM>");
- TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
assertTokenStreamContents(stream, new String[]{"is", "palindrome", "while", "is", "not"});
}
@@ -59,12 +60,12 @@ public class TestTypeTokenFilter extends
// with increments
StringReader reader = new StringReader(sb.toString());
- TypeTokenFilter typeTokenFilter = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
+ TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
testPositons(typeTokenFilter);
// without increments
reader = new StringReader(sb.toString());
- typeTokenFilter = new TypeTokenFilter(false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
+ typeTokenFilter = new TypeTokenFilter(Version.LUCENE_43, false, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
testPositons(typeTokenFilter);
}
@@ -87,7 +88,7 @@ public class TestTypeTokenFilter extends
public void testTypeFilterWhitelist() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set<String> stopTypes = Collections.singleton("<NUM>");
- TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
assertTokenStreamContents(stream, new String[]{"121", "123"});
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilterFactory.java Sat May 4 18:18:08 2013
@@ -50,7 +50,7 @@ public class TestTypeTokenFilterFactory
public void testCreationWithBlackList() throws Exception {
TokenFilterFactory factory = tokenFilterFactory("Type",
"types", "stoptypes-1.txt, stoptypes-2.txt",
- "enablePositionIncrements", "false");
+ "enablePositionIncrements", "true");
NumericTokenStream input = new NumericTokenStream();
input.setIntValue(123);
factory.create(input);
@@ -59,7 +59,7 @@ public class TestTypeTokenFilterFactory
public void testCreationWithWhiteList() throws Exception {
TokenFilterFactory factory = tokenFilterFactory("Type",
"types", "stoptypes-1.txt, stoptypes-2.txt",
- "enablePositionIncrements", "false",
+ "enablePositionIncrements", "true",
"useWhitelist", "true");
NumericTokenStream input = new NumericTokenStream();
input.setIntValue(123);
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishAnalyzer.java Sat May 4 18:18:08 2013
@@ -61,7 +61,7 @@ public class TestIrishAnalyzer extends B
Analyzer a = new IrishAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(a, "n-athair",
new String[] { "athair" },
- new int[] { 1 });
+ new int[] { 2 });
}
/** blast some random strings through the analyzer */
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java Sat May 4 18:18:08 2013
@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.Version;
/** Test {@link KeepWordFilter} */
public class TestKeepWordFilter extends BaseTokenStreamTestCase {
@@ -42,22 +43,22 @@ public class TestKeepWordFilter extends
// Test Stopwords
TokenStream stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- stream = new KeepWordFilter(true, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
+ stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
assertTokenStreamContents(stream, new String[] { "aaa", "BBB" }, new int[] { 3, 2 });
// Now force case
stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- stream = new KeepWordFilter(true, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
+ stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
assertTokenStreamContents(stream, new String[] { "aaa" }, new int[] { 3 });
// Test Stopwords
stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- stream = new KeepWordFilter(false, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
+ stream = new KeepWordFilter(Version.LUCENE_43, false, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
assertTokenStreamContents(stream, new String[] { "aaa", "BBB" }, new int[] { 1, 1 });
// Now force case
stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- stream = new KeepWordFilter(false, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
+ stream = new KeepWordFilter(Version.LUCENE_43, false, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
assertTokenStreamContents(stream, new String[] { "aaa" }, new int[] { 1 });
}
@@ -72,7 +73,7 @@ public class TestKeepWordFilter extends
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream stream = new KeepWordFilter(true, tokenizer, new CharArraySet(TEST_VERSION_CURRENT, words, true));
+ TokenStream stream = new KeepWordFilter(TEST_VERSION_CURRENT, tokenizer, new CharArraySet(TEST_VERSION_CURRENT, words, true));
return new TokenStreamComponents(tokenizer, stream);
}
};
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java Sat May 4 18:18:08 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.analysis.misce
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -29,7 +30,7 @@ public class TestLengthFilter extends Ba
public void testFilterNoPosIncr() throws Exception {
TokenStream stream = new MockTokenizer(
new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
- LengthFilter filter = new LengthFilter(false, stream, 2, 6);
+ LengthFilter filter = new LengthFilter(Version.LUCENE_43, false, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
new int[]{1, 1, 1}
@@ -39,7 +40,7 @@ public class TestLengthFilter extends Ba
public void testFilterWithPosIncr() throws Exception {
TokenStream stream = new MockTokenizer(
new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
- LengthFilter filter = new LengthFilter(true, stream, 2, 6);
+ LengthFilter filter = new LengthFilter(TEST_VERSION_CURRENT, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
new int[]{1, 4, 2}
@@ -51,7 +52,7 @@ public class TestLengthFilter extends Ba
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
- return new TokenStreamComponents(tokenizer, new LengthFilter(true, tokenizer, 0, 5));
+ return new TokenStreamComponents(tokenizer, new LengthFilter(TEST_VERSION_CURRENT, tokenizer, 0, 5));
}
};
checkOneTermReuse(a, "", "");
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java Sat May 4 18:18:08 2013
@@ -22,6 +22,8 @@ import java.io.StringReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.util.ClasspathResourceLoader;
+import org.apache.lucene.util.Version;
public class TestLengthFilterFactory extends BaseTokenStreamFactoryTestCase {
@@ -29,8 +31,10 @@ public class TestLengthFilterFactory ext
Reader reader = new StringReader("foo foobar super-duper-trooper");
TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
stream = tokenFilterFactory("Length",
+ Version.LUCENE_43, new ClasspathResourceLoader(getClass()),
"min", "4",
- "max", "10").create(stream);
+ "max", "10",
+ "enablePositionIncrements", "false").create(stream);
assertTokenStreamContents(stream, new String[] { "foobar" }, new int[] { 1 });
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java Sat May 4 18:18:08 2013
@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.util.Version;
/**
*/
@@ -46,7 +47,7 @@ public class TestTrimFilter extends Base
new Token(ccc, 0, ccc.length, 11, 15),
new Token(whitespace, 0, whitespace.length, 16, 20),
new Token(empty, 0, empty.length, 21, 21));
- ts = new TrimFilter(ts, false);
+ ts = new TrimFilter(TEST_VERSION_CURRENT, ts, false);
assertTokenStreamContents(ts, new String[] { "a", "b", "cCc", "", ""});
@@ -59,7 +60,7 @@ public class TestTrimFilter extends Base
new Token(b, 0, b.length, 0, 2),
new Token(ccc, 0, ccc.length, 0, 3),
new Token(whitespace, 0, whitespace.length, 0, 3));
- ts = new TrimFilter(ts, true);
+ ts = new TrimFilter(Version.LUCENE_43, ts, true);
assertTokenStreamContents(ts,
new String[] { "a", "b", "c", "" },
@@ -120,7 +121,7 @@ public class TestTrimFilter extends Base
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
- return new TokenStreamComponents(tokenizer, new TrimFilter(tokenizer, false));
+ return new TokenStreamComponents(tokenizer, new TrimFilter(Version.LUCENE_43, tokenizer, true));
}
};
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
@@ -130,7 +131,7 @@ public class TestTrimFilter extends Base
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
- return new TokenStreamComponents(tokenizer, new TrimFilter(tokenizer, true));
+ return new TokenStreamComponents(tokenizer, new TrimFilter(TEST_VERSION_CURRENT, tokenizer, false));
}
};
checkRandomData(random(), b, 1000*RANDOM_MULTIPLIER);
@@ -141,7 +142,9 @@ public class TestTrimFilter extends Base
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
- return new TokenStreamComponents(tokenizer, new TrimFilter(tokenizer, random().nextBoolean()));
+ final boolean updateOffsets = random().nextBoolean();
+ final Version version = updateOffsets ? Version.LUCENE_43 : TEST_VERSION_CURRENT;
+ return new TokenStreamComponents(tokenizer, new TrimFilter(version, tokenizer, updateOffsets));
}
};
checkOneTermReuse(a, "", "");
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java Sat May 4 18:18:08 2013
@@ -306,7 +306,6 @@ public class TestWordDelimiterFilter ext
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
StopFilter filter = new StopFilter(TEST_VERSION_CURRENT,
tokenizer, StandardAnalyzer.STOP_WORDS_SET);
- filter.setEnablePositionIncrements(true);
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(filter, flags, protWords));
}
};
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java Sat May 4 18:18:08 2013
@@ -89,7 +89,7 @@ public class JapaneseAnalyzer extends St
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
- stream = new JapanesePartOfSpeechStopFilter(true, stream, stoptags);
+ stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
stream = new CJKWidthFilter(stream);
stream = new StopFilter(matchVersion, stream, stopwords);
stream = new JapaneseKatakanaStemFilter(stream);
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java Sat May 4 18:18:08 2013
@@ -22,6 +22,7 @@ import java.util.Set;
import org.apache.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Version;
/**
* Removes tokens that match a set of part-of-speech tags.
@@ -30,8 +31,21 @@ public final class JapanesePartOfSpeechS
private final Set<String> stopTags;
private final PartOfSpeechAttribute posAtt = addAttribute(PartOfSpeechAttribute.class);
- public JapanesePartOfSpeechStopFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTags) {
- super(enablePositionIncrements, input);
+ /** @deprecated enablePositionIncrements=false is not supported anymore as of Lucene 4.4. */
+ @Deprecated
+ public JapanesePartOfSpeechStopFilter(Version version, boolean enablePositionIncrements, TokenStream input, Set<String> stopTags) {
+ super(version, enablePositionIncrements, input);
+ this.stopTags = stopTags;
+ }
+
+ /**
+ * Create a new {@link JapanesePartOfSpeechStopFilter}.
+ * @param version the Lucene match version
+ * @param input the {@link TokenStream} to consume
+ * @param stopTags the part-of-speech tags that should be removed
+ */
+ public JapanesePartOfSpeechStopFilter(Version version, TokenStream input, Set<String> stopTags) {
+ super(version, input);
this.stopTags = stopTags;
}
Modified: lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilterFactory.java Sat May 4 18:18:08 2013
@@ -50,7 +50,7 @@ public class JapanesePartOfSpeechStopFil
public JapanesePartOfSpeechStopFilterFactory(Map<String,String> args) {
super(args);
stopTagFiles = get(args, "tags");
- enablePositionIncrements = getBoolean(args, "enablePositionIncrements", false);
+ enablePositionIncrements = getBoolean(args, "enablePositionIncrements", true);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -72,6 +72,12 @@ public class JapanesePartOfSpeechStopFil
@Override
public TokenStream create(TokenStream stream) {
// if stoptags is null, it means the file is empty
- return stopTags == null ? stream : new JapanesePartOfSpeechStopFilter(enablePositionIncrements, stream, stopTags);
+ if (stopTags != null) {
+ @SuppressWarnings("deprecation")
+ final TokenStream filter = new JapanesePartOfSpeechStopFilter(luceneMatchVersion, enablePositionIncrements, stream, stopTags);
+ return filter;
+ } else {
+ return stream;
+ }
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java Sat May 4 18:18:08 2013
@@ -17,12 +17,8 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
@@ -43,8 +39,16 @@ import org.apache.lucene.util.automaton.
* @lucene.experimental */
public class TokenStreamToAutomaton {
+ private boolean preservePositionIncrements;
+
/** Sole constructor. */
public TokenStreamToAutomaton() {
+ this.preservePositionIncrements = true;
+ }
+
+ /** Whether to generate holes in the automaton for missing positions, <code>true</code> by default. */
+ public void setPreservePositionIncrements(boolean enablePositionIncrements) {
+ this.preservePositionIncrements = enablePositionIncrements;
}
private static class Position implements RollingBuffer.Resettable {
@@ -108,6 +112,9 @@ public class TokenStreamToAutomaton {
int maxOffset = 0;
while (in.incrementToken()) {
int posInc = posIncAtt.getPositionIncrement();
+ if (!preservePositionIncrements && posInc > 1) {
+ posInc = 1;
+ }
assert pos > -1 || posInc > 0;
if (posInc > 0) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/package.html?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/package.html (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/package.html Sat May 4 18:18:08 2013
@@ -282,18 +282,18 @@ and proximity searches (though sentence
<p>
If the selected analyzer filters the stop words "is" and "the", then for a document
containing the string "blue is the sky", only the tokens "blue", "sky" are indexed,
- with position("sky") = 1 + position("blue"). Now, a phrase query "blue is the sky"
+ with position("sky") = 3 + position("blue"). Now, a phrase query "blue is the sky"
would find that document, because the same analyzer filters the same stop words from
- that query. But also the phrase query "blue sky" would find that document.
+ that query. But the phrase query "blue sky" would not find that document because the
+ position increment between "blue" and "sky" is only 1.
</p>
<p>
- If this behavior does not fit the application needs, a modified analyzer can
- be used, that would increment further the positions of tokens following a
- removed stop word, using
- {@link org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute#setPositionIncrement(int)}.
- This can be done with something like the following (note, however, that
- StopFilter natively includes this capability by subclassing
- FilteringTokenFilter}:
+ If this behavior does not fit the application needs, the query parser needs to be
+ configured to not take position increments into account when generating phrase queries.
+</p>
+<p>
+ Note that a StopFilter MUST increment the position increment in order not to generate corrupt
+ tokenstream graphs. Here is the logic used by StopFilter to increment positions when filtering out tokens:
</p>
<PRE class="prettyprint">
public TokenStream tokenStream(final String fieldName, Reader reader) {
@@ -308,7 +308,7 @@ and proximity searches (though sentence
boolean hasNext = ts.incrementToken();
if (hasNext) {
if (stopWords.contains(termAtt.toString())) {
- extraIncrement++; // filter this word
+ extraIncrement += posIncrAtt.getPositionIncrement(); // filter this word
continue;
}
if (extraIncrement>0) {
@@ -323,11 +323,6 @@ and proximity searches (though sentence
}
</PRE>
<p>
- Now, with this modified analyzer, the phrase query "blue sky" would find that document.
- But note that this is yet not a perfect solution, because any phrase query "blue w1 w2 sky"
- where both w1 and w2 are stop words would match that document.
-</p>
-<p>
A few more use cases for modifying position increments are:
</p>
<ol>
@@ -338,6 +333,72 @@ and proximity searches (though sentence
As result, all synonyms of a token would be considered to appear in exactly the
same position as that token, and so would they be seen by phrase and proximity searches.</li>
</ol>
+
+<h3>Token Position Length</h3>
+<p>
+ By default, all tokens created by Analyzers and Tokenizers have a
+ {@link org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute#getPositionLength() position length} of one.
+ This means that the token occupies a single position. This attribute is not indexed
+ and thus not taken into account for positional queries, but is used by eg. suggesters.
+</p>
+<p>
+ The main use case for positions lengths is multi-word synonyms. With single-word
+ synonyms, setting the position increment to 0 is enough to denote the fact that two
+ words are synonyms, for example:
+</p>
+<table>
+<tr><td>Term</td><td>red</td><td>magenta</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td></tr>
+</table>
+<p>
+ Given that position(magenta) = 0 + position(red), they are at the same position, so anything
+ working with analyzers will return the exact same result if you replace "magenta" with "red"
+ in the input. However, multi-word synonyms are more tricky. Let's say that you want to build
+ a TokenStream where "IBM" is a synonym of "Internal Business Machines". Position increments
+ are not enough anymore:
+</p>
+<table>
+<tr><td>Term</td><td>IBM</td><td>International</td><td>Business</td><td>Machines</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td><td>1</td><td>1</td></tr>
+</table>
+<p>
+ The problem with this token stream is that "IBM" is at the same position as "International"
+ although it is a synonym with "International Business Machines" as a whole. Setting
+ the position increment of "Business" and "Machines" to 0 wouldn't help as it would mean
+ than "International" is a synonym of "Business". The only way to solve this issue is to
+ make "IBM" span across 3 positions, this is where position lengths come to rescue.
+</p>
+<table>
+<tr><td>Term</td><td>IBM</td><td>International</td><td>Business</td><td>Machines</td></tr>
+<tr><td>Position increment</td><td>1</td><td>0</td><td>1</td><td>1</td></tr>
+<tr><td>Position length</td><td>3</td><td>1</td><td>1</td><td>1</td></tr>
+</table>
+<p>
+ This new attribute makes clear that "IBM" and "International Business Machines" start and end
+ at the same positions.
+</p>
+<a name="corrupt" />
+<h3>How to not write corrupt token streams</h3>
+<p>
+ There are a few rules to observe when writing custom Tokenizers and TokenFilters:
+</p>
+<ul>
+ <li>The first position increment must be > 0.</li>
+ <li>Positions must not go backward.</li>
+ <li>Tokens that have the same start position must have the same start offset.</li>
+ <li>Tokens that have the same end position (taking into account the position length) must have the same end offset.</li>
+</ul>
+<p>
+ Although these rules might seem easy to follow, problems can quickly happen when chaining
+ badly implemented filters that play with positions and offsets, such as synonym or n-grams
+ filters. Here are good practices for writing correct filters:
+</p>
+<ul>
+ <li>Token filters should not modify offsets. If you feel that your filter would need to modify offsets, then it should probably be implemented as a tokenizer.</li>
+ <li>Token filters should not insert positions. If a filter needs to add tokens, then they shoud all have a position increment of 0.</li>
+ <li>When they remove tokens, token filters should increment the position increment of the following token.</li>
+ <li>Token filters should preserve position lengths.</li>
+</ul>
<h2>TokenStream API</h2>
<p>
"Flexible Indexing" summarizes the effort of making the Lucene indexer
@@ -383,6 +444,10 @@ and proximity searches (though sentence
<td>See above for detailed information about position increment.</td>
</tr>
<tr>
+ <td>{@link org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute}</td>
+ <td>The number of positions occupied by a token.</td>
+ </tr>
+ <tr>
<td>{@link org.apache.lucene.analysis.tokenattributes.PayloadAttribute}</td>
<td>The payload that a Token can optionally have.</td>
</tr>
@@ -532,20 +597,26 @@ public final class LengthFilter extends
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
/**
- * Build a filter that removes words that are too long or too
- * short from the text.
+ * Create a new LengthFilter. This will filter out tokens whose
+ * CharTermAttribute is either too short
+ * (< min) or too long (> max).
+ * @param version the Lucene match version
+ * @param in the TokenStream to consume
+ * @param min the minimum length
+ * @param max the maximum length
*/
- public LengthFilter(boolean enablePositionIncrements, TokenStream in, int min, int max) {
- super(enablePositionIncrements, in);
+ public LengthFilter(Version version, TokenStream in, int min, int max) {
+ super(version, in);
this.min = min;
this.max = max;
}
-
+
{@literal @Override}
- public boolean accept() throws IOException {
+ public boolean accept() {
final int len = termAtt.length();
- return (len >= min && len <= max);
+ return (len >= min && len <= max);
}
+
}
</pre>
<p>
@@ -573,66 +644,39 @@ public final class LengthFilter extends
public abstract class FilteringTokenFilter extends TokenFilter {
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- private boolean enablePositionIncrements; // no init needed, as ctor enforces setting value!
- public FilteringTokenFilter(boolean enablePositionIncrements, TokenStream input){
- super(input);
- this.enablePositionIncrements = enablePositionIncrements;
+ /**
+ * Create a new FilteringTokenFilter.
+ * @param in the TokenStream to consume
+ */
+ public FilteringTokenFilter(Version version, TokenStream in) {
+ super(in);
}
- /** Override this method and return if the current input token should be returned by {@literal {@link #incrementToken}}. */
+ /** Override this method and return if the current input token should be returned by incrementToken. */
protected abstract boolean accept() throws IOException;
{@literal @Override}
public final boolean incrementToken() throws IOException {
- if (enablePositionIncrements) {
- int skippedPositions = 0;
- while (input.incrementToken()) {
- if (accept()) {
- if (skippedPositions != 0) {
- posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
- }
- return true;
- }
- skippedPositions += posIncrAtt.getPositionIncrement();
- }
- } else {
- while (input.incrementToken()) {
- if (accept()) {
- return true;
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (accept()) {
+ if (skippedPositions != 0) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
}
+ return true;
}
+ skippedPositions += posIncrAtt.getPositionIncrement();
}
// reached EOS -- return false
return false;
}
- /**
- * {@literal @see #setEnablePositionIncrements(boolean)}
- */
- public boolean getEnablePositionIncrements() {
- return enablePositionIncrements;
+ {@literal @Override}
+ public void reset() throws IOException {
+ super.reset();
}
- /**
- * If <code>true</code>, this TokenFilter will preserve
- * positions of the incoming tokens (ie, accumulate and
- * set position increments of the removed tokens).
- * Generally, <code>true</code> is best as it does not
- * lose information (positions of the original tokens)
- * during indexing.
- *
- * <p> When set, when a token is stopped
- * (omitted), the position increment of the following
- * token is incremented.
- *
- * <p> <b>NOTE</b>: be sure to also
- * set org.apache.lucene.queryparser.classic.QueryParser#setEnablePositionIncrements if
- * you use QueryParser to create queries.
- */
- public void setEnablePositionIncrements(boolean enable) {
- this.enablePositionIncrements = enable;
- }
}
</pre>
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Sat May 4 18:18:08 2013
@@ -64,16 +64,10 @@ public class TestMockAnalyzer extends Ba
/** Test a configuration that behaves a lot like StopAnalyzer */
public void testStop() throws Exception {
- Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
assertAnalyzesTo(a, "the quick brown a fox",
new String[] { "quick", "brown", "fox" },
new int[] { 2, 1, 2 });
-
- // disable positions
- a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, false);
- assertAnalyzesTo(a, "the quick brown a fox",
- new String[] { "quick", "brown", "fox" },
- new int[] { 1, 1, 1 });
}
/** Test a configuration that behaves a lot like KeepWordFilter */
@@ -83,7 +77,7 @@ public class TestMockAnalyzer extends Ba
BasicOperations.complement(
Automaton.union(
Arrays.asList(BasicAutomata.makeString("foo"), BasicAutomata.makeString("bar")))));
- Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, keepWords, true);
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, keepWords);
assertAnalyzesTo(a, "quick foo brown bar bar fox foo",
new String[] { "foo", "bar", "bar", "foo" },
new int[] { 2, 2, 1, 2 });
@@ -92,7 +86,7 @@ public class TestMockAnalyzer extends Ba
/** Test a configuration that behaves a lot like LengthFilter */
public void testLength() throws Exception {
CharacterRunAutomaton length5 = new CharacterRunAutomaton(new RegExp(".{5,}").toAutomaton());
- Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, length5, true);
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, length5);
assertAnalyzesTo(a, "ok toolong fine notfine",
new String[] { "ok", "fine" },
new int[] { 1, 2 });
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Sat May 4 18:18:08 2013
@@ -213,7 +213,7 @@ public class TestTermVectorsWriter exten
public void testEndOffsetPositionStopFilter() throws Exception {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
customType.setStoreTermVectors(true);
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java Sat May 4 18:18:08 2013
@@ -222,7 +222,7 @@ public class TestPhraseQuery extends Luc
public void testPhraseQueryWithStopAnalyzer() throws Exception {
Directory directory = newDirectory();
- Analyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, false);
+ Analyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig( Version.LUCENE_40, stopAnalyzer));
Document doc = new Document();
@@ -241,16 +241,6 @@ public class TestPhraseQuery extends Luc
assertEquals(1, hits.length);
QueryUtils.check(random(), query,searcher);
-
- // StopAnalyzer as of 2.4 does not leave "holes", so this matches.
- query = new PhraseQuery();
- query.add(new Term("field", "words"));
- query.add(new Term("field", "here"));
- hits = searcher.search(query, null, 1000).scoreDocs;
- assertEquals(1, hits.length);
- QueryUtils.check(random(), query,searcher);
-
-
reader.close();
directory.close();
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpanFirstQuery.java Sat May 4 18:18:08 2013
@@ -37,7 +37,7 @@ public class TestSpanFirstQuery extends
// mimic StopAnalyzer
CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|a|of").toAutomaton());
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet, true);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer);
Document doc = new Document();
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java Sat May 4 18:18:08 2013
@@ -60,7 +60,7 @@ public class TestSpansAdvanced extends L
mDirectory = newDirectory();
final RandomIndexWriter writer = new RandomIndexWriter(random(), mDirectory,
newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true))
+ new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))
.setMergePolicy(newLogMergePolicy()).setSimilarity(new DefaultSimilarity()));
addDocument(writer, "1", "I think it should work.");
addDocument(writer, "2", "I think it should work.");
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java Sat May 4 18:18:08 2013
@@ -49,7 +49,7 @@ public class TestSpansAdvanced2 extends
// create test index
final RandomIndexWriter writer = new RandomIndexWriter(random(), mDirectory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(),
- MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true))
+ MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))
.setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy())
.setSimilarity(new DefaultSimilarity()));
addDocument(writer, "A", "Should we, could we, would we?");
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1479148&r1=1479147&r2=1479148&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Sat May 4 18:18:08 2013
@@ -247,7 +247,7 @@ public class HighlighterTest extends Bas
*/
private String highlightField(Query query, String fieldName, String text)
throws IOException, InvalidTokenOffsetsException {
- TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)
+ TokenStream tokenStream = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)
.tokenStream(fieldName, new StringReader(text));
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
@@ -1308,7 +1308,7 @@ public class HighlighterTest extends Bas
}
public void testMaxSizeHighlight() throws Exception {
- final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
analyzer.setEnableChecks(false);
@@ -1343,7 +1343,7 @@ public class HighlighterTest extends Bas
CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("stoppedtoken"));
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
- final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords, true);
+ final MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);
analyzer.setEnableChecks(false);
TermQuery query = new TermQuery(new Term("data", goodWord));
@@ -1394,7 +1394,7 @@ public class HighlighterTest extends Bas
Highlighter hg = getHighlighter(query, "text", fm);
hg.setTextFragmenter(new NullFragmenter());
hg.setMaxDocCharsToAnalyze(36);
- String match = hg.getBestFragment(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords, true), "text", text);
+ String match = hg.getBestFragment(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords), "text", text);
assertTrue(
"Matched text should contain remainder of text after highlighted query ",
match.endsWith("in it"));
@@ -1411,7 +1411,7 @@ public class HighlighterTest extends Bas
numHighlights = 0;
// test to show how rewritten query can still be used
searcher = newSearcher(reader);
- Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
BooleanQuery query = new BooleanQuery();
query.add(new WildcardQuery(new Term(FIELD_NAME, "jf?")), Occur.SHOULD);
@@ -1875,11 +1875,11 @@ public class HighlighterTest extends Bas
super.setUp();
a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
- analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
dir = newDirectory();
ramDir = newDirectory();
IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
for (String text : texts) {
addDoc(writer, text);
}