You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/10/23 19:18:55 UTC
svn commit: r829134 [1/2] - in /lucene/java/branches/lucene_2_9: ./ contrib/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/
contrib/analyzers/common/src/java/org/apache/...
Author: mikemccand
Date: Fri Oct 23 17:18:53 2009
New Revision: 829134
URL: http://svn.apache.org/viewvc?rev=829134&view=rev
Log:
LUCENE-2002 (on 2.9 branch): add Version to QueryParser, MultiFieldQueryParser, StopAnalyzer and the contrib analzyers, to uniformly apply the version-dependent defaults
Modified:
lucene/java/branches/lucene_2_9/CHANGES.txt
lucene/java/branches/lucene_2_9/build.xml
lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java
lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java
lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParser.java
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParser.jj
lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/queryParser/TestQueryParser.java
Modified: lucene/java/branches/lucene_2_9/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/CHANGES.txt?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/CHANGES.txt Fri Oct 23 17:18:53 2009
@@ -42,6 +42,11 @@
settings in StandardAnalyzer to be compatible with the coming
Lucene 3.0. (Uwe Schindler)
+ * LUCENE-2002: Add required Version matchVersion argument when
+ constructing QueryParser or MultiFieldQueryParser and, default (as
+ of 2.9) enablePositionIncrements to true to match
+ StandardAnalyzer's 2.9 default (Uwe Schindler, Mike McCandless)
+
Documentation
* LUCENE-1955: Fix Hits deprecation notice to point users in right
Modified: lucene/java/branches/lucene_2_9/build.xml
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/build.xml?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/build.xml (original)
+++ lucene/java/branches/lucene_2_9/build.xml Fri Oct 23 17:18:53 2009
@@ -587,9 +587,21 @@
<target name="javacc" depends="clean-javacc,javacc-QueryParser,javacc-HTMLParser,javacc-contrib-queryparser, javacc-contrib-surround"/>
<target name="javacc-QueryParser" depends="init,javacc-check" if="javacc.present">
- <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
- outputDir="src/java/org/apache/lucene/queryParser"
- />
+ <sequential>
+ <invoke-javacc target="src/java/org/apache/lucene/queryParser/QueryParser.jj"
+ outputDir="src/java/org/apache/lucene/queryParser"/>
+
+ <!-- Change the inccorrect public ctors for QueryParser to be protected instead -->
+ <replaceregexp file="src/java/org/apache/lucene/queryParser/QueryParser.java"
+ byline="true"
+ match="public QueryParser\(CharStream "
+ replace="protected QueryParser(CharStream "/>
+ <replaceregexp file="src/java/org/apache/lucene/queryParser/QueryParser.java"
+ byline="true"
+ match="public QueryParser\(QueryParserTokenManager "
+ replace="protected QueryParser(QueryParserTokenManager "/>
+
+ </sequential>
</target>
<target name="javacc-HTMLParser" depends="init,javacc-check" if="javacc.present">
Modified: lucene/java/branches/lucene_2_9/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/CHANGES.txt (original)
+++ lucene/java/branches/lucene_2_9/contrib/CHANGES.txt Fri Oct 23 17:18:53 2009
@@ -10,6 +10,15 @@
list. This has no effect on Arabic text, but if you are using a custom
stopword list that contains some non-Arabic words, you'll need to fully
reindex. (DM Smith via Robert Muir)
+
+API Changes:
+
+ * LUCENE-2002: Add required Version matchVersion argument when
+ constructing ComplexPhraseQueryParser and default (as of 2.9)
+ enablePositionIncrements to true to match StandardAnalyzer's
+ default. Also added required matchVersion to most of the analyzers
+ (Uwe Schindler, Mike McCandless)
+
Bug fixes
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -32,6 +32,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Arabic.
@@ -69,10 +70,22 @@
*/
public static final String STOPWORDS_COMMENT = "#";
+ private final Version matchVersion;
+
/**
* Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
+ *
+ * @deprecated Use {@link #ArabicAnalyzer(Version)} instead
*/
public ArabicAnalyzer() {
+ this(Version.LUCENE_24);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}.
+ */
+ public ArabicAnalyzer(Version matchVersion) {
+ this.matchVersion = matchVersion;
try {
InputStream stream = ArabicAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE);
InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
@@ -87,23 +100,53 @@
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #ArabicAnalyzer(Version, String[])} instead
*/
public ArabicAnalyzer( String[] stopwords ) {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public ArabicAnalyzer( Version matchVersion, String[] stopwords ) {
stoptable = StopFilter.makeStopSet( stopwords );
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #ArabicAnalyzer(Version, Hashtable)} instead
*/
public ArabicAnalyzer( Hashtable stopwords ) {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public ArabicAnalyzer( Version matchVersion, Hashtable stopwords ) {
stoptable = new HashSet(stopwords.keySet());
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words. Lines can be commented out using {@link #STOPWORDS_COMMENT}
+ *
+ * @deprecated Use {@link #ArabicAnalyzer(Version, File)} instead
*/
public ArabicAnalyzer( File stopwords ) throws IOException {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words. Lines can be commented out using {@link #STOPWORDS_COMMENT}
+ */
+ public ArabicAnalyzer( Version matchVersion, File stopwords ) throws IOException {
stoptable = WordlistLoader.getWordSet( stopwords, STOPWORDS_COMMENT);
+ this.matchVersion = matchVersion;
}
@@ -117,7 +160,8 @@
public final TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new ArabicLetterTokenizer( reader );
result = new LowerCaseFilter(result);
- result = new StopFilter( result, stoptable );
+ result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable );
result = new ArabicNormalizationFilter( result );
result = new ArabicStemFilter( result );
@@ -144,7 +188,8 @@
streams = new SavedStreams();
streams.source = new ArabicLetterTokenizer(reader);
streams.result = new LowerCaseFilter(streams.source);
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
streams.result = new ArabicNormalizationFilter(streams.result);
streams.result = new ArabicStemFilter(streams.result);
setPreviousTokenStream(streams);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -32,6 +32,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Brazilian Portuguese language.
@@ -40,6 +41,9 @@
* will not be indexed at all) and an external list of exclusions (words that will
* not be stemmed, but indexed).
* </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public final class BrazilianAnalyzer extends Analyzer {
@@ -77,33 +81,73 @@
* Contains words that should be indexed but not stemmed.
*/
private Set excltable = new HashSet();
+ private final Version matchVersion;
/**
* Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
+ *
+ * @deprecated Use {@link #BrazilianAnalyzer(Version)} instead
*/
public BrazilianAnalyzer() {
- stoptable = StopFilter.makeStopSet( BRAZILIAN_STOP_WORDS );
+ this(Version.LUCENE_23);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words ({@link #BRAZILIAN_STOP_WORDS}).
+ */
+ public BrazilianAnalyzer(Version matchVersion) {
+ stoptable = StopFilter.makeStopSet( BRAZILIAN_STOP_WORDS );
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #BrazilianAnalyzer(Version, String[])} instead
*/
public BrazilianAnalyzer( String[] stopwords ) {
- stoptable = StopFilter.makeStopSet( stopwords );
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public BrazilianAnalyzer( Version matchVersion, String[] stopwords ) {
+ stoptable = StopFilter.makeStopSet( stopwords );
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #BrazilianAnalyzer(Version, Map)} instead
*/
public BrazilianAnalyzer( Map stopwords ) {
- stoptable = new HashSet(stopwords.keySet());
+ this(Version.LUCENE_23, stopwords);
}
/**
* Builds an analyzer with the given stop words.
*/
+ public BrazilianAnalyzer( Version matchVersion, Map stopwords ) {
+ stoptable = new HashSet(stopwords.keySet());
+ this.matchVersion = matchVersion;
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ * @deprecated Use {@link #BrazilianAnalyzer(Version, File)} instead
+ */
public BrazilianAnalyzer( File stopwords ) throws IOException {
- stoptable = WordlistLoader.getWordSet( stopwords );
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public BrazilianAnalyzer( Version matchVersion, File stopwords ) throws IOException {
+ stoptable = WordlistLoader.getWordSet( stopwords );
+ this.matchVersion = matchVersion;
}
/**
@@ -136,10 +180,11 @@
* {@link BrazilianStemFilter}.
*/
public final TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer( reader );
+ TokenStream result = new StandardTokenizer( matchVersion, reader );
result = new LowerCaseFilter( result );
result = new StandardFilter( result );
- result = new StopFilter( result, stoptable );
+ result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable );
result = new BrazilianStemFilter( result, excltable );
return result;
}
@@ -162,10 +207,11 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new LowerCaseFilter(streams.source);
streams.result = new StandardFilter(streams.result);
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
streams.result = new BrazilianStemFilter(streams.result, excltable);
setPreviousTokenStream(streams);
} else {
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -21,6 +21,7 @@
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -56,23 +57,45 @@
* stop word list
*/
private Set stopTable;
+ private final Version matchVersion;
//~ Constructors -----------------------------------------------------------
/**
* Builds an analyzer which removes words in {@link #STOP_WORDS}.
+ *
+ * @deprecated Use {@link #CJKAnalyzer(Version)} instead
*/
public CJKAnalyzer() {
+ this(Version.LUCENE_24);
+ }
+
+ /**
+ * Builds an analyzer which removes words in {@link #STOP_WORDS}.
+ */
+ public CJKAnalyzer(Version matchVersion) {
stopTable = StopFilter.makeStopSet(STOP_WORDS);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer which removes words in the provided array.
*
* @param stopWords stop word array
+ * @deprecated Use {@link #CJKAnalyzer(Version, String[])} instead
*/
public CJKAnalyzer(String[] stopWords) {
+ this(Version.LUCENE_24, stopWords);
+ }
+
+ /**
+ * Builds an analyzer which removes words in the provided array.
+ *
+ * @param stopWords stop word array
+ */
+ public CJKAnalyzer(Version matchVersion, String[] stopWords) {
stopTable = StopFilter.makeStopSet(stopWords);
+ this.matchVersion = matchVersion;
}
//~ Methods ----------------------------------------------------------------
@@ -86,7 +109,8 @@
* {@link StopFilter}
*/
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new StopFilter(new CJKTokenizer(reader), stopTable);
+ return new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ new CJKTokenizer(reader), stopTable);
}
private class SavedStreams {
@@ -109,7 +133,8 @@
if (streams == null) {
streams = new SavedStreams();
streams.source = new CJKTokenizer(reader);
- streams.result = new StopFilter(streams.source, stopTable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.source, stopTable);
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
import java.io.*;
import java.util.HashSet;
@@ -37,6 +38,9 @@
* will not be indexed at all).
* A default set of stopwords is used unless an alternative list is specified.
* </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public final class CzechAnalyzer extends Analyzer {
@@ -68,30 +72,68 @@
* Contains the stopwords used with the {@link StopFilter}.
*/
private Set stoptable;
+ private final Version matchVersion;
/**
* Builds an analyzer with the default stop words ({@link #CZECH_STOP_WORDS}).
+ *
+ * @deprecated Use {@link #CzechAnalyzer(Version)} instead
*/
public CzechAnalyzer() {
- stoptable = StopFilter.makeStopSet( CZECH_STOP_WORDS );
+ this(Version.LUCENE_23);
+ }
+ /**
+ * Builds an analyzer with the default stop words ({@link #CZECH_STOP_WORDS}).
+ */
+ public CzechAnalyzer(Version matchVersion) {
+ stoptable = StopFilter.makeStopSet( CZECH_STOP_WORDS );
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #CzechAnalyzer(Version, String[])} instead
*/
public CzechAnalyzer( String[] stopwords ) {
- stoptable = StopFilter.makeStopSet( stopwords );
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public CzechAnalyzer(Version matchVersion, String[] stopwords) {
+ stoptable = StopFilter.makeStopSet( stopwords );
+ this.matchVersion = matchVersion;
}
+ /**
+ * @deprecated Use {@link #CzechAnalyzer(Version, HashSet)} instead
+ */
public CzechAnalyzer( HashSet stopwords ) {
- stoptable = stopwords;
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ public CzechAnalyzer(Version matchVersion, HashSet stopwords) {
+ stoptable = stopwords;
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #CzechAnalyzer(Version, File)} instead
*/
public CzechAnalyzer( File stopwords ) throws IOException {
- stoptable = WordlistLoader.getWordSet( stopwords );
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public CzechAnalyzer(Version matchVersion, File stopwords ) throws IOException {
+ stoptable = WordlistLoader.getWordSet( stopwords );
+ this.matchVersion = matchVersion;
}
/**
@@ -135,10 +177,11 @@
* {@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
*/
public final TokenStream tokenStream( String fieldName, Reader reader ) {
- TokenStream result = new StandardTokenizer( reader );
+ TokenStream result = new StandardTokenizer( matchVersion, reader );
result = new StandardFilter( result );
result = new LowerCaseFilter( result );
- result = new StopFilter( result, stoptable );
+ result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable );
return result;
}
@@ -159,10 +202,11 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
streams.result = new LowerCaseFilter(streams.result);
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -33,6 +33,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for German language.
@@ -44,6 +45,9 @@
* exclusion list is empty by default.
* </p>
*
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
+ *
* @version $Id$
*/
public class GermanAnalyzer extends Analyzer {
@@ -76,37 +80,80 @@
*/
private Set exclusionSet = new HashSet();
+ private final Version matchVersion;
+
/**
* Builds an analyzer with the default stop words:
* {@link #GERMAN_STOP_WORDS}.
+ *
+ * @deprecated Use {@link #GermanAnalyzer(Version)} instead
*/
public GermanAnalyzer() {
+ this(Version.LUCENE_23);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words:
+ * {@link #GERMAN_STOP_WORDS}.
+ */
+ public GermanAnalyzer(Version matchVersion) {
stopSet = StopFilter.makeStopSet(GERMAN_STOP_WORDS);
setOverridesTokenStreamMethod(GermanAnalyzer.class);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #GermanAnalyzer(Version, String[])} instead
*/
public GermanAnalyzer(String[] stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public GermanAnalyzer(Version matchVersion, String[] stopwords) {
stopSet = StopFilter.makeStopSet(stopwords);
setOverridesTokenStreamMethod(GermanAnalyzer.class);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #GermanAnalyzer(Version, Map)} instead
*/
public GermanAnalyzer(Map stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public GermanAnalyzer(Version matchVersion, Map stopwords) {
stopSet = new HashSet(stopwords.keySet());
setOverridesTokenStreamMethod(GermanAnalyzer.class);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #GermanAnalyzer(Version, File)} instead
*/
public GermanAnalyzer(File stopwords) throws IOException {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public GermanAnalyzer(Version matchVersion, File stopwords) throws IOException {
stopSet = WordlistLoader.getWordSet(stopwords);
setOverridesTokenStreamMethod(GermanAnalyzer.class);
+ this.matchVersion = matchVersion;
}
/**
@@ -141,10 +188,11 @@
* {@link GermanStemFilter}
*/
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer(reader);
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
- result = new StopFilter(result, stopSet);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
result = new GermanStemFilter(result, exclusionSet);
return result;
}
@@ -173,10 +221,11 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
streams.result = new LowerCaseFilter(streams.result);
- streams.result = new StopFilter(streams.result, stopSet);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
streams.result = new GermanStemFilter(streams.result, exclusionSet);
setPreviousTokenStream(streams);
} else {
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -36,6 +37,9 @@
* that will not be indexed at all).
* A default set of stopwords is used unless an alternative list is specified.
* </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public final class GreekAnalyzer extends Analyzer
{
@@ -159,40 +163,62 @@
*/
private char[] charset;
+ private final Version matchVersion;
+
+ /** @deprecated Use {@link #GreekAnalyzer(Version)} instead */
public GreekAnalyzer() {
+ this(Version.LUCENE_23);
+ }
+
+ public GreekAnalyzer(Version matchVersion) {
charset = GreekCharsets.UnicodeGreek;
stopSet = StopFilter.makeStopSet(
makeStopWords(GreekCharsets.UnicodeGreek));
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer.
- * @deprecated Use {@link #GreekAnalyzer()} instead.
+ * @deprecated Use {@link #GreekAnalyzer(Version)} instead.
*/
public GreekAnalyzer(char[] charset)
{
this.charset = charset;
stopSet = StopFilter.makeStopSet(makeStopWords(charset));
+ matchVersion = Version.LUCENE_23;
}
/**
* Builds an analyzer with the given stop words.
* @param stopwords Array of stopwords to use.
+ *
+ * @deprecated Use {@link #GreekAnalyzer(Version, String[])} instead
*/
public GreekAnalyzer(String [] stopwords)
{
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ * @param stopwords Array of stopwords to use.
+ */
+ public GreekAnalyzer(Version matchVersion, String [] stopwords)
+ {
charset = GreekCharsets.UnicodeGreek;
stopSet = StopFilter.makeStopSet(stopwords);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
- * @deprecated Use {@link #GreekAnalyzer(String[])} instead.
+ * @deprecated Use {@link #GreekAnalyzer(Version, String[])} instead.
*/
public GreekAnalyzer(char[] charset, String[] stopwords)
{
this.charset = charset;
stopSet = StopFilter.makeStopSet(stopwords);
+ matchVersion = Version.LUCENE_23;
}
/**
@@ -219,21 +245,33 @@
/**
* Builds an analyzer with the given stop words.
- * @deprecated Use {@link #GreekAnalyzer(Map)} instead.
+ * @deprecated Use {@link #GreekAnalyzer(Version, Map)} instead.
*/
public GreekAnalyzer(char[] charset, Map stopwords)
{
this.charset = charset;
stopSet = new HashSet(stopwords.keySet());
+ matchVersion = Version.LUCENE_23;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #GreekAnalyzer(Version,Map)} instead
*/
public GreekAnalyzer(Map stopwords)
{
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public GreekAnalyzer(Version matchVersion, Map stopwords)
+ {
charset = GreekCharsets.UnicodeGreek;
stopSet = new HashSet(stopwords.keySet());
+ this.matchVersion = matchVersion;
}
/**
@@ -244,9 +282,10 @@
*/
public TokenStream tokenStream(String fieldName, Reader reader)
{
- TokenStream result = new StandardTokenizer(reader);
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new GreekLowerCaseFilter(result, charset);
- result = new StopFilter(result, stopSet);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
return result;
}
@@ -267,9 +306,10 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new GreekLowerCaseFilter(streams.source, charset);
- streams.result = new StopFilter(streams.result, stopSet);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -34,6 +34,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.ar.ArabicLetterTokenizer;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Persian.
@@ -66,11 +67,24 @@
*/
public static final String STOPWORDS_COMMENT = "#";
+ private final Version matchVersion;
+
/**
* Builds an analyzer with the default stop words:
* {@link #DEFAULT_STOPWORD_FILE}.
+ *
+ * @deprecated Use {@link #PersianAnalyzer(Version)} instead
*/
public PersianAnalyzer() {
+ this(Version.LUCENE_24);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words:
+ * {@link #DEFAULT_STOPWORD_FILE}.
+ */
+ public PersianAnalyzer(Version matchVersion) {
+ this.matchVersion = matchVersion;
try {
InputStream stream = PersianAnalyzer.class
.getResourceAsStream(DEFAULT_STOPWORD_FILE);
@@ -86,24 +100,55 @@
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #PersianAnalyzer(Version, String[])} instead
*/
public PersianAnalyzer(String[] stopwords) {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public PersianAnalyzer(Version matchVersion, String[] stopwords) {
stoptable = StopFilter.makeStopSet(stopwords);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #PersianAnalyzer(Version, Hashtable)} instead
*/
public PersianAnalyzer(Hashtable stopwords) {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public PersianAnalyzer(Version matchVersion, Hashtable stopwords) {
stoptable = new HashSet(stopwords.keySet());
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words. Lines can be commented out
* using {@link #STOPWORDS_COMMENT}
+ *
+ * @deprecated Use {@link #PersianAnalyzer(Version, File)} instead
*/
public PersianAnalyzer(File stopwords) throws IOException {
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words. Lines can be commented out
+ * using {@link #STOPWORDS_COMMENT}
+ */
+ public PersianAnalyzer(Version matchVersion, File stopwords) throws IOException {
stoptable = WordlistLoader.getWordSet(stopwords, STOPWORDS_COMMENT);
+ this.matchVersion = matchVersion;
}
/**
@@ -125,7 +170,8 @@
* the order here is important: the stopword list is normalized with the
* above!
*/
- result = new StopFilter(result, stoptable);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable);
return result;
}
@@ -158,7 +204,8 @@
* the order here is important: the stopword list is normalized with the
* above!
*/
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
@@ -43,6 +44,17 @@
* exclusion list is empty by default.
* </p>
*
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating FrenchAnalyzer:
+ * <ul>
+ * <li> As of 2.9, StopFilter preserves position
+ * increments
+ * </ul>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
+ *
* @version $Id$
*/
public final class FrenchAnalyzer extends Analyzer {
@@ -84,26 +96,60 @@
*/
private Set excltable = new HashSet();
+ private final Version matchVersion;
+
/**
* Builds an analyzer with the default stop words ({@link #FRENCH_STOP_WORDS}).
+ *
+ * @deprecated Use {@link #FrenchAnalyzer(Version)} instead.
*/
public FrenchAnalyzer() {
+ this(Version.LUCENE_23);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words ({@link #FRENCH_STOP_WORDS}).
+ */
+ public FrenchAnalyzer(Version matchVersion) {
stoptable = StopFilter.makeStopSet(FRENCH_STOP_WORDS);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #FrenchAnalyzer(Version,
+ * String[])} instead.
*/
public FrenchAnalyzer(String[] stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public FrenchAnalyzer(Version matchVersion, String[] stopwords) {
stoptable = StopFilter.makeStopSet(stopwords);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
* @throws IOException
+ *
+ * @deprecated Use {@link #FrenchAnalyzer(Version, File)} instead
*/
public FrenchAnalyzer(File stopwords) throws IOException {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ * @throws IOException
+ */
+ public FrenchAnalyzer(Version matchVersion, File stopwords) throws IOException {
stoptable = new HashSet(WordlistLoader.getWordSet(stopwords));
+ this.matchVersion = matchVersion;
}
/**
@@ -144,9 +190,10 @@
if (fieldName == null) throw new IllegalArgumentException("fieldName must not be null");
if (reader == null) throw new IllegalArgumentException("reader must not be null");
- TokenStream result = new StandardTokenizer(reader);
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
- result = new StopFilter(result, stoptable);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable);
result = new FrenchStemFilter(result, excltable);
// Convert to lowercase after stemming!
result = new LowerCaseFilter(result);
@@ -171,9 +218,10 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
streams.result = new FrenchStemFilter(streams.result, excltable);
// Convert to lowercase after stemming!
streams.result = new LowerCaseFilter(streams.result);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,6 +23,7 @@
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
@@ -42,6 +43,9 @@
* A default set of stopwords is used unless an alternative list is specified, but the
* exclusion list is empty by default.
* </p>
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public class DutchAnalyzer extends Analyzer {
/**
@@ -73,48 +77,92 @@
private Set excltable = new HashSet();
private Map stemdict = new HashMap();
-
+ private final Version matchVersion;
/**
* Builds an analyzer with the default stop words ({@link #DUTCH_STOP_WORDS})
* and a few default entries for the stem exclusion table.
*
+ * @deprecated Use {@link #DutchAnalyzer(Version)} instead
*/
public DutchAnalyzer() {
+ this(Version.LUCENE_23);
+ }
+
+ /**
+ * Builds an analyzer with the default stop words ({@link #DUTCH_STOP_WORDS})
+ * and a few default entries for the stem exclusion table.
+ *
+ */
+ public DutchAnalyzer(Version matchVersion) {
setOverridesTokenStreamMethod(DutchAnalyzer.class);
stoptable = StopFilter.makeStopSet(DUTCH_STOP_WORDS);
stemdict.put("fiets", "fiets"); //otherwise fiet
stemdict.put("bromfiets", "bromfiets"); //otherwise bromfiet
stemdict.put("ei", "eier");
stemdict.put("kind", "kinder");
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
*
* @param stopwords
+ * @deprecated Use {@link #DutchAnalyzer(Version, String[])} instead
*/
public DutchAnalyzer(String[] stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ *
+ * @param matchVersion
+ * @param stopwords
+ */
+ public DutchAnalyzer(Version matchVersion, String[] stopwords) {
setOverridesTokenStreamMethod(DutchAnalyzer.class);
stoptable = StopFilter.makeStopSet(stopwords);
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
*
* @param stopwords
+ * @deprecated Use {@link #DutchAnalyzer(Version, HashSet)} instead
*/
public DutchAnalyzer(HashSet stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ *
+ * @param stopwords
+ */
+ public DutchAnalyzer(Version matchVersion, HashSet stopwords) {
setOverridesTokenStreamMethod(DutchAnalyzer.class);
stoptable = stopwords;
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer with the given stop words.
*
* @param stopwords
+ * @deprecated Use {@link #DutchAnalyzer(Version, File)} instead
*/
public DutchAnalyzer(File stopwords) {
+ this(Version.LUCENE_23, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ *
+ * @param stopwords
+ */
+ public DutchAnalyzer(Version matchVersion, File stopwords) {
setOverridesTokenStreamMethod(DutchAnalyzer.class);
try {
stoptable = org.apache.lucene.analysis.WordlistLoader.getWordSet(stopwords);
@@ -122,6 +170,7 @@
// TODO: throw IOException
throw new RuntimeException(e);
}
+ this.matchVersion = matchVersion;
}
/**
@@ -179,9 +228,10 @@
* and {@link DutchStemFilter}
*/
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer(reader);
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
- result = new StopFilter(result, stoptable);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stoptable);
result = new DutchStemFilter(result, excltable, stemdict);
return result;
}
@@ -211,9 +261,10 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
- streams.result = new StopFilter(streams.result, stoptable);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stoptable);
streams.result = new DutchStemFilter(streams.result, excltable, stemdict);
setPreviousTokenStream(streams);
} else {
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,6 +23,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -48,15 +49,27 @@
//The default maximum percentage (40%) of index documents which
//can contain a term, after which the term is considered to be a stop word.
public static final float defaultMaxDocFreqPercent = 0.4f;
+ private final Version matchVersion;
/**
* Initializes this analyzer with the Analyzer object that actually produces the tokens
*
* @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
+ * @deprecated Use {@link #QueryAutoStopWordAnalyzer(Version, Analyzer)} instead
*/
public QueryAutoStopWordAnalyzer(Analyzer delegate) {
+ this(Version.LUCENE_24, delegate);
+ }
+
+ /**
+ * Initializes this analyzer with the Analyzer object that actually produces the tokens
+ *
+ * @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
+ */
+ public QueryAutoStopWordAnalyzer(Version matchVersion, Analyzer delegate) {
this.delegate = delegate;
setOverridesTokenStreamMethod(QueryAutoStopWordAnalyzer.class);
+ this.matchVersion = matchVersion;
}
/**
@@ -175,7 +188,8 @@
}
HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
if (stopWords != null) {
- result = new StopFilter(result, stopWords);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopWords);
}
return result;
}
@@ -217,7 +231,8 @@
/* if there are any stopwords for the field, save the stopfilter */
HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
if (stopWords != null)
- streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
+ streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.wrapped, stopWords);
else
streams.withStopFilter = streams.wrapped;
@@ -238,7 +253,8 @@
streams.wrapped = result;
HashSet stopWords = (HashSet) stopWordsPerField.get(fieldName);
if (stopWords != null)
- streams.withStopFilter = new StopFilter(streams.wrapped, stopWords);
+ streams.withStopFilter = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.wrapped, stopWords);
else
streams.withStopFilter = streams.wrapped;
}
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -27,6 +27,7 @@
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Russian language.
@@ -193,41 +194,60 @@
* @deprecated Support for non-Unicode encodings will be removed in Lucene 3.0
*/
private char[] charset;
+ private final Version matchVersion;
-
+ /** @deprecated Use {@link #RussianAnalyzer(Version)} instead */
public RussianAnalyzer() {
+ this(Version.LUCENE_24);
+ }
+
+ public RussianAnalyzer(Version matchVersion) {
charset = RussianCharsets.UnicodeRussian;
stopSet = StopFilter.makeStopSet(
makeStopWords(RussianCharsets.UnicodeRussian));
+ this.matchVersion = matchVersion;
}
/**
* Builds an analyzer.
- * @deprecated Use {@link #RussianAnalyzer()} instead.
+ * @deprecated Use {@link #RussianAnalyzer(Version)} instead.
*/
public RussianAnalyzer(char[] charset)
{
this.charset = charset;
stopSet = StopFilter.makeStopSet(makeStopWords(charset));
+ matchVersion = Version.LUCENE_24;
}
/**
* Builds an analyzer with the given stop words.
- * @deprecated Use {@link #RussianAnalyzer(String[])} instead.
+ * @deprecated Use {@link #RussianAnalyzer(Version,String[])} instead.
*/
public RussianAnalyzer(char[] charset, String[] stopwords)
{
this.charset = charset;
stopSet = StopFilter.makeStopSet(stopwords);
+ matchVersion = Version.LUCENE_24;
}
/**
* Builds an analyzer with the given stop words.
+ *
+ * @deprecated Use {@link #RussianAnalyzer(Version,String[])} instead.
*/
public RussianAnalyzer(String[] stopwords)
{
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ */
+ public RussianAnalyzer(Version matchVersion, String[] stopwords)
+ {
this.charset = RussianCharsets.UnicodeRussian;
stopSet = StopFilter.makeStopSet(stopwords);
+ this.matchVersion = matchVersion;
}
/** Takes russian stop words and translates them to a String array, using
@@ -254,22 +274,36 @@
/**
* Builds an analyzer with the given stop words.
* TODO: create a Set version of this ctor
- * @deprecated Use {@link #RussianAnalyzer(Map)} instead.
+ *
+ * @deprecated Use {@link #RussianAnalyzer(Version, Map)} instead.
*/
public RussianAnalyzer(char[] charset, Map stopwords)
{
this.charset = charset;
stopSet = new HashSet(stopwords.keySet());
+ matchVersion = Version.LUCENE_24;
}
/**
* Builds an analyzer with the given stop words.
* TODO: create a Set version of this ctor
+ *
+ * @deprecated Use {@link #RussianAnalyzer(Version, Map)} instead.
*/
public RussianAnalyzer(Map stopwords)
{
+ this(Version.LUCENE_24, stopwords);
+ }
+
+ /**
+ * Builds an analyzer with the given stop words.
+ * TODO: create a Set version of this ctor
+ */
+ public RussianAnalyzer(Version matchVersion, Map stopwords)
+ {
charset = RussianCharsets.UnicodeRussian;
stopSet = new HashSet(stopwords.keySet());
+ this.matchVersion = matchVersion;
}
/**
@@ -285,7 +319,8 @@
{
TokenStream result = new RussianLetterTokenizer(reader, charset);
result = new RussianLowerCaseFilter(result, charset);
- result = new StopFilter(result, stopSet);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
result = new RussianStemFilter(result, charset);
return result;
}
@@ -311,7 +346,8 @@
streams = new SavedStreams();
streams.source = new RussianLetterTokenizer(reader, charset);
streams.result = new RussianLowerCaseFilter(streams.source, charset);
- streams.result = new StopFilter(streams.result, stopSet);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
streams.result = new RussianStemFilter(streams.result, charset);
setPreviousTokenStream(streams);
} else {
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -25,22 +25,34 @@
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.util.Version;
/**
* {@link Analyzer} for Thai language. It uses {@link java.text.BreakIterator} to break words.
* @version 0.2
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public class ThaiAnalyzer extends Analyzer {
-
+ private final Version matchVersion;
+
+ /** @deprecated Use {@link #ThaiAnalyzer(Version)} instead */
public ThaiAnalyzer() {
+ this(Version.LUCENE_23);
+ }
+
+ public ThaiAnalyzer(Version matchVersion) {
setOverridesTokenStreamMethod(ThaiAnalyzer.class);
+ this.matchVersion = matchVersion;
}
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream ts = new StandardTokenizer(reader);
+ TokenStream ts = new StandardTokenizer(matchVersion, reader);
ts = new StandardFilter(ts);
ts = new ThaiWordFilter(ts);
- ts = new StopFilter(ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+ ts = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
return ts;
}
@@ -60,10 +72,11 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
streams.result = new ThaiWordFilter(streams.result);
- streams.result = new StopFilter(streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test/org/apache/lucene/analysis/fr/TestFrenchAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.util.Version;
/**
* Test case for FrenchAnalyzer.
@@ -32,7 +33,7 @@
public class TestFrenchAnalyzer extends BaseTokenStreamTestCase {
public void testAnalyzer() throws Exception {
- FrenchAnalyzer fa = new FrenchAnalyzer();
+ FrenchAnalyzer fa = new FrenchAnalyzer(Version.LUCENE_CURRENT);
// test null reader
boolean iaeFlag = false;
Modified: lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -31,6 +31,7 @@
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
import org.apache.lucene.analysis.cn.smart.WordTokenFilter;
+import org.apache.lucene.util.Version;
/**
* <p>
@@ -59,12 +60,22 @@
public class SmartChineseAnalyzer extends Analyzer {
private final Set stopWords;
+ private final Version matchVersion;
/**
* Create a new SmartChineseAnalyzer, using the default stopword list.
+ *
+ * @deprecated Use {@link #SmartChineseAnalyzer(Version)} instead
*/
public SmartChineseAnalyzer() {
- this(true);
+ this(Version.LUCENE_24, true);
+ }
+
+ /**
+ * Create a new SmartChineseAnalyzer, using the default stopword list.
+ */
+ public SmartChineseAnalyzer(Version matchVersion) {
+ this(matchVersion, true);
}
/**
@@ -77,8 +88,26 @@
* </p>
*
* @param useDefaultStopWords true to use the default stopword list.
+ *
+ * @deprecated Use {@link #SmartChineseAnalyzer(Version, boolean)} instead
*/
public SmartChineseAnalyzer(boolean useDefaultStopWords) {
+ this(Version.LUCENE_24, useDefaultStopWords);
+ }
+
+ /**
+ * <p>
+ * Create a new SmartChineseAnalyzer, optionally using the default stopword list.
+ * </p>
+ * <p>
+ * The included default stopword list is simply a list of punctuation.
+ * If you do not use this list, punctuation will not be removed from the text!
+ * </p>
+ *
+ * @param useDefaultStopWords true to use the default stopword list.
+ */
+ public SmartChineseAnalyzer(Version matchVersion, boolean useDefaultStopWords) {
+ this.matchVersion = matchVersion;
if (useDefaultStopWords) {
try {
InputStream stream = this.getClass().getResourceAsStream("stopwords.txt");
@@ -101,9 +130,25 @@
* Note: the set should include punctuation, unless you want to index punctuation!
* </p>
* @param stopWords {@link Set} of stopwords to use.
+ *
+ * @deprecated Use {@link #SmartChineseAnalyzer(Version, Set)} instead
*/
public SmartChineseAnalyzer(Set stopWords) {
+ this(Version.LUCENE_24, stopWords);
+ }
+
+ /**
+ * <p>
+ * Create a new SmartChineseAnalyzer, using the provided {@link Set} of stopwords.
+ * </p>
+ * <p>
+ * Note: the set should include punctuation, unless you want to index punctuation!
+ * </p>
+ * @param stopWords {@link Set} of stopwords to use.
+ */
+ public SmartChineseAnalyzer(Version matchVersion, Set stopWords) {
this.stopWords = stopWords;
+ this.matchVersion = matchVersion;
}
public TokenStream tokenStream(String fieldName, Reader reader) {
@@ -114,7 +159,8 @@
// The porter stemming is too strict, this is not a bug, this is a feature:)
result = new PorterStemFilter(result);
if (stopWords != null) {
- result = new StopFilter(result, stopWords, false);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopWords, false);
}
return result;
}
@@ -134,7 +180,8 @@
streams.filteredTokenStream = new WordTokenFilter(streams.tokenStream);
streams.filteredTokenStream = new PorterStemFilter(streams.filteredTokenStream);
if (stopWords != null) {
- streams.filteredTokenStream = new StopFilter(streams.filteredTokenStream, stopWords, false);
+ streams.filteredTokenStream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.filteredTokenStream, stopWords, false);
}
} else {
streams.tokenStream.reset(reader);
Modified: lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -33,6 +33,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.util.Version;
/**
* Efficient Lucene analyzer/tokenizer that preferably operates on a String rather than a
@@ -139,6 +140,8 @@
private final Pattern pattern;
private final boolean toLowerCase;
private final Set stopWords;
+
+ private final Version matchVersion;
/**
* Constructs a new instance with the given parameters.
@@ -157,8 +160,33 @@
* <code>WordlistLoader.getWordSet(new File("samples/fulltext/stopwords.txt")</code>
* or <a href="http://www.unine.ch/info/clef/">other stop words
* lists </a>.
+ *
+ * @deprecated Use {@link #PatternAnalyzer(Version, Pattern, boolean, Set)} instead
*/
public PatternAnalyzer(Pattern pattern, boolean toLowerCase, Set stopWords) {
+ this(Version.LUCENE_24, pattern, toLowerCase, stopWords);
+ }
+
+ /**
+ * Constructs a new instance with the given parameters.
+ *
+ * @param matchVersion If >= {@link Version#LUCENE_29}, StopFilter.enablePositionIncrement is set to true
+ * @param pattern
+ * a regular expression delimiting tokens
+ * @param toLowerCase
+ * if <code>true</code> returns tokens after applying
+ * String.toLowerCase()
+ * @param stopWords
+ * if non-null, ignores all tokens that are contained in the
+ * given stop set (after previously having applied toLowerCase()
+ * if applicable). For example, created via
+ * {@link StopFilter#makeStopSet(String[])}and/or
+ * {@link org.apache.lucene.analysis.WordlistLoader}as in
+ * <code>WordlistLoader.getWordSet(new File("samples/fulltext/stopwords.txt")</code>
+ * or <a href="http://www.unine.ch/info/clef/">other stop words
+ * lists </a>.
+ */
+ public PatternAnalyzer(Version matchVersion, Pattern pattern, boolean toLowerCase, Set stopWords) {
if (pattern == null)
throw new IllegalArgumentException("pattern must not be null");
@@ -170,6 +198,7 @@
this.pattern = pattern;
this.toLowerCase = toLowerCase;
this.stopWords = stopWords;
+ this.matchVersion = matchVersion;
}
/**
@@ -197,7 +226,7 @@
}
else {
stream = new PatternTokenizer(text, pattern, toLowerCase);
- if (stopWords != null) stream = new StopFilter(false, stream, stopWords);
+ if (stopWords != null) stream = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), stream, stopWords);
}
return stream;
Modified: lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/ComplexPhraseQueryParser.java Fri Oct 23 17:18:53 2009
@@ -38,6 +38,7 @@
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.Version;
/**
* QueryParser which permits complex phrase query syntax eg "(john jon
@@ -67,8 +68,15 @@
private ComplexPhraseQuery currentPhraseQuery = null;
+ /** @deprecated Use {@link
+ #ComplexPhraseQueryParser{Version, String, Analyzer)}
+ instead.*/
public ComplexPhraseQueryParser(String f, Analyzer a) {
- super(f, a);
+ this(Version.LUCENE_24, f, a);
+ }
+
+ public ComplexPhraseQueryParser(Version matchVersion, String f, Analyzer a) {
+ super(matchVersion, f, a);
}
protected Query getFieldQuery(String field, String queryText, int slop) {
Modified: lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -19,6 +19,7 @@
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.*;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -30,20 +31,39 @@
* Available stemmers are listed in org.tartarus.snowball.ext. The name of a
* stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
* {@link org.tartarus.snowball.ext.EnglishStemmer} is named "English".
+ *
+ * <p><b>NOTE</b>: This class uses the same {@link Version}
+ * dependent settings as {@link StandardAnalyzer}.</p>
*/
public class SnowballAnalyzer extends Analyzer {
private String name;
private Set stopSet;
+ private final Version matchVersion;
- /** Builds the named analyzer with no stop words. */
+ /** Builds the named analyzer with no stop words.
+ *
+ * @deprecated Use {@link {#SnowballAnalyzer(Version, String)} instead*/
public SnowballAnalyzer(String name) {
+ this(Version.LUCENE_23, name);
+ }
+
+ /** Builds the named analyzer with no stop words. */
+ public SnowballAnalyzer(Version matchVersion, String name) {
this.name = name;
setOverridesTokenStreamMethod(SnowballAnalyzer.class);
+ this.matchVersion = matchVersion;
}
- /** Builds the named analyzer with the given stop words. */
+ /** Builds the named analyzer with the given stop words.
+ *
+ * @deprecated Use {@link {#SnowballAnalyzer(Version, String, String[])} instead*/
public SnowballAnalyzer(String name, String[] stopWords) {
- this(name);
+ this(Version.LUCENE_23, name, stopWords);
+ }
+
+ /** Builds the named analyzer with the given stop words. */
+ public SnowballAnalyzer(Version matchVersion, String name, String[] stopWords) {
+ this(matchVersion, name);
stopSet = StopFilter.makeStopSet(stopWords);
}
@@ -51,11 +71,12 @@
StandardFilter}, a {@link LowerCaseFilter}, a {@link StopFilter},
and a {@link SnowballFilter} */
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new StandardTokenizer(reader);
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
result = new LowerCaseFilter(result);
if (stopSet != null)
- result = new StopFilter(result, stopSet);
+ result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
result = new SnowballFilter(result, name);
return result;
}
@@ -80,11 +101,12 @@
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
if (streams == null) {
streams = new SavedStreams();
- streams.source = new StandardTokenizer(reader);
+ streams.source = new StandardTokenizer(matchVersion, reader);
streams.result = new StandardFilter(streams.source);
streams.result = new LowerCaseFilter(streams.result);
if (stopSet != null)
- streams.result = new StopFilter(streams.result, stopSet);
+ streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
streams.result = new SnowballFilter(streams.result, name);
setPreviousTokenStream(streams);
} else {
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -23,7 +23,18 @@
import java.util.Arrays;
import java.util.Set;
-/** Filters {@link LetterTokenizer} with {@link LowerCaseFilter} and {@link StopFilter}. */
+import org.apache.lucene.util.Version;
+
+/** Filters {@link LetterTokenizer} with {@link
+ * LowerCaseFilter} and {@link StopFilter}.
+ *
+ * <a name="version"/>
+ * <p>You must specify the required {@link Version}
+ * compatibility when creating StopAnalyzer:
+ * <ul>
+ * <li> As of 2.9, position increments are preserved
+ * </ul>
+*/
public final class StopAnalyzer extends Analyzer {
private final Set/*<String>*/ stopWords;
@@ -61,7 +72,7 @@
/** Builds an analyzer which removes words in
* ENGLISH_STOP_WORDS.
- * @deprecated Use {@link #StopAnalyzer(boolean)} instead */
+ * @deprecated Use {@link #StopAnalyzer(Version)} instead */
public StopAnalyzer() {
stopWords = ENGLISH_STOP_WORDS_SET;
useDefaultStopPositionIncrement = true;
@@ -69,9 +80,18 @@
}
/** Builds an analyzer which removes words in
+ * ENGLISH_STOP_WORDS.*/
+ public StopAnalyzer(Version matchVersion) {
+ stopWords = ENGLISH_STOP_WORDS_SET;
+ useDefaultStopPositionIncrement = false;
+ enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+ }
+
+ /** Builds an analyzer which removes words in
* ENGLISH_STOP_WORDS.
* @param enablePositionIncrements See {@link
- * StopFilter#setEnablePositionIncrements} */
+ * StopFilter#setEnablePositionIncrements}
+ * @deprecated Use {@link #StopAnalyzer(Version)} instead */
public StopAnalyzer(boolean enablePositionIncrements) {
stopWords = ENGLISH_STOP_WORDS_SET;
this.enablePositionIncrements = enablePositionIncrements;
@@ -79,17 +99,26 @@
}
/** Builds an analyzer with the stop words from the given set.
- * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead */
+ * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
public StopAnalyzer(Set stopWords) {
this.stopWords = stopWords;
useDefaultStopPositionIncrement = true;
enablePositionIncrements = false;
}
+ /** Builds an analyzer with the stop words from the given
+ * set. */
+ public StopAnalyzer(Version matchVersion, Set stopWords) {
+ this.stopWords = stopWords;
+ useDefaultStopPositionIncrement = false;
+ enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+ }
+
/** Builds an analyzer with the stop words from the given set.
* @param stopWords Set of stop words
* @param enablePositionIncrements See {@link
- * StopFilter#setEnablePositionIncrements} */
+ * StopFilter#setEnablePositionIncrements}
+ * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
public StopAnalyzer(Set stopWords, boolean enablePositionIncrements) {
this.stopWords = stopWords;
this.enablePositionIncrements = enablePositionIncrements;
@@ -97,7 +126,7 @@
}
/** Builds an analyzer which removes words in the provided array.
- * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead */
+ * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead */
public StopAnalyzer(String[] stopWords) {
this.stopWords = StopFilter.makeStopSet(stopWords);
useDefaultStopPositionIncrement = true;
@@ -108,7 +137,7 @@
* @param stopWords Array of stop words
* @param enablePositionIncrements See {@link
* StopFilter#setEnablePositionIncrements}
- * @deprecated Use {@link #StopAnalyzer(Set, boolean)} instead*/
+ * @deprecated Use {@link #StopAnalyzer(Version, Set)} instead*/
public StopAnalyzer(String[] stopWords, boolean enablePositionIncrements) {
this.stopWords = StopFilter.makeStopSet(stopWords);
this.enablePositionIncrements = enablePositionIncrements;
@@ -117,7 +146,7 @@
/** Builds an analyzer with the stop words from the given file.
* @see WordlistLoader#getWordSet(File)
- * @deprecated Use {@link #StopAnalyzer(File, boolean)} instead */
+ * @deprecated Use {@link #StopAnalyzer(Version, File)} instead */
public StopAnalyzer(File stopwordsFile) throws IOException {
stopWords = WordlistLoader.getWordSet(stopwordsFile);
useDefaultStopPositionIncrement = true;
@@ -128,16 +157,27 @@
* @see WordlistLoader#getWordSet(File)
* @param stopwordsFile File to load stop words from
* @param enablePositionIncrements See {@link
- * StopFilter#setEnablePositionIncrements} */
+ * StopFilter#setEnablePositionIncrements}
+ * @deprecated Use {@link #StopAnalyzer(Version, File)} instead */
public StopAnalyzer(File stopwordsFile, boolean enablePositionIncrements) throws IOException {
stopWords = WordlistLoader.getWordSet(stopwordsFile);
this.enablePositionIncrements = enablePositionIncrements;
useDefaultStopPositionIncrement = false;
}
+ /** Builds an analyzer with the stop words from the given file.
+ * @see WordlistLoader#getWordSet(File)
+ * @param matchVersion See <a href="#version">above</a>
+ * @param stopwordsFile File to load stop words from */
+ public StopAnalyzer(Version matchVersion, File stopwordsFile) throws IOException {
+ stopWords = WordlistLoader.getWordSet(stopwordsFile);
+ this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+ useDefaultStopPositionIncrement = false;
+ }
+
/** Builds an analyzer with the stop words from the given reader.
* @see WordlistLoader#getWordSet(Reader)
- * @deprecated Use {@link #StopAnalyzer(Reader, boolean)} instead
+ * @deprecated Use {@link #StopAnalyzer(Version, Reader)} instead
*/
public StopAnalyzer(Reader stopwords) throws IOException {
stopWords = WordlistLoader.getWordSet(stopwords);
@@ -149,13 +189,24 @@
* @see WordlistLoader#getWordSet(Reader)
* @param stopwords Reader to load stop words from
* @param enablePositionIncrements See {@link
- * StopFilter#setEnablePositionIncrements} */
+ * StopFilter#setEnablePositionIncrements}
+ * @deprecated Use {@link #StopAnalyzer(Version, Reader)} instead */
public StopAnalyzer(Reader stopwords, boolean enablePositionIncrements) throws IOException {
stopWords = WordlistLoader.getWordSet(stopwords);
this.enablePositionIncrements = enablePositionIncrements;
useDefaultStopPositionIncrement = false;
}
+ /** Builds an analyzer with the stop words from the given reader.
+ * @see WordlistLoader#getWordSet(Reader)
+ * @param matchVersion See <a href="#version">above</a>
+ * @param stopwords Reader to load stop words from */
+ public StopAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
+ stopWords = WordlistLoader.getWordSet(stopwords);
+ this.enablePositionIncrements = StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion);
+ useDefaultStopPositionIncrement = false;
+ }
+
/** Filters LowerCaseTokenizer with StopFilter. */
public TokenStream tokenStream(String fieldName, Reader reader) {
if (useDefaultStopPositionIncrement) {
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/StopFilter.java Fri Oct 23 17:18:53 2009
@@ -25,6 +25,7 @@
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.queryParser.QueryParser; // for javadoc
+import org.apache.lucene.util.Version;
/**
* Removes stop words from a token stream.
@@ -243,6 +244,21 @@
}
/**
+ * Returns version-dependent default for
+ * enablePositionIncrements. Analyzers that embed
+ * StopFilter use this method when creating the
+ * StopFilter. Prior to 2.9, this returns false. On 2.9
+ * or later, it returns true.
+ */
+ public static boolean getEnablePositionIncrementsVersionDefault(Version matchVersion) {
+ if (matchVersion.onOrAfter(Version.LUCENE_29)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
* Set the default position increments behavior of every StopFilter created from now on.
* <p>
* Note: behavior of a single StopFilter instance can be modified
Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=829134&r1=829133&r2=829134&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Fri Oct 23 17:18:53 2009
@@ -35,7 +35,7 @@
* compatibility when creating StandardAnalyzer:
* <ul>
* <li> As of 2.9, StopFilter preserves position
- * increments by default
+ * increments
* <li> As of 2.4, Tokens incorrectly identified as acronyms
* are corrected (see <a href="https://issues.apache.org/jira/browse/LUCENE-1068">LUCENE-1608</a>
* </ul>