You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2013/09/19 21:06:38 UTC
svn commit: r1524809 - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/
lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/
lucene/analysis/common/src/test/org/apache/lucene/analysis/c...
Author: hossman
Date: Thu Sep 19 19:06:37 2013
New Revision: 1524809
URL: http://svn.apache.org/r1524809
Log:
LUCENE-5211: Better javadocs and error checking of 'format' option in StopFilterFactory, as well as comments in all snowball formated files about specifying format option
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/danish_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/dutch_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/english_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/finnish_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/french_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/german_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/hungarian_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/italian_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/norwegian_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/portuguese_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/russian_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/spanish_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/swedish_stop.txt
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_da.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_de.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_es.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fi.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fr.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_hu.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_it.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_nl.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_no.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_pt.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_ru.txt
lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_sv.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_da.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_de.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_es.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fi.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fr.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_hu.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_it.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_nl.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_no.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_pt.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_ru.txt
lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_sv.txt
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Sep 19 19:06:37 2013
@@ -78,6 +78,12 @@ API Changes:
with IndexSearcher when an ExecutorService is specified.
(Ryan Ernst, Mike McCandless, Robert Muir)
+Documentation
+
+* LUCENE-5211: Better javadocs and error checking of 'format' option in
+ StopFilterFactory, as well as comments in all snowball formated files
+ about specifying format option. (hossman)
+
Changes in backwards compatibility policy
* LUCENE-5204: Directory doesn't have default implementations for
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopFilterFactory.java Thu Sep 19 19:06:37 2013
@@ -22,22 +22,57 @@ import org.apache.lucene.analysis.util.C
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.WordlistLoader; // jdocs
import java.util.Map;
import java.io.IOException;
/**
* Factory for {@link StopFilter}.
+ *
* <pre class="prettyprint">
* <fieldType name="text_stop" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
* <filter class="solr.StopFilterFactory" ignoreCase="true"
- * words="stopwords.txt"
+ * words="stopwords.txt" format="wordset"
* </analyzer>
* </fieldType></pre>
+ *
+ * <p>
+ * All attributes are optional:
+ * </p>
+ * <ul>
+ * <li><code>ignoreCase</code> defaults to <code>false</code></li>
+ * <li><code>words</code> should be the name of a stopwords file to parse, if not
+ * specified the factory will use {@link StopAnalyzer#ENGLISH_STOP_WORDS_SET}
+ * </li>
+ * <li><code>format</code> defines how the <code>words</code> file will be parsed,
+ * and defaults to <code>wordset</code>. If <code>words</code> is not specified,
+ * then <code>format</code> must not be specified.
+ * </li>
+ * </ul>
+ * <p>
+ * The valid values for the <code>format</code> option are:
+ * </p>
+ * <ul>
+ * <li><code>wordset</code> - This is the default format, which supports one word per
+ * line (including any intra-word whitespace) and allows whole line comments
+ * begining with the "#" character. Blank lines are ignored. See
+ * {@link WordlistLoader#getLines WordlistLoader.getLines} for details.
+ * </li>
+ * <li><code>snowball</code> - This format allows for multiple words specified on each
+ * line, and trailing comments may be specified using the vertical line ("|").
+ * Blank lines are ignored. See
+ * {@link WordlistLoader#getSnowballWordSet WordlistLoader.getSnowballWordSet}
+ * for details.
+ * </li>
+ * </ul>
*/
public class StopFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
+ public static final String FORMAT_WORDSET = "wordset";
+ public static final String FORMAT_SNOWBALL = "snowball";
+
private CharArraySet stopWords;
private final String stopWordFiles;
private final String format;
@@ -48,7 +83,7 @@ public class StopFilterFactory extends T
super(args);
assureMatchVersion();
stopWordFiles = get(args, "words");
- format = get(args, "format");
+ format = get(args, "format", (null == stopWordFiles ? null : FORMAT_WORDSET));
ignoreCase = getBoolean(args, "ignoreCase", false);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
@@ -58,12 +93,17 @@ public class StopFilterFactory extends T
@Override
public void inform(ResourceLoader loader) throws IOException {
if (stopWordFiles != null) {
- if ("snowball".equalsIgnoreCase(format)) {
+ if (FORMAT_WORDSET.equalsIgnoreCase(format)) {
+ stopWords = getWordSet(loader, stopWordFiles, ignoreCase);
+ } else if (FORMAT_SNOWBALL.equalsIgnoreCase(format)) {
stopWords = getSnowballWordSet(loader, stopWordFiles, ignoreCase);
} else {
- stopWords = getWordSet(loader, stopWordFiles, ignoreCase);
+ throw new IllegalArgumentException("Unknown 'format' specified for 'words' file: " + format);
}
} else {
+ if (null != format) {
+ throw new IllegalArgumentException("'format' can not be specified w/o an explicit 'words' file: " + format);
+ }
stopWords = new CharArraySet(luceneMatchVersion, StopAnalyzer.ENGLISH_STOP_WORDS_SET, ignoreCase);
}
}
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/danish_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/danish_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/danish_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/danish_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Danish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/dutch_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/dutch_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/dutch_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/dutch_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Dutch stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/english_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/english_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/english_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/english_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| An English stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/finnish_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/finnish_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/finnish_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/finnish_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| forms of BE
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/french_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/french_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/french_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/french_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A French stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/german_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/german_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/german_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/german_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A German stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/hungarian_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/hungarian_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/hungarian_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/hungarian_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| Hungarian stop word list
| prepared by Anna Tordai
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/italian_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/italian_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/italian_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/italian_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| An Italian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/norwegian_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/norwegian_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/norwegian_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/norwegian_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Norwegian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/portuguese_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/portuguese_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/portuguese_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/portuguese_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Portuguese stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/russian_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/russian_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/russian_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/russian_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| a russian stop word list. comments begin with vertical bar. each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/spanish_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/spanish_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/spanish_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/spanish_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Spanish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/swedish_stop.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/swedish_stop.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/swedish_stop.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/resources/org/apache/lucene/analysis/snowball/swedish_stop.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Swedish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilterFactory.java Thu Sep 19 19:06:37 2013
@@ -57,6 +57,11 @@ public class TestStopFilterFactory exten
assertTrue(words.contains("her"));
assertTrue(words.contains("hers"));
assertTrue(words.contains("herself"));
+
+ // defaults
+ factory = (StopFilterFactory) tokenFilterFactory("Stop");
+ assertEquals(StopAnalyzer.ENGLISH_STOP_WORDS_SET, factory.getStopWords());
+ assertEquals(false, factory.isIgnoreCase());
}
/** Test that bogus arguments result in exception */
@@ -68,4 +73,30 @@ public class TestStopFilterFactory exten
assertTrue(expected.getMessage().contains("Unknown parameters"));
}
}
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusFormats() throws Exception {
+ try {
+ tokenFilterFactory("Stop",
+ "words", "stop-snowball.txt",
+ "format", "bogus");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ String msg = expected.getMessage();
+ assertTrue(msg, msg.contains("Unknown"));
+ assertTrue(msg, msg.contains("format"));
+ assertTrue(msg, msg.contains("bogus"));
+ }
+ try {
+ tokenFilterFactory("Stop",
+ // implicit default words file
+ "format", "bogus");
+ fail();
+ } catch (IllegalArgumentException expected) {
+ String msg = expected.getMessage();
+ assertTrue(msg, msg.contains("can not be specified"));
+ assertTrue(msg, msg.contains("format"));
+ assertTrue(msg, msg.contains("bogus"));
+ }
+ }
}
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_da.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_da.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_da.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_da.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Danish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_de.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_de.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_de.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_de.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A German stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_es.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_es.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_es.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_es.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Spanish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fi.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fi.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fi.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fi.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| forms of BE
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fr.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fr.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fr.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_fr.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A French stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_hu.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_hu.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_hu.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_hu.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| Hungarian stop word list
| prepared by Anna Tordai
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_it.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_it.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_it.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_it.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| An Italian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_nl.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_nl.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_nl.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_nl.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Dutch stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_no.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_no.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_no.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_no.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Norwegian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_pt.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_pt.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_pt.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_pt.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Portuguese stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_ru.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_ru.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_ru.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_ru.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| a russian stop word list. comments begin with vertical bar. each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_sv.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_sv.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_sv.txt (original)
+++ lucene/dev/trunk/solr/example/example-schemaless/solr/collection1/conf/lang/stopwords_sv.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Swedish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_da.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_da.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_da.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_da.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Danish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_de.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_de.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_de.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_de.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A German stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_es.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_es.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_es.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_es.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Spanish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fi.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fi.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fi.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fi.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| forms of BE
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fr.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fr.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fr.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_fr.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A French stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_hu.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_hu.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_hu.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_hu.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| Hungarian stop word list
| prepared by Anna Tordai
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_it.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_it.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_it.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_it.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| An Italian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_nl.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_nl.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_nl.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_nl.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Dutch stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_no.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_no.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_no.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_no.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Norwegian stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_pt.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_pt.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_pt.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_pt.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Portuguese stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_ru.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_ru.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_ru.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_ru.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| a russian stop word list. comments begin with vertical bar. each stop
| word is at the start of a line.
Modified: lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_sv.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_sv.txt?rev=1524809&r1=1524808&r2=1524809&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_sv.txt (original)
+++ lucene/dev/trunk/solr/example/solr/collection1/conf/lang/stopwords_sv.txt Thu Sep 19 19:06:37 2013
@@ -4,6 +4,8 @@
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
+ |
+ | NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
| A Swedish stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.