You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/05/16 22:56:59 UTC
svn commit: r944908 - in /lucene/dev/trunk/lucene:
contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/
contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/
contrib/queryparser/src/test/org/apache/lucene/queryParser/standar...
Author: rmuir
Date: Sun May 16 20:56:58 2010
New Revision: 944908
URL: http://svn.apache.org/viewvc?rev=944908&view=rev
Log:
LUCENE-2413: clean up/doc MockAnalyzer, add a MockTokenFilter, which can simulate stopword/lengthfilter/keepfilter,etc
Added:
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java (with props)
Modified:
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java Sun May 16 20:56:58 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.queryParser.ex
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryParser.TestQueryParser;
@@ -46,7 +47,7 @@ public class TestExtendableQueryParser e
public QueryParser getParser(Analyzer a, Extensions extensions)
throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
QueryParser qp = extensions == null ? new ExtendableQueryParser(
TEST_VERSION_CURRENT, "field", a) : new ExtendableQueryParser(
TEST_VERSION_CURRENT, "field", a, extensions);
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Sun May 16 20:56:58 2010
@@ -20,7 +20,6 @@ package org.apache.lucene.queryParser.pr
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
@@ -100,7 +99,7 @@ public class TestPrecedenceQueryParser e
/** Filters MockTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true));
+ return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true));
}
}
@@ -130,7 +129,7 @@ public class TestPrecedenceQueryParser e
public PrecedenceQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a);
qp.setDefaultOperator(PrecedenceQueryParser.OR_OPERATOR);
return qp;
@@ -175,7 +174,7 @@ public class TestPrecedenceQueryParser e
public Query getQueryDOA(String query, Analyzer a)
throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a);
qp.setDefaultOperator(PrecedenceQueryParser.AND_OPERATOR);
return qp.parse(query);
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Sun May 16 20:56:58 2010
@@ -144,7 +144,7 @@ public class TestQPHelper extends Locali
/** Filters MockTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true));
+ return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true));
}
}
@@ -204,7 +204,7 @@ public class TestQPHelper extends Locali
public StandardQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
@@ -294,7 +294,7 @@ public class TestQPHelper extends Locali
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
StandardQueryParser qp = new StandardQueryParser();
qp.setAnalyzer(a);
qp.setDefaultOperator(Operator.AND);
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java Sun May 16 20:56:58 2010
@@ -141,7 +141,7 @@ public class TestQueryParserWrapper exte
/** Filters MockTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true));
+ return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true));
}
}
@@ -219,7 +219,7 @@ public class TestQueryParserWrapper exte
public QueryParserWrapper getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
QueryParserWrapper qp = new QueryParserWrapper("field", a);
qp.setDefaultOperator(QueryParserWrapper.OR_OPERATOR);
return qp;
@@ -304,7 +304,7 @@ public class TestQueryParserWrapper exte
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
QueryParserWrapper qp = new QueryParserWrapper("field", a);
qp.setDefaultOperator(QueryParserWrapper.AND_OPERATOR);
return qp.parse(query);
@@ -554,7 +554,7 @@ public class TestQueryParserWrapper exte
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod());
QueryParserWrapper qp = new QueryParserWrapper("field",
- new MockAnalyzer(MockAnalyzer.SIMPLE, true));
+ new MockAnalyzer(MockTokenizer.SIMPLE, true));
qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod());
@@ -685,7 +685,7 @@ public class TestQueryParserWrapper exte
final String monthField = "month";
final String hourField = "hour";
QueryParserWrapper qp = new QueryParserWrapper("field",
- new MockAnalyzer(MockAnalyzer.SIMPLE, true));
+ new MockAnalyzer(MockTokenizer.SIMPLE, true));
// Don't set any date resolution and verify if DateField is used
assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java Sun May 16 20:56:58 2010
@@ -21,52 +21,72 @@ import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
/**
* Analyzer for testing
*/
-public final class MockAnalyzer extends Analyzer {
- /** Acts Similar to WhitespaceAnalyzer */
- public static final CharacterRunAutomaton WHITESPACE =
- new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton());
- /** Acts Similar to KeywordAnalyzer.
- * TODO: Keyword returns an "empty" token for an empty reader...
- */
- public static final CharacterRunAutomaton KEYWORD =
- new CharacterRunAutomaton(new RegExp(".*").toAutomaton());
- /** Acts like SimpleAnalyzer/LetterTokenizer. */
- // the ugly regex below is Unicode 5.2 [:Letter:]
- public static final CharacterRunAutomaton SIMPLE =
- new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÃ-ÃÃ-öø-ËË-ËË -ˤˬˮͰ-ʹͶͷͺ-ͽÎÎ-ÎÎÎ-ΡΣ-ϵϷ-ÒÒ-Ô¥Ô±-ÕÕÕ¡-Ö×-תװ-ײء-Ùٮٯٱ-ÛÛÛ¥Û¦Û®Û¯Ûº-Û¼Û¿ÜÜ-ܯÝ-Þ¥Þ±ß-ߪߴߵߺà -à à à ¤à ¨à¤-हऽà¥à¥-ॡॱॲॹ-ॿà¦
-à¦à¦à¦à¦-নপ-রলশ-হঽà§à§à§à§-ৡৰৱà¨
-à¨à¨à¨à¨-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹà©-à©à©à©²-à©´àª
-àªàª-àªàª-નપ-રલળવ-હઽà«à« à«¡à¬
-à¬à¬à¬à¬-ନପ-ରଲଳଵ-ହଽààà-à¡à±à®à®
-à®à®-à®à®-à®à®à®à®à®à®à®£à®¤à®¨-பà®
®-ஹà¯à°
-à°à°-à°à°-నప-ళవ-హఽà±à±à± ౡà²
-à²à²-à²à²-ನಪ-ಳವ-ಹಽà³à³ ೡà´
-à´à´-à´à´-നപ-ഹഽൠൡൺ-ൿà¶
-à¶à¶-නඳ-රලà·-à·à¸-ะาำà¹-à¹àºàºàºàºàºàºàºàº-àºàº-àºàº¡-ຣລວສຫàº-ະາຳຽà»-à»à»à»à»à¼à½-à½à½-ཬà¾-à¾á-áªá¿á-áá-áá¡á¥á¦á®-á°áµ-ááá -á
á-áºá¼á-áá-áá-ááá-áá -áá-áá-á°á²-áµá¸-á¾áá-á
á-áá-áá-áá-áá-áá -á´á-á¬á¯-á¿á-áá -áª
á-áá-áá -á±á-áá -á¬á®-á°á-á³ááá -á¡·á¢-ᢨᢪᢰ-ᣵá¤-á¤á¥-á¥á¥°-ᥴá¦-ᦫá§-á§á¨-á¨á¨ -á©áª§á¬
-ᬳá
-áá®-ᮠᮮᮯá°-á°£á±-á±á±-ᱽᳩ-ᳬᳮ-á³±á´-ᶿá¸-á¼á¼-á¼á¼ -á½
á½-á½á½-á½á½á½á½á½-á½½á¾-ᾴᾶ-ᾼιá¿-á¿á¿-á¿á¿-á¿á¿-á¿á¿ -Ῥῲ-ῴῶ-ῼâ±â¿â-ââââ-âââ-ââ¤â¦â¨âª-ââ¯-â¹â¼-â¿â
-â
â
âââ°-Ⱞⰰ-â±â± -ⳤⳫ-â³®â´-ⴥⴰ-ⵥⵯâ¶-â¶â¶ -ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾâ·-â·â·-â·â·-â·â·-â·â¸¯ã
ãã±-ãµã»�
�¼ã-ãã-ãã¡-ãºã¼-ã¿ã
-ãã±-ãã -ã·ã°-ã¿ã-䶵ä¸-é¿ê-êê-ê½ê-êê-êêªê«ê-êê¢-ê®ê¿-êê -ê¥ê-êê¢-êêêê»-ê ê -ê
ê -ê ê -ê ¢ê¡-ꡳê¢-ꢳꣲ-ꣷꣻê¤-ꤥꤰ-ê¥ê¥ -ꥼê¦-ꦲê§ê¨-ꨨê©-ê©ê©-ê©ê© -ꩶꩺêª-ꪯꪱꪵꪶꪹ-ꪽê«ê«ê«-ê«ê¯-ꯢê°-í£í°-íí-í»ï¤-ï¨ï¨°-ï©ï©°-ï«ï¬-ï¬ï¬-ï¬ï¬ï¬-ﬨשׁ-זּטּ-לּמּïïïïï-ï®±ï¯-ï´½ïµ-ï¶ï¶-ï·ï·°-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚï½-ï½ï½¦-ï¾¾ï¿-ï¿ï¿-ï¿ï¿-ï¿ï¿-ï¿ð-ðð-ð¦ð¨-�
�ºð¼ð½ð¿-ðð-ðð-ðºð-ðð -ðð-ðð°-ðð-ðð-ðð -ðð-ðð-ðð -ð
ð ð -ð µð ·ð ¸ð ¼ð ¿-ð¡ð¤-ð¤ð¤ -ð¤¹ð¨ð¨-ð¨ð¨-ð¨ð¨-ð¨³ð© -ð©¼ð¬-ð¬µð-ðð -ð²ð°-ð±ð-ð¯ð-ð®ð-ð®ð-ðð-ðððð¢ð¥ð¦ð©-ð¬ð®-ð¹ð»ð½-ðð
-ð
ð-ðð-ðð-ðð-ð¹ð»-ð¾ð-ððð-ðð-ð¥ð¨-ðð-ðð-ðºð¼-ðð-ð´ð¶-ðð-ð®ð°-ðð-ð¨ð�
�ª-ðð-ðð -ðªðª-ð«´ð¯ -ð¯¨]+").toAutomaton());
-
+public final class MockAnalyzer extends Analyzer {
private final CharacterRunAutomaton runAutomaton;
private final boolean lowerCase;
-
- public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ private final CharacterRunAutomaton filter;
+ private final boolean enablePositionIncrements;
+
+ /**
+ * Creates a new MockAnalyzer.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ * @param filter DFA describing how terms should be filtered (set of stopwords, etc)
+ * @param enablePositionIncrements true if position increments should reflect filtered terms.
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ /**
+ * Creates a new MockAnalyzer, with no filtering.
+ *
+ * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+)
+ * @param lowerCase true if the tokenizer should lowercase terms
+ */
+ public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false);
}
+ /**
+ * Create a Whitespace-lowercasing analyzer with no stopwords removal
+ */
public MockAnalyzer() {
- this(WHITESPACE, true);
+ this(MockTokenizer.WHITESPACE, true);
}
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, runAutomaton, lowerCase);
+ MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ return new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
+ }
+
+ private class SavedStreams {
+ MockTokenizer tokenizer;
+ MockTokenFilter filter;
}
@Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
- MockTokenizer t = (MockTokenizer) getPreviousTokenStream();
- if (t == null) {
- t = new MockTokenizer(reader, runAutomaton, lowerCase);
- setPreviousTokenStream(t);
+ SavedStreams saved = (SavedStreams) getPreviousTokenStream();
+ if (saved == null) {
+ saved = new SavedStreams();
+ saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
+ setPreviousTokenStream(saved);
+ return saved.filter;
} else {
- t.reset(reader);
+ saved.tokenizer.reset(reader);
+ return saved.filter;
}
- return t;
}
}
\ No newline at end of file
Added: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java?rev=944908&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java (added)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java Sun May 16 20:56:58 2010
@@ -0,0 +1,101 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty;
+import static org.apache.lucene.util.automaton.BasicAutomata.makeString;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+
+/**
+ * A tokenfilter for testing that removes terms accepted by a DFA.
+ * <ul>
+ * <li>Union a list of singletons to act like a stopfilter.
+ * <li>Use the complement to act like a keepwordfilter
+ * <li>Use a regex like <code>.{12,}</code> to act like a lengthfilter
+ * </ul>
+ */
+public final class MockTokenFilter extends TokenFilter {
+ /** Empty set of stopwords */
+ public static final CharacterRunAutomaton EMPTY_STOPSET =
+ new CharacterRunAutomaton(makeEmpty());
+
+ /** Set of common english stopwords */
+ public static final CharacterRunAutomaton ENGLISH_STOPSET =
+ new CharacterRunAutomaton(BasicOperations.union(Arrays.asList(
+ makeString("a"), makeString("an"), makeString("and"), makeString("are"),
+ makeString("as"), makeString("at"), makeString("be"), makeString("but"),
+ makeString("by"), makeString("for"), makeString("if"), makeString("in"),
+ makeString("into"), makeString("is"), makeString("it"), makeString("no"),
+ makeString("not"), makeString("of"), makeString("on"), makeString("or"),
+ makeString("such"), makeString("that"), makeString("the"), makeString("their"),
+ makeString("then"), makeString("there"), makeString("these"), makeString("they"),
+ makeString("this"), makeString("to"), makeString("was"), makeString("will"),
+ makeString("with"))));
+
+ private final CharacterRunAutomaton filter;
+ private boolean enablePositionIncrements = false;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+
+ public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ super(input);
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ // return the first non-stop word found
+ int skippedPositions = 0;
+ while (input.incrementToken()) {
+ if (!filter.run(termAtt.buffer(), 0, termAtt.length())) {
+ if (enablePositionIncrements) {
+ posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
+ }
+ return true;
+ }
+ skippedPositions += posIncrAtt.getPositionIncrement();
+ }
+ // reached EOS -- return false
+ return false;
+ }
+
+ /**
+ * @see #setEnablePositionIncrements(boolean)
+ */
+ public boolean getEnablePositionIncrements() {
+ return enablePositionIncrements;
+ }
+
+ /**
+ * If <code>true</code>, this Filter will preserve
+ * positions of the incoming tokens (ie, accumulate and
+ * set position increments of the removed stop tokens).
+ */
+ public void setEnablePositionIncrements(boolean enable) {
+ this.enablePositionIncrements = enable;
+ }
+}
Propchange: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenFilter.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java Sun May 16 20:56:58 2010
@@ -22,11 +22,25 @@ import java.io.Reader;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
/**
* Automaton-based tokenizer for testing. Optionally lowercases.
*/
public class MockTokenizer extends CharTokenizer {
+ /** Acts Similar to WhitespaceTokenizer */
+ public static final CharacterRunAutomaton WHITESPACE =
+ new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton());
+ /** Acts Similar to KeywordTokenizer.
+ * TODO: Keyword returns an "empty" token for an empty reader...
+ */
+ public static final CharacterRunAutomaton KEYWORD =
+ new CharacterRunAutomaton(new RegExp(".*").toAutomaton());
+ /** Acts like LetterTokenizer. */
+ // the ugly regex below is Unicode 5.2 [:Letter:]
+ public static final CharacterRunAutomaton SIMPLE =
+ new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÃ-ÃÃ-öø-ËË-ËË -ˤˬˮͰ-ʹͶͷͺ-ͽÎÎ-ÎÎÎ-ΡΣ-ϵϷ-ÒÒ-Ô¥Ô±-ÕÕÕ¡-Ö×-תװ-ײء-Ùٮٯٱ-ÛÛÛ¥Û¦Û®Û¯Ûº-Û¼Û¿ÜÜ-ܯÝ-Þ¥Þ±ß-ߪߴߵߺà -à à à ¤à ¨à¤-हऽà¥à¥-ॡॱॲॹ-ॿà¦
-à¦à¦à¦à¦-নপ-রলশ-হঽà§à§à§à§-ৡৰৱà¨
-à¨à¨à¨à¨-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹà©-à©à©à©²-à©´àª
-àªàª-àªàª-નપ-રલળવ-હઽà«à« à«¡à¬
-à¬à¬à¬à¬-ନପ-ରଲଳଵ-ହଽààà-à¡à±à®à®
-à®à®-à®à®-à®à®à®à®à®à®à®£à®¤à®¨-பà®
®-ஹà¯à°
-à°à°-à°à°-నప-ళవ-హఽà±à±à± ౡà²
-à²à²-à²à²-ನಪ-ಳವ-ಹಽà³à³ ೡà´
-à´à´-à´à´-നപ-ഹഽൠൡൺ-ൿà¶
-à¶à¶-නඳ-රලà·-à·à¸-ะาำà¹-à¹àºàºàºàºàºàºàºàº-àºàº-àºàº¡-ຣລວສຫàº-ະາຳຽà»-à»à»à»à»à¼à½-à½à½-ཬà¾-à¾á-áªá¿á-áá-áá¡á¥á¦á®-á°áµ-ááá -á
á-áºá¼á-áá-áá-ááá-áá -áá-áá-á°á²-áµá¸-á¾áá-á
á-áá-áá-áá-áá-áá -á´á-á¬á¯-á¿á-áá -áª
á-áá-áá -á±á-áá -á¬á®-á°á-á³ááá -á¡·á¢-ᢨᢪᢰ-ᣵá¤-á¤á¥-á¥á¥°-ᥴá¦-ᦫá§-á§á¨-á¨á¨ -á©áª§á¬
-ᬳá
-áá®-ᮠᮮᮯá°-á°£á±-á±á±-ᱽᳩ-ᳬᳮ-á³±á´-ᶿá¸-á¼á¼-á¼á¼ -á½
á½-á½á½-á½á½á½á½á½-á½½á¾-ᾴᾶ-ᾼιá¿-á¿á¿-á¿á¿-á¿á¿-á¿á¿ -Ῥῲ-ῴῶ-ῼâ±â¿â-ââââ-âââ-ââ¤â¦â¨âª-ââ¯-â¹â¼-â¿â
-â
â
âââ°-Ⱞⰰ-â±â± -ⳤⳫ-â³®â´-ⴥⴰ-ⵥⵯâ¶-â¶â¶ -ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾâ·-â·â·-â·â·-â·â·-â·â¸¯ã
ãã±-ãµã»�
�¼ã-ãã-ãã¡-ãºã¼-ã¿ã
-ãã±-ãã -ã·ã°-ã¿ã-䶵ä¸-é¿ê-êê-ê½ê-êê-êêªê«ê-êê¢-ê®ê¿-êê -ê¥ê-êê¢-êêêê»-ê ê -ê
ê -ê ê -ê ¢ê¡-ꡳê¢-ꢳꣲ-ꣷꣻê¤-ꤥꤰ-ê¥ê¥ -ꥼê¦-ꦲê§ê¨-ꨨê©-ê©ê©-ê©ê© -ꩶꩺêª-ꪯꪱꪵꪶꪹ-ꪽê«ê«ê«-ê«ê¯-ꯢê°-í£í°-íí-í»ï¤-ï¨ï¨°-ï©ï©°-ï«ï¬-ï¬ï¬-ï¬ï¬ï¬-ﬨשׁ-זּטּ-לּמּïïïïï-ï®±ï¯-ï´½ïµ-ï¶ï¶-ï·ï·°-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚï½-ï½ï½¦-ï¾¾ï¿-ï¿ï¿-ï¿ï¿-ï¿ï¿-ï¿ð-ðð-ð¦ð¨-�
�ºð¼ð½ð¿-ðð-ðð-ðºð-ðð -ðð-ðð°-ðð-ðð-ðð -ðð-ðð-ðð -ð
ð ð -ð µð ·ð ¸ð ¼ð ¿-ð¡ð¤-ð¤ð¤ -ð¤¹ð¨ð¨-ð¨ð¨-ð¨ð¨-ð¨³ð© -ð©¼ð¬-ð¬µð-ðð -ð²ð°-ð±ð-ð¯ð-ð®ð-ð®ð-ðð-ðððð¢ð¥ð¦ð©-ð¬ð®-ð¹ð»ð½-ðð
-ð
ð-ðð-ðð-ðð-ð¹ð»-ð¾ð-ððð-ðð-ð¥ð¨-ðð-ðð-ðºð¼-ðð-ð´ð¶-ðð-ð®ð°-ðð-ð¨ð�
�ª-ðð-ðð -ðªðª-ð«´ð¯ -ð¯¨]+").toAutomaton());
+
private final CharacterRunAutomaton runAutomaton;
private final boolean lowerCase;
private int state;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Sun May 16 20:56:58 2010
@@ -1,5 +1,13 @@
package org.apache.lucene.analysis;
+import java.util.Arrays;
+
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.BasicAutomata;
+import org.apache.lucene.util.automaton.BasicOperations;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,6 +27,7 @@ package org.apache.lucene.analysis;
public class TestMockAnalyzer extends BaseTokenStreamTestCase {
+ /** Test a configuration that behaves a lot like WhitespaceAnalyzer */
public void testWhitespace() throws Exception {
Analyzer a = new MockAnalyzer();
assertAnalyzesTo(a, "A bc defg hiJklmn opqrstuv wxy z ",
@@ -29,8 +38,9 @@ public class TestMockAnalyzer extends Ba
new String[] { "break", "on", "whitespace" });
}
+ /** Test a configuration that behaves a lot like SimpleAnalyzer */
public void testSimple() throws Exception {
- Analyzer a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ",
new String[] { "a", "bc", "defg", "hijklmn", "opqrstuv", "wxy", "z" });
assertAnalyzesToReuse(a, "aba4cadaba-Shazam",
@@ -39,8 +49,9 @@ public class TestMockAnalyzer extends Ba
new String[] { "break", "on", "letters" });
}
+ /** Test a configuration that behaves a lot like KeywordAnalyzer */
public void testKeyword() throws Exception {
- Analyzer a = new MockAnalyzer(MockAnalyzer.KEYWORD, false);
+ Analyzer a = new MockAnalyzer(MockTokenizer.KEYWORD, false);
assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ",
new String[] { "a-bc123 defg+hijklmn567opqrstuv78wxy_z " });
assertAnalyzesToReuse(a, "aba4cadaba-Shazam",
@@ -48,4 +59,40 @@ public class TestMockAnalyzer extends Ba
assertAnalyzesToReuse(a, "break+on/Nothing",
new String[] { "break+on/Nothing" });
}
+
+ /** Test a configuration that behaves a lot like StopAnalyzer */
+ public void testStop() throws Exception {
+ Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ assertAnalyzesTo(a, "the quick brown a fox",
+ new String[] { "quick", "brown", "fox" },
+ new int[] { 2, 1, 2 });
+
+ // disable positions
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, false);
+ assertAnalyzesTo(a, "the quick brown a fox",
+ new String[] { "quick", "brown", "fox" },
+ new int[] { 1, 1, 1 });
+ }
+
+ /** Test a configuration that behaves a lot like KeepWordFilter */
+ public void testKeep() throws Exception {
+ CharacterRunAutomaton keepWords =
+ new CharacterRunAutomaton(
+ BasicOperations.complement(
+ Automaton.union(
+ Arrays.asList(BasicAutomata.makeString("foo"), BasicAutomata.makeString("bar")))));
+ Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true, keepWords, true);
+ assertAnalyzesTo(a, "quick foo brown bar bar fox foo",
+ new String[] { "foo", "bar", "bar", "foo" },
+ new int[] { 2, 2, 1, 2 });
+ }
+
+ /** Test a configuration that behaves a lot like LengthFilter */
+ public void testLength() throws Exception {
+ CharacterRunAutomaton length5 = new CharacterRunAutomaton(new RegExp(".{5,}").toAutomaton());
+ Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, true, length5, true);
+ assertAnalyzesTo(a, "ok toolong fine notfine",
+ new String[] { "ok", "fine" },
+ new int[] { 1, 2 });
+ }
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Sun May 16 20:56:58 2010
@@ -101,7 +101,7 @@ public class TestMultiLevelSkipList exte
private static class PayloadAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new PayloadFilter(new MockTokenizer(reader, MockAnalyzer.WHITESPACE, true));
+ return new PayloadFilter(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java Sun May 16 20:56:58 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MockRAMDirectory;
@@ -35,7 +36,7 @@ import java.io.File;
public class TestThreadedOptimize extends LuceneTestCase {
- private static final Analyzer ANALYZER = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ private static final Analyzer ANALYZER = new MockAnalyzer(MockTokenizer.SIMPLE, true);
private final static int NUM_THREADS = 3;
//private final static int NUM_THREADS = 5;
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java Sun May 16 20:56:58 2010
@@ -128,7 +128,7 @@ public class TestQueryParser extends Loc
/** Filters MockTokenizer with StopFilter. */
@Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
- return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true));
+ return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true));
}
}
@@ -158,7 +158,7 @@ public class TestQueryParser extends Loc
public QueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", a);
qp.setDefaultOperator(QueryParser.OR_OPERATOR);
return qp;
@@ -228,7 +228,7 @@ public class TestQueryParser extends Loc
public Query getQueryDOA(String query, Analyzer a)
throws Exception {
if (a == null)
- a = new MockAnalyzer(MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(MockTokenizer.SIMPLE, true);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", a);
qp.setDefaultOperator(QueryParser.AND_OPERATOR);
return qp.parse(query);
@@ -456,7 +456,7 @@ public class TestQueryParser extends Loc
assertQueryEquals("[ a TO z]", null, "[a TO z]");
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod());
- QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockAnalyzer.SIMPLE, true));
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockTokenizer.SIMPLE, true));
qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod());
@@ -579,7 +579,7 @@ public class TestQueryParser extends Loc
final String defaultField = "default";
final String monthField = "month";
final String hourField = "hour";
- QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockAnalyzer.SIMPLE, true));
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockTokenizer.SIMPLE, true));
// Don't set any date resolution and verify if DateField is used
assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java Sun May 16 20:56:58 2010
@@ -22,6 +22,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
@@ -77,7 +78,7 @@ public class TestFuzzyQuery2 extends Luc
int terms = (int) Math.pow(2, bits);
RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(MockAnalyzer.KEYWORD, false),
+ IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(MockTokenizer.KEYWORD, false),
IndexWriter.MaxFieldLength.UNLIMITED);
Document doc = new Document();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java Sun May 16 20:56:58 2010
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
@@ -617,7 +618,7 @@ public class TestMultiTermConstantScore
/* build an index */
RAMDirectory farsiIndex = new RAMDirectory();
IndexWriter writer = new IndexWriter(farsiIndex, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
Document doc = new Document();
doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES,
Field.Index.NOT_ANALYZED));
@@ -657,7 +658,7 @@ public class TestMultiTermConstantScore
/* build an index */
RAMDirectory danishIndex = new RAMDirectory();
IndexWriter writer = new IndexWriter(danishIndex, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
// Danish collation orders the words below in the given order
// (example taken from TestSort.testInternationalSort() ).
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java Sun May 16 20:56:58 2010
@@ -334,7 +334,7 @@ final class TestPayloadAnalyzer extends
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.WHITESPACE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
return new PayloadFilter(result, fieldName);
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermVectors.java Sun May 16 20:56:58 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -44,7 +45,7 @@ public class TestTermVectors extends Luc
protected void setUp() throws Exception {
super.setUp();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
//writer.setUseCompoundFile(true);
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
@@ -96,7 +97,7 @@ public class TestTermVectors extends Luc
public void testTermVectorsFieldOrder() throws IOException {
Directory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
Document doc = new Document();
doc.add(new Field("c", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
doc.add(new Field("a", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
@@ -236,7 +237,7 @@ public class TestTermVectors extends Luc
try {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT,
- new MockAnalyzer(MockAnalyzer.SIMPLE, true))
+ new MockAnalyzer(MockTokenizer.SIMPLE, true))
.setOpenMode(OpenMode.CREATE));
writer.addDocument(testDoc1);
writer.addDocument(testDoc2);
@@ -352,7 +353,7 @@ public class TestTermVectors extends Luc
// Test only a few docs having vectors
public void testRareVectors() throws IOException {
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))
.setOpenMode(OpenMode.CREATE));
for (int i = 0; i < 100; i++) {
Document doc = new Document();
@@ -386,7 +387,7 @@ public class TestTermVectors extends Luc
public void testMixedVectrosVectors() throws IOException {
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
TEST_VERSION_CURRENT,
- new MockAnalyzer(MockAnalyzer.SIMPLE, true)).setOpenMode(OpenMode.CREATE));
+ new MockAnalyzer(MockTokenizer.SIMPLE, true)).setOpenMode(OpenMode.CREATE));
Document doc = new Document();
doc.add(new Field("field", "one",
Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java Sun May 16 20:56:58 2010
@@ -52,7 +52,7 @@ public class PayloadHelper {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
result = new PayloadFilter(result, fieldName);
return result;
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sun May 16 20:56:58 2010
@@ -20,7 +20,6 @@ import java.io.Reader;
import java.util.Collection;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -59,7 +58,7 @@ public class TestPayloadNearQuery extend
private class PayloadAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
result = new PayloadFilter(result, fieldName);
return result;
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Sun May 16 20:56:58 2010
@@ -30,10 +30,8 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.TermSpans;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.index.IndexWriterConfig;
@@ -69,7 +67,7 @@ public class TestPayloadTermQuery extend
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
result = new PayloadFilter(result, fieldName);
return result;
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java Sun May 16 20:56:58 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.search.spans;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
@@ -58,7 +58,7 @@ public class TestBasics extends LuceneTe
super.setUp();
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)));
+ TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)));
//writer.infoStream = System.out;
for (int i = 0; i < 1000; i++) {
Document doc = new Document();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=944908&r1=944907&r2=944908&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Sun May 16 20:56:58 2010
@@ -467,7 +467,7 @@ public class TestPayloadSpans extends Lu
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
result = new PayloadFilter(result, fieldName);
return result;
}
@@ -519,7 +519,7 @@ public class TestPayloadSpans extends Lu
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true);
+ TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
result = new PayloadFilter(result, fieldName);
return result;
}