You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/10/01 05:05:07 UTC
svn commit: r1177888 [10/16] - in /lucene/dev/branches/lucene2621: ./
dev-tools/eclipse/ dev-tools/idea/lucene/contrib/ dev-tools/maven/ lucene/
lucene/contrib/ lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/demo/src/java/org/apach...
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Sat Oct 1 03:04:53 2011
@@ -192,108 +192,108 @@ DOUBLE_EQUALS = "="{2}
//First {ALPHANUM} is always the link, set positioninc to 1 for double bracket, but then inside the internal link state
//set it to 0 for the next token, such that the link and the first token are in the same position, but then subsequent
//tokens within the link are incremented
- {DOUBLE_BRACKET} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);}
- {DOUBLE_BRACKET_CAT} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);}
- {EXTERNAL_LINK} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);}
- {TWO_SINGLE_QUOTES} {numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}}
- {DOUBLE_EQUALS} {numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);}
- {DOUBLE_BRACE} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);}
- {CITATION} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);}
+ {DOUBLE_BRACKET} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = INTERNAL_LINK; yybegin(INTERNAL_LINK_STATE);/* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACKET_CAT} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CATEGORY; yybegin(CATEGORY_STATE);/* Break so we don't hit fall-through warning: */ break;}
+ {EXTERNAL_LINK} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = EXTERNAL_LINK_URL; yybegin(EXTERNAL_LINK_STATE);/* Break so we don't hit fall-through warning: */ break;}
+ {TWO_SINGLE_QUOTES} {numWikiTokensSeen = 0; positionInc = 1; if (numBalanced == 0){numBalanced++;yybegin(TWO_SINGLE_QUOTES_STATE);} else{numBalanced = 0;}/* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_EQUALS} {numWikiTokensSeen = 0; positionInc = 1; yybegin(DOUBLE_EQUALS_STATE);/* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACE} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);/* Break so we don't hit fall-through warning: */ break;}
+ {CITATION} {numWikiTokensSeen = 0; positionInc = 1; currentTokType = CITATION; yybegin(DOUBLE_BRACE_STATE);/* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} |{INFOBOX} {numWikiTokensSeen = 0; positionInc = 1; }
+ . | {WHITESPACE} |{INFOBOX} {numWikiTokensSeen = 0; positionInc = 1; /* Break so we don't hit fall-through warning: */ break;}
}
<INTERNAL_LINK_STATE>{
//First {ALPHANUM} is always the link, set position to 0 for these
//This is slightly different from EXTERNAL_LINK_STATE because that one has an explicit grammar for capturing the URL
{ALPHANUM} {yybegin(INTERNAL_LINK_STATE); numWikiTokensSeen++; return currentTokType;}
- {DOUBLE_BRACKET_CLOSE} {numLinkToks = 0; yybegin(YYINITIAL);}
+ {DOUBLE_BRACKET_CLOSE} {numLinkToks = 0; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { positionInc = 1; }
+ . | {WHITESPACE} { positionInc = 1; /* Break so we don't hit fall-through warning: */ break;}
}
<EXTERNAL_LINK_STATE>{
//increment the link token, but then don't increment the tokens after that which are still in the link
("http://"|"https://"){HOST}("/"?({ALPHANUM}|{P}|\?|"&"|"="|"#")*)* {positionInc = 1; numWikiTokensSeen++; yybegin(EXTERNAL_LINK_STATE); return currentTokType;}
{ALPHANUM} {if (numLinkToks == 0){positionInc = 0;} else{positionInc = 1;} numWikiTokensSeen++; currentTokType = EXTERNAL_LINK; yybegin(EXTERNAL_LINK_STATE); numLinkToks++; return currentTokType;}
- "]" {numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL);}
- {WHITESPACE} { positionInc = 1; }
+ "]" {numLinkToks = 0; positionInc = 0; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;}
+ {WHITESPACE} { positionInc = 1; /* Break so we don't hit fall-through warning: */ break;}
}
<CATEGORY_STATE>{
{ALPHANUM} {yybegin(CATEGORY_STATE); numWikiTokensSeen++; return currentTokType;}
- {DOUBLE_BRACKET_CLOSE} {yybegin(YYINITIAL);}
+ {DOUBLE_BRACKET_CLOSE} {yybegin(YYINITIAL);/* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { positionInc = 1; }
+ . | {WHITESPACE} { positionInc = 1; /* Break so we don't hit fall-through warning: */ break;}
}
//italics
<TWO_SINGLE_QUOTES_STATE>{
- "'" {currentTokType = BOLD; yybegin(THREE_SINGLE_QUOTES_STATE);}
- "'''" {currentTokType = BOLD_ITALICS; yybegin(FIVE_SINGLE_QUOTES_STATE);}
+ "'" {currentTokType = BOLD; yybegin(THREE_SINGLE_QUOTES_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ "'''" {currentTokType = BOLD_ITALICS; yybegin(FIVE_SINGLE_QUOTES_STATE); /* Break so we don't hit fall-through warning: */ break;}
{ALPHANUM} {currentTokType = ITALICS; numWikiTokensSeen++; yybegin(STRING); return currentTokType;/*italics*/}
//we can have links inside, let those override
- {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);}
- {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);}
- {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);}
+ {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { /* ignore */ }
+ . | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
}
//bold
<THREE_SINGLE_QUOTES_STATE>{
{ALPHANUM} {yybegin(STRING); numWikiTokensSeen++; return currentTokType;}
//we can have links inside, let those override
- {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);}
- {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);}
- {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);}
+ {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { /* ignore */ }
+ . | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
}
//bold italics
<FIVE_SINGLE_QUOTES_STATE>{
{ALPHANUM} {yybegin(STRING); numWikiTokensSeen++; return currentTokType;}
//we can have links inside, let those override
- {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE);}
- {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE);}
- {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE);}
+ {DOUBLE_BRACKET} {currentTokType = INTERNAL_LINK; numWikiTokensSeen = 0; yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACKET_CAT} {currentTokType = CATEGORY; numWikiTokensSeen = 0; yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {EXTERNAL_LINK} {currentTokType = EXTERNAL_LINK; numWikiTokensSeen = 0; yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { /* ignore */ }
+ . | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
}
<DOUBLE_EQUALS_STATE>{
- "=" {currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING);}
+ "=" {currentTokType = SUB_HEADING; numWikiTokensSeen = 0; yybegin(STRING); /* Break so we don't hit fall-through warning: */ break;}
{ALPHANUM} {currentTokType = HEADING; yybegin(DOUBLE_EQUALS_STATE); numWikiTokensSeen++; return currentTokType;}
- {DOUBLE_EQUALS} {yybegin(YYINITIAL);}
+ {DOUBLE_EQUALS} {yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { /* ignore */ }
+ . | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
}
<DOUBLE_BRACE_STATE>{
{ALPHANUM} {yybegin(DOUBLE_BRACE_STATE); numWikiTokensSeen = 0; return currentTokType;}
- {DOUBLE_BRACE_CLOSE} {yybegin(YYINITIAL);}
- {CITATION_CLOSE} {yybegin(YYINITIAL);}
+ {DOUBLE_BRACE_CLOSE} {yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;}
+ {CITATION_CLOSE} {yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;}
//ignore
- . | {WHITESPACE} { /* ignore */ }
+ . | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
}
<STRING> {
- "'''''" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end bold italics*/}
- "'''" {numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL);/*end bold*/}
- "''" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end italics*/}
- "===" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL);/*end sub header*/}
+ "'''''" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end bold italics*/}
+ "'''" {numBalanced = 0;currentTokType = ALPHANUM;yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end bold*/}
+ "''" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end italics*/}
+ "===" {numBalanced = 0;currentTokType = ALPHANUM; yybegin(YYINITIAL); /* Break so we don't hit fall-through warning: */ break;/*end sub header*/}
{ALPHANUM} {yybegin(STRING); numWikiTokensSeen++; return currentTokType;/* STRING ALPHANUM*/}
//we can have links inside, let those override
- {DOUBLE_BRACKET} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE);}
- {DOUBLE_BRACKET_CAT} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE);}
- {EXTERNAL_LINK} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE);}
+ {DOUBLE_BRACKET} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = INTERNAL_LINK;yybegin(INTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {DOUBLE_BRACKET_CAT} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = CATEGORY;yybegin(CATEGORY_STATE); /* Break so we don't hit fall-through warning: */ break;}
+ {EXTERNAL_LINK} {numBalanced = 0; numWikiTokensSeen = 0; currentTokType = EXTERNAL_LINK;yybegin(EXTERNAL_LINK_STATE); /* Break so we don't hit fall-through warning: */ break;}
{PIPE} {yybegin(STRING); return currentTokType;/*pipe*/}
- .|{WHITESPACE} { /* ignore STRING */ }
+ .|{WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore STRING */ }
}
@@ -315,7 +315,7 @@ DOUBLE_EQUALS = "="{2}
//end wikipedia
/** Ignore the rest */
-. | {WHITESPACE}|{TAGS} { /* ignore */ }
+. | {WHITESPACE}|{TAGS} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
//INTERNAL_LINK = "["{2}({ALPHANUM}+{WHITESPACE}*)+"]"{2}
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -18,8 +18,8 @@ package org.apache.lucene.collation;
*/
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.IndexableBinaryStringTools; // javadoc @link
import org.apache.lucene.util.Version;
@@ -82,7 +82,7 @@ import java.io.Reader;
* versions will encode the bytes with {@link IndexableBinaryStringTools}.
* </ul>
*/
-public final class CollationKeyAnalyzer extends ReusableAnalyzerBase {
+public final class CollationKeyAnalyzer extends Analyzer {
private final Collator collator;
private final CollationAttributeFactory factory;
private final Version matchVersion;
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java Sat Oct 1 03:04:53 2011
@@ -64,7 +64,7 @@ public class TestChineseTokenizer extend
* Analyzer that just uses ChineseTokenizer, not ChineseFilter.
* convenience to show the behavior of the tokenizer
*/
- private class JustChineseTokenizerAnalyzer extends ReusableAnalyzerBase {
+ private class JustChineseTokenizerAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
return new TokenStreamComponents(new ChineseTokenizer(reader));
@@ -75,7 +75,7 @@ public class TestChineseTokenizer extend
* Analyzer that just uses ChineseFilter, not ChineseTokenizer.
* convenience to show the behavior of the filter.
*/
- private class JustChineseFilterAnalyzer extends ReusableAnalyzerBase {
+ private class JustChineseFilterAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_CURRENT, reader);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java Sat Oct 1 03:04:53 2011
@@ -84,7 +84,7 @@ public class CommonGramsFilterTest exten
* @return Map<String,String>
*/
public void testCommonGramsQueryFilter() throws Exception {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String field, Reader in) {
Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
@@ -154,7 +154,7 @@ public class CommonGramsFilterTest exten
}
public void testCommonGramsFilter() throws Exception {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String field, Reader in) {
Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Sat Oct 1 03:04:53 2011
@@ -117,7 +117,7 @@ public class TestAnalyzers extends BaseT
String[] y = StandardTokenizer.TOKEN_TYPES;
}
- private static class LowerCaseWhitespaceAnalyzer extends ReusableAnalyzerBase {
+ private static class LowerCaseWhitespaceAnalyzer extends Analyzer {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -48,8 +48,8 @@ public class TestKeywordAnalyzer extends
TEST_VERSION_CURRENT, new SimpleAnalyzer(TEST_VERSION_CURRENT)));
Document doc = new Document();
- doc.add(new Field("partnum", StringField.TYPE_STORED, "Q36"));
- doc.add(new Field("description", TextField.TYPE_STORED, "Illidium Space Modulator"));
+ doc.add(new Field("partnum", "Q36", StringField.TYPE_STORED));
+ doc.add(new Field("description", "Illidium Space Modulator", TextField.TYPE_STORED));
writer.addDocument(doc);
writer.close();
@@ -76,10 +76,10 @@ public class TestKeywordAnalyzer extends
RAMDirectory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer()));
Document doc = new Document();
- doc.add(new Field("partnum", TextField.TYPE_STORED, "Q36"));
+ doc.add(new Field("partnum", "Q36", TextField.TYPE_STORED));
writer.addDocument(doc);
doc = new Document();
- doc.add(new Field("partnum", TextField.TYPE_STORED, "Q37"));
+ doc.add(new Field("partnum", "Q37", TextField.TYPE_STORED));
writer.addDocument(doc);
writer.close();
@@ -100,6 +100,7 @@ public class TestKeywordAnalyzer extends
public void testOffsets() throws Exception {
TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"));
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
+ stream.reset();
assertTrue(stream.incrementToken());
assertEquals(0, offsetAtt.startOffset());
assertEquals(4, offsetAtt.endOffset());
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -5,7 +5,6 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.Version;
import java.io.IOException;
@@ -43,7 +42,7 @@ public class TestStandardAnalyzer extend
BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
- private Analyzer a = new ReusableAnalyzerBase() {
+ private Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents
(String fieldName, Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.core;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.Version;
@@ -51,6 +50,7 @@ public class TestStopAnalyzer extends Ba
TokenStream stream = stop.tokenStream("test", reader);
assertTrue(stream != null);
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
+ stream.reset();
while (stream.incrementToken()) {
assertFalse(inValidTokens.contains(termAtt.toString()));
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java Sat Oct 1 03:04:53 2011
@@ -8,7 +8,6 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.Version;
import java.io.BufferedReader;
@@ -50,7 +49,7 @@ public class TestUAX29URLEmailTokenizer
BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
- private Analyzer a = new ReusableAnalyzerBase() {
+ private Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents
(String fieldName, Reader reader) {
@@ -99,7 +98,7 @@ public class TestUAX29URLEmailTokenizer
}
}
- private Analyzer urlAnalyzer = new ReusableAnalyzerBase() {
+ private Analyzer urlAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, reader);
@@ -109,7 +108,7 @@ public class TestUAX29URLEmailTokenizer
}
};
- private Analyzer emailAnalyzer = new ReusableAnalyzerBase() {
+ private Analyzer emailAnalyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, reader);
@@ -431,7 +430,7 @@ public class TestUAX29URLEmailTokenizer
/** @deprecated remove this and sophisticated backwards layer in 5.0 */
@Deprecated
public void testCombiningMarksBackwards() throws Exception {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents
(String fieldName, Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link GermanLightStemFilter}
*/
public class TestGermanLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link GermanMinimalStemFilter}
*/
public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Sat Oct 1 03:04:53 2011
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -36,7 +35,7 @@ import static org.apache.lucene.analysis
*
*/
public class TestGermanStemFilter extends BaseTokenStreamTestCase {
- Analyzer analyzer = new ReusableAnalyzerBase() {
+ Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,13 +24,12 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Simple tests for {@link EnglishMinimalStemFilter}
*/
public class TestEnglishMinimalStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java Sat Oct 1 03:04:53 2011
@@ -25,13 +25,12 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Tests for {@link KStemmer}
*/
public class TestKStemmer extends BaseTokenStreamTestCase {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
@@ -36,7 +35,7 @@ import static org.apache.lucene.analysis
* Test the PorterStemFilter with Martin Porter's test data.
*/
public class TestPorterStemFilter extends BaseTokenStreamTestCase {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link SpanishLightStemFilter}
*/
public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -203,7 +203,7 @@ public class TestPersianAnalyzer extends
}
/**
- * Basic test ensuring that reusableTokenStream works correctly.
+ * Basic test ensuring that tokenStream works correctly.
*/
public void testReusableTokenStream() throws Exception {
Analyzer a = new PersianAnalyzer(TEST_VERSION_CURRENT);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link FinnishLightStemFilter}
*/
public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link FrenchLightStemFilter}
*/
public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link FrenchMinimalStemFilter}
*/
public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java Sat Oct 1 03:04:53 2011
@@ -28,13 +28,12 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Simple tests for {@link GalicianStemFilter}
*/
public class TestGalicianStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link HungarianLightStemFilter}
*/
public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java Sat Oct 1 03:04:53 2011
@@ -24,14 +24,13 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Tests {@link IndonesianStemmer}
*/
public class TestIndonesianStemmer extends BaseTokenStreamTestCase {
/* full stemming, no stopwords */
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
@@ -112,7 +111,7 @@ public class TestIndonesianStemmer exten
}
/* inflectional-only stemming */
- Analyzer b = new ReusableAnalyzerBase() {
+ Analyzer b = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link ItalianLightStemFilter}
*/
public class TestItalianLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java Sat Oct 1 03:04:53 2011
@@ -24,13 +24,12 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Basic tests for {@link LatvianStemmer}
*/
public class TestLatvianStemmer extends BaseTokenStreamTestCase {
- private Analyzer a = new ReusableAnalyzerBase() {
+ private Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -39,12 +39,11 @@ public class TestLimitTokenCountAnalyzer
Analyzer a = new LimitTokenCountAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 2);
// dont use assertAnalyzesTo here, as the end offset is not the end of the string!
assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, 4);
- assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
a = new LimitTokenCountAnalyzer(new StandardAnalyzer(TEST_VERSION_CURRENT), 2);
// dont use assertAnalyzesTo here, as the end offset is not the end of the string!
assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
- assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
}
public void testLimitTokenCountIndexWriter() throws IOException {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java Sat Oct 1 03:04:53 2011
@@ -37,8 +37,9 @@ public class TestPerFieldAnalzyerWrapper
new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
TokenStream tokenStream = analyzer.tokenStream("field",
- new StringReader(text));
+ new StringReader(text));
CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
+ tokenStream.reset();
assertTrue(tokenStream.incrementToken());
assertEquals("WhitespaceAnalyzer does not lowercase",
@@ -46,8 +47,10 @@ public class TestPerFieldAnalzyerWrapper
termAtt.toString());
tokenStream = analyzer.tokenStream("special",
- new StringReader(text));
+ new StringReader(text));
termAtt = tokenStream.getAttribute(CharTermAttribute.class);
+ tokenStream.reset();
+
assertTrue(tokenStream.incrementToken());
assertEquals("SimpleAnalyzer lowercases",
"qwerty",
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java Sat Oct 1 03:04:53 2011
@@ -213,7 +213,7 @@ public class TestWordDelimiterFilter ext
final CharArraySet protWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<String>(Arrays.asList("NUTCH")), false);
/* analyzer that uses whitespace + wdf */
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String field, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -241,7 +241,7 @@ public class TestWordDelimiterFilter ext
new int[] { 1, 1, 1 });
/* analyzer that will consume tokens with large position increments */
- Analyzer a2 = new ReusableAnalyzerBase() {
+ Analyzer a2 = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String field, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -274,7 +274,7 @@ public class TestWordDelimiterFilter ext
new int[] { 6, 14, 19 },
new int[] { 1, 11, 1 });
- Analyzer a3 = new ReusableAnalyzerBase() {
+ Analyzer a3 = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String field, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -34,7 +33,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link PortugueseLightStemFilter}
*/
public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java Sat Oct 1 03:04:53 2011
@@ -26,7 +26,6 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -34,7 +33,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link PortugueseMinimalStemFilter}
*/
public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java Sat Oct 1 03:04:53 2011
@@ -28,13 +28,12 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
/**
* Simple tests for {@link PortugueseStemFilter}
*/
public class TestPortugueseStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Sat Oct 1 03:04:53 2011
@@ -48,8 +48,8 @@ public class QueryAutoStopWordAnalyzerTe
Document doc = new Document();
String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.length];
- doc.add(new Field("variedField", TextField.TYPE_STORED, variedFieldValue));
- doc.add(new Field("repetitiveField", TextField.TYPE_STORED, repetitiveFieldValue));
+ doc.add(new Field("variedField", variedFieldValue, TextField.TYPE_STORED));
+ doc.add(new Field("repetitiveField", repetitiveFieldValue, TextField.TYPE_STORED));
writer.addDocument(doc);
}
writer.close();
@@ -64,45 +64,45 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoStopwords() throws Exception {
// Note: an empty list of fields passed in
- protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Collections.EMPTY_LIST, 1);
- TokenStream protectedTokenStream = protectedAnalyzer.reusableTokenStream("variedField", new StringReader("quick"));
+ protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Collections.<String>emptyList(), 1);
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("quick"));
assertTokenStreamContents(protectedTokenStream, new String[]{"quick"});
- protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
public void testDefaultStopwordsAllFields() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader);
- TokenStream protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
assertTokenStreamContents(protectedTokenStream, new String[0]); // Default stop word filtering will remove boring
}
public void testStopwordsAllFieldsMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
// A filter on terms in > one half of docs remove boring
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
// A filter on terms in > half of docs should not remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[]{"vaguelyboring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 4f);
- protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
// A filter on terms in > quarter of docs should remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
public void testStopwordsPerFieldMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField"), 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
// A filter on one Field should not affect queries on another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField", "repetitiveField"), 1f / 2f);
- protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
// A filter on the right Field should affect queries on it
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
@@ -120,11 +120,11 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoFieldNamePollution() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("repetitiveField"), 10);
- TokenStream protectedTokenStream = protectedAnalyzer.reusableTokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
// Check filter set up OK
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.reusableTokenStream("variedField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("boring"));
// Filter should not prevent stopwords in one field being used in another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link RussianLightStemFilter}
*/
public class TestRussianLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Sat Oct 1 03:04:53 2011
@@ -56,15 +56,15 @@ public class ShingleAnalyzerWrapperTest
Document doc;
doc = new Document();
- doc.add(new Field("content", TextField.TYPE_STORED, "please divide this sentence into shingles"));
+ doc.add(new Field("content", "please divide this sentence into shingles", TextField.TYPE_STORED));
writer.addDocument(doc);
doc = new Document();
- doc.add(new Field("content", TextField.TYPE_STORED, "just another test sentence"));
+ doc.add(new Field("content", "just another test sentence", TextField.TYPE_STORED));
writer.addDocument(doc);
doc = new Document();
- doc.add(new Field("content", TextField.TYPE_STORED, "a sentence which contains no test"));
+ doc.add(new Field("content", "a sentence which contains no test", TextField.TYPE_STORED));
writer.addDocument(doc);
writer.close();
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Sat Oct 1 03:04:53 2011
@@ -87,14 +87,15 @@ public class TestTeeSinkTokenFilter exte
Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.tokenStream("field", new StringReader("abcd ")));
+ TokenStream tokenStream = analyzer.tokenStream("field", new StringReader("abcd "));
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(tokenStream);
TokenStream sink = tee.newSinkTokenStream();
FieldType ft = new FieldType(TextField.TYPE_UNSTORED);
ft.setStoreTermVectors(true);
ft.setStoreTermVectorOffsets(true);
ft.setStoreTermVectorPositions(true);
- Field f1 = new Field("field", ft, tee);
- Field f2 = new Field("field", ft, sink);
+ Field f1 = new Field("field", tee, ft);
+ Field f2 = new Field("field", sink, ft);
doc.add(f1);
doc.add(f2);
w.addDocument(doc);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java Sat Oct 1 03:04:53 2011
@@ -23,7 +23,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -69,7 +68,7 @@ public class TestSnowballVocab extends L
throws IOException {
if (VERBOSE) System.out.println("checking snowball language: " + snowballLanguage);
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java Sat Oct 1 03:04:53 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -32,7 +31,7 @@ import static org.apache.lucene.analysis
* Simple tests for {@link SwedishLightStemFilter}
*/
public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new ReusableAnalyzerBase() {
+ private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java Sat Oct 1 03:04:53 2011
@@ -27,7 +27,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.junit.Test;
/**
@@ -48,7 +47,7 @@ public class TestSolrSynonymParser exten
parser.add(new StringReader(testFile));
final SynonymMap map = parser.build();
- Analyzer analyzer = new ReusableAnalyzerBase() {
+ Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
@@ -121,7 +120,7 @@ public class TestSolrSynonymParser exten
SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random, MockTokenizer.KEYWORD, false));
parser.add(new StringReader(testFile));
final SynonymMap map = parser.build();
- Analyzer analyzer = new ReusableAnalyzerBase() {
+ Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Sat Oct 1 03:04:53 2011
@@ -33,7 +33,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util._TestUtil;
@@ -387,7 +386,7 @@ public class TestSynonymMapFilter extend
final SynonymMap map = b.build();
final boolean ignoreCase = random.nextBoolean();
- final Analyzer analyzer = new ReusableAnalyzerBase() {
+ final Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
@@ -409,7 +408,7 @@ public class TestSynonymMapFilter extend
parser.add(new StringReader(testFile));
final SynonymMap map = parser.build();
- Analyzer analyzer = new ReusableAnalyzerBase() {
+ Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
@@ -467,7 +466,7 @@ public class TestSynonymMapFilter extend
add("z x c v", "zxcv", keepOrig);
add("x c", "xc", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -507,7 +506,7 @@ public class TestSynonymMapFilter extend
add("a b", "ab", keepOrig);
add("a b", "ab", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -527,7 +526,7 @@ public class TestSynonymMapFilter extend
add("a b", "ab", keepOrig);
add("a b", "ab", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -545,7 +544,7 @@ public class TestSynonymMapFilter extend
final boolean keepOrig = false;
add("zoo", "zoo", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -564,7 +563,7 @@ public class TestSynonymMapFilter extend
add("zoo", "zoo", keepOrig);
add("zoo", "zoo zoo", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -588,7 +587,7 @@ public class TestSynonymMapFilter extend
add("z x c v", "zxcv", keepOrig);
add("x c", "xc", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -633,7 +632,7 @@ public class TestSynonymMapFilter extend
final boolean keepOrig = true;
add("zoo zoo", "zoo", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -652,7 +651,7 @@ public class TestSynonymMapFilter extend
add("zoo zoo", "zoo", keepOrig);
add("zoo", "zoo zoo", keepOrig);
final SynonymMap map = b.build();
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java Sat Oct 1 03:04:53 2011
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
public class TestWordnetSynonymParser extends BaseTokenStreamTestCase {
Analyzer analyzer;
@@ -46,7 +45,7 @@ public class TestWordnetSynonymParser ex
parser.add(new StringReader(synonymsFile));
final SynonymMap map = parser.build();
- Analyzer analyzer = new ReusableAnalyzerBase() {
+ Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -156,10 +156,10 @@ public class TestThaiAnalyzer extends Ba
assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_30);
// just consume
- TokenStream ts = analyzer.reusableTokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ TokenStream ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
// this consumer adds flagsAtt, which this analyzer does not use.
- ts = analyzer.reusableTokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
ts.addAttribute(FlagsAttribute.class);
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
}
Modified: lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java Sat Oct 1 03:04:53 2011
@@ -52,7 +52,7 @@ public class TestCollationKeyFilter exte
(collator.getCollationKey(secondRangeEndOriginal).toByteArray()));
- public final class TestAnalyzer extends ReusableAnalyzerBase {
+ public final class TestAnalyzer extends Analyzer {
private Collator _collator;
TestAnalyzer(Collator collator) {
Modified: lucene/dev/branches/lucene2621/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java Sat Oct 1 03:04:53 2011
@@ -19,8 +19,8 @@ package org.apache.lucene.collation;
import com.ibm.icu.text.Collator;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.collation.CollationKeyAnalyzer; // javadocs
import org.apache.lucene.util.IndexableBinaryStringTools; // javadocs
import org.apache.lucene.util.Version;
@@ -75,7 +75,7 @@ import java.io.Reader;
* versions will encode the bytes with {@link IndexableBinaryStringTools}.
* </ul>
*/
-public final class ICUCollationKeyAnalyzer extends ReusableAnalyzerBase {
+public final class ICUCollationKeyAnalyzer extends Analyzer {
private final Collator collator;
private final ICUCollationAttributeFactory factory;
private final Version matchVersion;
Modified: lucene/dev/branches/lucene2621/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java?rev=1177888&r1=1177887&r2=1177888&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (original)
+++ lucene/dev/branches/lucene2621/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java Sat Oct 1 03:04:53 2011
@@ -27,7 +27,7 @@ import org.apache.lucene.analysis.core.W
* Tests ICUFoldingFilter
*/
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
- Analyzer a = new ReusableAnalyzerBase() {
+ Analyzer a = new Analyzer() {
@Override
public TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);