You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/13 18:57:21 UTC
svn commit: r1102812 [1/2] - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
lucene/contrib/...
Author: rmuir
Date: Fri May 13 16:57:20 2011
New Revision: 1102812
URL: http://svn.apache.org/viewvc?rev=1102812&view=rev
Log:
LUCENE-3064: add checks to MockTokenizer to enforce proper consumption
Added:
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (with props)
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (with props)
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/branches/branch_3x/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java
lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java
lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/similar/TestMoreLikeThis.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Fri May 13 16:57:20 2011
@@ -355,6 +355,7 @@ public class Highlighter
{
try
{
+ tokenStream.end();
tokenStream.close();
}
catch (Exception e)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Fri May 13 16:57:20 2011
@@ -20,9 +20,9 @@ package org.apache.lucene.search.highlig
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -57,7 +57,7 @@ public class HighlighterPhraseTest exten
final String TEXT = "the fox jumped";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@@ -101,7 +101,7 @@ public class HighlighterPhraseTest exten
final String TEXT = "the fox jumped";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@@ -171,7 +171,7 @@ public class HighlighterPhraseTest exten
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),
@@ -214,7 +214,7 @@ public class HighlighterPhraseTest exten
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, TEXT, Store.YES, Index.ANALYZED,
@@ -255,7 +255,7 @@ public class HighlighterPhraseTest exten
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri May 13 16:57:20 2011
@@ -35,9 +35,12 @@ import javax.xml.parsers.DocumentBuilder
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
@@ -108,7 +111,7 @@ public class HighlighterTest extends Bas
"wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
public void testQueryScorerHits() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random, MockAnalyzer.SIMPLE, true);
+ Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, analyzer);
query = qp.parse("\"very long\"");
searcher = new IndexSearcher(ramDir, true);
@@ -217,7 +220,7 @@ public class HighlighterTest extends Bas
String f2c = f2 + ":";
String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2
+ " OR " + f2c + ph2 + ")";
- Analyzer analyzer = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, f1, analyzer);
Query query = qp.parse(q);
@@ -1102,6 +1105,10 @@ public class HighlighterTest extends Bas
}
public void testMaxSizeHighlight() throws Exception {
+ final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, (CharArraySet) StopAnalyzer.ENGLISH_STOP_WORDS_SET, true);
+ // we disable MockTokenizer checks because we will forcefully limit the
+ // tokenstream and call end() before incrementToken() returns false.
+ analyzer.setEnableChecks(false);
TestHighlightRunner helper = new TestHighlightRunner() {
@Override
@@ -1522,64 +1529,64 @@ public class HighlighterTest extends Bas
Highlighter highlighter;
String result;
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("foo");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("10");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("10");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hi");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("speed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hispeed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hi speed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
// ///////////////// same tests, just put the bigger overlapping token
// first
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("foo");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("10");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("10");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hi");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("speed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hispeed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
- query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("hi speed");
+ query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
@@ -1590,7 +1597,7 @@ public class HighlighterTest extends Bas
}
private Directory dir;
- private Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ private Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
public void testWeightedTermsWithDeletes() throws IOException, ParseException, InvalidTokenOffsetsException {
makeIndex();
@@ -1605,7 +1612,7 @@ public class HighlighterTest extends Bas
}
private void makeIndex() throws IOException {
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
writer.addDocument( doc( "t_text1", "more random words for second field del" ) );
writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
@@ -1615,7 +1622,7 @@ public class HighlighterTest extends Bas
}
private void deleteDocument() throws IOException {
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));
writer.deleteDocuments( new Term( "t_text1", "del" ) );
// To see negative idf, keep comment the following line
//writer.optimize();
@@ -1802,6 +1809,11 @@ final class SynonymAnalyzer extends Anal
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
+ try {
+ stream.reset();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
return new SynonymTokenizer(stream, synonyms);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java Fri May 13 16:57:20 2011
@@ -22,6 +22,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -30,34 +31,38 @@ import org.apache.lucene.util.Version;
public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
-
- TokenStream stream = new WhitespaceAnalyzer(Version.LUCENE_CURRENT)
- .tokenStream("field", new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"));
-
+ // we disable MockTokenizer checks because we will forcefully limit the
+ // tokenstream and call end() before incrementToken() returns false.
+ MockTokenizer stream = new MockTokenizer(new StringReader(
+ "short toolong evenmuchlongertext a ab toolong foo"),
+ MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
- stream = new WhitespaceAnalyzer(Version.LUCENE_CURRENT).tokenStream("field", new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"));
-
+ stream = new MockTokenizer(new StringReader(
+ "short toolong evenmuchlongertext a ab toolong foo"),
+ MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 12);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
- stream = new WhitespaceAnalyzer(Version.LUCENE_CURRENT).tokenStream(
- "field", new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"));
-
+ stream = new MockTokenizer(new StringReader(
+ "short toolong evenmuchlongertext a ab toolong foo"),
+ MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 30);
assertTokenStreamContents(filter, new String[] {"short", "toolong",
"evenmuchlongertext"});
-
+ // TODO: This is not actually testing reuse! (reusableTokenStream is not implemented)
checkOneTermReuse(new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new WhitespaceAnalyzer(Version.LUCENE_CURRENT).tokenStream(fieldName, reader);
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false);
+ return new OffsetLimitTokenFilter(tokenizer, 10);
}
}, "llenges", "llenges");
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Fri May 13 16:57:20 2011
@@ -23,6 +23,7 @@ import java.util.Collection;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -86,9 +87,9 @@ public abstract class AbstractTestCase e
@Override
public void setUp() throws Exception {
super.setUp();
- analyzerW = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ analyzerW = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
analyzerB = new BigramAnalyzer();
- analyzerK = new MockAnalyzer(random, MockAnalyzer.KEYWORD, false);
+ analyzerK = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
paW = new QueryParser(TEST_VERSION_CURRENT, F, analyzerW );
paB = new QueryParser(TEST_VERSION_CURRENT, F, analyzerB );
dir = newDirectory();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Fri May 13 16:57:20 2011
@@ -27,6 +27,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -142,9 +143,9 @@ public class MemoryIndexTest extends Bas
*/
private Analyzer randomAnalyzer() {
switch(random.nextInt(3)) {
- case 0: return new MockAnalyzer(random, MockAnalyzer.SIMPLE, true);
+ case 0: return new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
case 1: return new StopAnalyzer(TEST_VERSION_CURRENT);
- default: return new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ default: return new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java Fri May 13 16:57:20 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.junit.AfterClass;
@@ -38,7 +39,7 @@ public class TestHighFreqTerms extends L
public static void setUpClass() throws Exception {
dir = newDirectory();
writer = new IndexWriter(dir, newIndexWriterConfig(random,
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false))
+ TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(2));
indexDocs(writer);
reader = IndexReader.open(dir, true);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri May 13 16:57:20 2011
@@ -190,6 +190,7 @@ public class FuzzyLikeThisQuery extends
int corpusNumDocs=reader.numDocs();
Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet<String> processedTerms=new HashSet<String>();
+ ts.reset();
while (ts.incrementToken())
{
String term = termAtt.toString();
@@ -244,7 +245,9 @@ public class FuzzyLikeThisQuery extends
}
}
}
- }
+ }
+ ts.end();
+ ts.close();
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri May 13 16:57:20 2011
@@ -894,7 +894,7 @@ public final class MoreLikeThis {
int tokenCount=0;
// for every token
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
+ ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
tokenCount++;
@@ -915,6 +915,8 @@ public final class MoreLikeThis {
cnt.x++;
}
}
+ ts.end();
+ ts.close();
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java Fri May 13 16:57:20 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -36,7 +37,7 @@ public class BooleanFilterTest extends L
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random, directory, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ RandomIndexWriter writer = new RandomIndexWriter(random, directory, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
//Add series of docs with filterable fields : acces rights, prices, dates and "in-stock" flags
addDoc(writer, "admin guest", "010", "20040101","Y");
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/similar/TestMoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/similar/TestMoreLikeThis.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/similar/TestMoreLikeThis.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/similar/TestMoreLikeThis.java Fri May 13 16:57:20 2011
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -73,7 +74,7 @@ public class TestMoreLikeThis extends Lu
Map<String,Float> originalValues = getOriginalValues();
MoreLikeThis mlt = new MoreLikeThis(reader);
- mlt.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ mlt.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
mlt.setMinDocFreq(1);
mlt.setMinTermFreq(1);
mlt.setMinWordLen(1);
@@ -108,7 +109,7 @@ public class TestMoreLikeThis extends Lu
private Map<String,Float> getOriginalValues() throws IOException {
Map<String,Float> originalValues = new HashMap<String,Float>();
MoreLikeThis mlt = new MoreLikeThis(reader);
- mlt.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ mlt.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
mlt.setMinDocFreq(1);
mlt.setMinTermFreq(1);
mlt.setMinWordLen(1);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Fri May 13 16:57:20 2011
@@ -110,6 +110,11 @@ public class AnalyzingQueryParser extend
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
int countTokens = 0;
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
while (true) {
try {
if (!source.incrementToken()) break;
@@ -126,6 +131,7 @@ public class AnalyzingQueryParser extend
}
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -191,7 +197,11 @@ public class AnalyzingQueryParser extend
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
List<String> tlist = new ArrayList<String>();
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
while (true) {
try {
if (!source.incrementToken()) break;
@@ -202,6 +212,7 @@ public class AnalyzingQueryParser extend
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -242,6 +253,7 @@ public class AnalyzingQueryParser extend
boolean multipleTokens = false;
try {
+ source.reset();
if (source.incrementToken()) {
nextToken = termAtt.toString();
}
@@ -251,6 +263,7 @@ public class AnalyzingQueryParser extend
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -272,18 +285,37 @@ public class AnalyzingQueryParser extend
protected Query getRangeQuery(String field, String part1, String part2, boolean inclusive)
throws ParseException {
// get Analyzer from superclass and tokenize the terms
- TokenStream source = getAnalyzer().tokenStream(field, new StringReader(part1));
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
+ TokenStream source = null;
+ CharTermAttribute termAtt = null;
boolean multipleTokens = false;
- // part1
- try {
- if (source.incrementToken()) {
- part1 = termAtt.toString();
+ if (part1 != null) {
+ // part1
+ try {
+ source = getAnalyzer().tokenStream(field, new StringReader(part1));
+ termAtt = source.addAttribute(CharTermAttribute.class);
+ source.reset();
+ multipleTokens = false;
+
+
+ if (source.incrementToken()) {
+ part1 = termAtt.toString();
+ }
+ multipleTokens = source.incrementToken();
+ } catch (IOException e) {
+ // ignore
+ }
+
+ try {
+ source.end();
+ source.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ if (multipleTokens) {
+ throw new ParseException("Cannot build RangeQuery with analyzer " + getAnalyzer().getClass()
+ + " - tokens were added to part1");
}
- multipleTokens = source.incrementToken();
- } catch (IOException e) {
- // ignore
}
try {
source.close();
@@ -295,17 +327,30 @@ public class AnalyzingQueryParser extend
+ " - tokens were added to part1");
}
- // part2
- source = getAnalyzer().tokenStream(field, new StringReader(part2));
- termAtt = source.addAttribute(CharTermAttribute.class);
-
- try {
- if (source.incrementToken()) {
- part2 = termAtt.toString();
+ if (part2 != null) {
+ // part2
+ source = getAnalyzer().tokenStream(field, new StringReader(part2));
+ termAtt = source.addAttribute(CharTermAttribute.class);
+
+ try {
+ source.reset();
+ if (source.incrementToken()) {
+ part2 = termAtt.toString();
+ }
+ multipleTokens = source.incrementToken();
+ } catch (IOException e) {
+ // ignore
+ }
+ try {
+ source.end();
+ source.close();
+ } catch (IOException e) {
+ // ignore
+ }
+ if (multipleTokens) {
+ throw new ParseException("Cannot build RangeQuery with analyzer " + getAnalyzer().getClass()
+ + " - tokens were added to part2");
}
- multipleTokens = source.incrementToken();
- } catch (IOException e) {
- // ignore
}
try {
source.close();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Fri May 13 16:57:20 2011
@@ -123,6 +123,11 @@ public class AnalyzerQueryNodeProcessor
TokenStream source = this.analyzer.tokenStream(field, new StringReader(
text));
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
CachingTokenFilter buffer = new CachingTokenFilter(source);
PositionIncrementAttribute posIncrAtt = null;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java Fri May 13 16:57:20 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.queryParser.ex
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
@@ -43,7 +44,7 @@ public class TestExtendableQueryParser e
public QueryParser getParser(Analyzer a, Extensions extensions)
throws Exception {
if (a == null)
- a = new MockAnalyzer(random, MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
QueryParser qp = extensions == null ? new ExtendableQueryParser(
TEST_VERSION_CURRENT, "field", a) : new ExtendableQueryParser(
TEST_VERSION_CURRENT, "field", a, extensions);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Fri May 13 16:57:20 2011
@@ -30,6 +30,7 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -128,7 +129,7 @@ public class TestPrecedenceQueryParser e
public PrecedenceQueryParser getParser(Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(random, MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
PrecedenceQueryParser qp = new PrecedenceQueryParser();
qp.setAnalyzer(a);
qp.setDefaultOperator(Operator.OR);
@@ -174,7 +175,7 @@ public class TestPrecedenceQueryParser e
public Query getQueryDOA(String query, Analyzer a) throws Exception {
if (a == null)
- a = new MockAnalyzer(random, MockAnalyzer.SIMPLE, true);
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
PrecedenceQueryParser qp = new PrecedenceQueryParser();
qp.setAnalyzer(a);
qp.setDefaultOperator(Operator.AND);
@@ -249,7 +250,7 @@ public class TestPrecedenceQueryParser e
}
public void testPunct() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -484,7 +485,7 @@ public class TestPrecedenceQueryParser e
}
public void testEscaped() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("a\\-b:c", a, "a-b:c");
assertQueryEquals("a\\+b:c", a, "a+b:c");
@@ -579,7 +580,7 @@ public class TestPrecedenceQueryParser e
public void testBooleanQuery() throws Exception {
BooleanQuery.setMaxClauseCount(2);
try {
- getParser(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("one two three", "field");
+ getParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("one two three", "field");
fail("ParseException expected due to too many boolean clauses");
} catch (QueryNodeException expected) {
// too many boolean clauses, so ParseException is expected
@@ -588,7 +589,7 @@ public class TestPrecedenceQueryParser e
// LUCENE-792
public void testNOT() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("NOT foo AND bar", a, "-foo +bar");
}
@@ -597,7 +598,7 @@ public class TestPrecedenceQueryParser e
* issue has been corrected.
*/
public void testPrecedence() throws Exception {
- PrecedenceQueryParser parser = getParser(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ PrecedenceQueryParser parser = getParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
Query query1 = parser.parse("A AND B OR C AND D", "field");
Query query2 = parser.parse("(A AND B) OR (C AND D)", "field");
assertEquals(query1, query2);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Fri May 13 16:57:20 2011
@@ -34,6 +34,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.KeywordAnalyzer;
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.StopFilter;
@@ -301,7 +302,7 @@ public class TestQPHelper extends Lucene
}
public void testConstantScoreAutoRewrite() throws Exception {
- StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ StandardQueryParser qp = new StandardQueryParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
Query q = qp.parse("foo*bar", "field");
assertTrue(q instanceof WildcardQuery);
assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((MultiTermQuery) q).getRewriteMethod());
@@ -384,9 +385,9 @@ public class TestQPHelper extends Lucene
public void testSimple() throws Exception {
assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
assertQueryEquals("term term term", null, "term term term");
- assertQueryEquals("t�rm term term", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false),
+ assertQueryEquals("t�rm term term", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false),
"t�rm term term");
- assertQueryEquals("�mlaut", new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false), "�mlaut");
+ assertQueryEquals("�mlaut", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), "�mlaut");
assertQueryEquals("\"\"", new KeywordAnalyzer(), "");
assertQueryEquals("foo:\"\"", new KeywordAnalyzer(), "foo:");
@@ -443,7 +444,7 @@ public class TestQPHelper extends Lucene
}
public void testPunct() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("a&b", a, "a&b");
assertQueryEquals("a&&b", a, "a&&b");
assertQueryEquals(".NET", a, ".NET");
@@ -464,7 +465,7 @@ public class TestQPHelper extends Lucene
assertQueryEquals("term 1.0 1 2", null, "term");
assertQueryEquals("term term1 term2", null, "term term term");
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertQueryEquals("3", a, "3");
assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
assertQueryEquals("term term1 term2", a, "term term1 term2");
@@ -781,7 +782,7 @@ public class TestQPHelper extends Lucene
}
public void testEscaped() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
/*
* assertQueryEquals("\\[brackets", a, "\\[brackets");
@@ -880,7 +881,7 @@ public class TestQPHelper extends Lucene
}
public void testQueryStringEscaping() throws Exception {
- Analyzer a = new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false);
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
@@ -995,7 +996,7 @@ public class TestQPHelper extends Lucene
public void testCustomQueryParserWildcard() {
try {
- new QPTestParser(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("a?t", "contents");
+ new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("a?t", "contents");
fail("Wildcard queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -1004,7 +1005,7 @@ public class TestQPHelper extends Lucene
public void testCustomQueryParserFuzzy() throws Exception {
try {
- new QPTestParser(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)).parse("xunit~", "contents");
+ new QPTestParser(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("xunit~", "contents");
fail("Fuzzy queries should not be allowed");
} catch (QueryNodeException expected) {
// expected exception
@@ -1015,7 +1016,7 @@ public class TestQPHelper extends Lucene
BooleanQuery.setMaxClauseCount(2);
try {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
qp.parse("one two three", "field");
fail("ParseException expected due to too many boolean clauses");
} catch (QueryNodeException expected) {
@@ -1028,7 +1029,7 @@ public class TestQPHelper extends Lucene
*/
public void testPrecedence() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
Query query1 = qp.parse("A AND B OR C AND D", "field");
Query query2 = qp.parse("+A +B +C +D", "field");
@@ -1164,7 +1165,7 @@ public class TestQPHelper extends Lucene
public void testMatchAllDocs() throws Exception {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
assertEquals(new MatchAllDocsQuery(), qp.parse("*:*", "field"));
assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)", "field"));
@@ -1176,7 +1177,7 @@ public class TestQPHelper extends Lucene
private void assertHits(int expected, String query, IndexSearcher is)
throws IOException, QueryNodeException {
StandardQueryParser qp = new StandardQueryParser();
- qp.setAnalyzer(new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false));
+ qp.setAnalyzer(new MockAnalyzer(random, MockTokenizer.WHITESPACE, false));
qp.setLocale(Locale.ENGLISH);
Query q = qp.parse(query, "date");
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Fri May 13 16:57:20 2011
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -46,7 +47,7 @@ public class TestLuceneDictionary extend
public void setUp() throws Exception {
super.setUp();
store = newDirectory();
- IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false)));
+ IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
Document doc;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Fri May 13 16:57:20 2011
@@ -118,12 +118,14 @@ public final class SynExpand {
// [1] Parse query into separate words so that when we expand we can avoid dups
TokenStream ts = a.tokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
+ ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
if ( already.add( word))
top.add( word);
}
+ ts.end();
+ ts.close();
final BooleanQuery tmp = new BooleanQuery();
// [2] form query
Modified: lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Fri May 13 16:57:20 2011
@@ -114,7 +114,6 @@ public class TestSynonymTokenFilter exte
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
- streams.result.reset(); // reset the SynonymTokenFilter
}
return streams.result;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Fri May 13 16:57:20 2011
@@ -80,9 +80,12 @@ public class LikeThisQueryBuilder implem
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
+ ts.reset();
while(ts.incrementToken()) {
stopWordsSet.add(termAtt.toString());
}
+ ts.end();
+ ts.close();
}
catch(IOException ioe)
{
Modified: lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Fri May 13 16:57:20 2011
@@ -58,10 +58,13 @@ public class SpanOrTermsBuilder extends
TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
while (ts.incrementToken()) {
SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.toString()));
clausesList.add(stq);
}
+ ts.end();
+ ts.close();
SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
return soq;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Fri May 13 16:57:20 2011
@@ -62,6 +62,7 @@ public class TermsFilterBuilder implemen
try
{
Term term = null;
+ ts.reset();
while (ts.incrementToken()) {
if (term == null)
{
@@ -73,6 +74,8 @@ public class TermsFilterBuilder implemen
}
tf.addTerm(term);
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Fri May 13 16:57:20 2011
@@ -59,6 +59,7 @@ public class TermsQueryBuilder implement
{
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
Term term = null;
+ ts.reset();
while (ts.incrementToken()) {
if (term == null)
{
@@ -70,6 +71,8 @@ public class TermsQueryBuilder implement
}
bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Fri May 13 16:57:20 2011
@@ -262,6 +262,7 @@ public abstract class BaseTokenStreamTes
tokens.add(termAtt.toString());
// TODO: we could collect offsets etc here for better checking that reset() really works.
}
+ ts.end();
ts.close();
// verify reusing is "reproducable" and also get the normal tokenstream sanity checks
if (!tokens.isEmpty())
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java Fri May 13 16:57:20 2011
@@ -17,58 +17,79 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-
import java.io.IOException;
import java.io.Reader;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import org.apache.lucene.util.LuceneTestCase;
+
/**
* Analyzer for testing
*/
-public final class MockAnalyzer extends Analyzer {
- private boolean lowerCase;
- public static final int KEYWORD = 0;
- public static final int WHITESPACE = 1;
- public static final int SIMPLE = 2;
- private int tokenizer;
+public final class MockAnalyzer extends Analyzer {
+ private final int pattern;
+ private final boolean lowerCase;
+ private final CharArraySet filter;
+ private final boolean enablePositionIncrements;
+ private int positionIncrementGap;
private final Random random;
private Map<String,Integer> previousMappings = new HashMap<String,Integer>();
-
- public MockAnalyzer(Random random) {
- this(random, WHITESPACE, true);
- }
-
- public MockAnalyzer(Random random, int tokenizer, boolean lowercase) {
- this.tokenizer = tokenizer;
- this.lowerCase = lowercase;
+ private boolean enableChecks = true;
+
+ /**
+ * Creates a new MockAnalyzer.
+ *
+ * @param random Random for payloads behavior
+ * @param pattern pattern constant describing how tokenization should happen
+ * @param lowerCase true if the tokenizer should lowercase terms
+ * @param filter CharArraySet describing how terms should be filtered (set of stopwords, etc)
+ * @param enablePositionIncrements true if position increments should reflect filtered terms.
+ */
+ public MockAnalyzer(Random random, int pattern, boolean lowerCase, CharArraySet filter, boolean enablePositionIncrements) {
this.random = random;
+ this.pattern = pattern;
+ this.lowerCase = lowerCase;
+ this.filter = filter;
+ this.enablePositionIncrements = enablePositionIncrements;
+ }
+
+ /**
+ * Calls {@link #MockAnalyzer(Random, int, boolean, CharArraySet, boolean)
+ * MockAnalyzer(random, pattern, lowerCase, CharArraySet.EMPTY_STOPSET, false}).
+ */
+ public MockAnalyzer(Random random, int pattern, boolean lowerCase) {
+ this(random, pattern, lowerCase, CharArraySet.EMPTY_SET, false);
+ }
+
+ /**
+ * Create a Whitespace-lowercasing analyzer with no stopwords removal.
+ * <p>
+ * Calls {@link #MockAnalyzer(Random, int, boolean)
+ * MockAnalyzer(random, MockTokenizer.WHITESPACE, true)}.
+ */
+ public MockAnalyzer(Random random) {
+ this(random, MockTokenizer.WHITESPACE, true);
}
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result;
- if (tokenizer == KEYWORD)
- result = new KeywordTokenizer(reader);
- else if (tokenizer == SIMPLE)
- result = new LetterTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
- else
- result = new WhitespaceTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
- if (lowerCase)
- result = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, result);
- result = maybePayload(result, fieldName);
- return result;
+ MockTokenizer tokenizer = new MockTokenizer(reader, pattern, lowerCase);
+ tokenizer.setEnableChecks(enableChecks);
+ StopFilter filt = new StopFilter(LuceneTestCase.TEST_VERSION_CURRENT, tokenizer, filter);
+ filt.setEnablePositionIncrements(enablePositionIncrements);
+ return maybePayload(filt, fieldName);
}
private class SavedStreams {
- Tokenizer upstream;
- TokenStream filter;
+ MockTokenizer tokenizer;
+ TokenFilter filter;
}
@Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
@SuppressWarnings("unchecked") Map<String,SavedStreams> map = (Map) getPreviousTokenStream();
if (map == null) {
map = new HashMap<String,SavedStreams>();
@@ -76,29 +97,23 @@ public final class MockAnalyzer extends
}
SavedStreams saved = map.get(fieldName);
- if (saved == null){
+ if (saved == null) {
saved = new SavedStreams();
- if (tokenizer == KEYWORD)
- saved.upstream = new KeywordTokenizer(reader);
- else if (tokenizer == SIMPLE)
- saved.upstream = new LetterTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
- else
- saved.upstream = new WhitespaceTokenizer(LuceneTestCase.TEST_VERSION_CURRENT, reader);
- saved.filter = saved.upstream;
- if (lowerCase)
- saved.filter = new LowerCaseFilter(LuceneTestCase.TEST_VERSION_CURRENT, saved.filter);
-
+ saved.tokenizer = new MockTokenizer(reader, pattern, lowerCase);
+ saved.tokenizer.setEnableChecks(enableChecks);
+ StopFilter filt = new StopFilter(LuceneTestCase.TEST_VERSION_CURRENT, saved.tokenizer, filter);
+ filt.setEnablePositionIncrements(enablePositionIncrements);
+ saved.filter = filt;
saved.filter = maybePayload(saved.filter, fieldName);
map.put(fieldName, saved);
return saved.filter;
} else {
- saved.upstream.reset(reader);
- saved.filter.reset();
+ saved.tokenizer.reset(reader);
return saved.filter;
- }
+ }
}
- private synchronized TokenStream maybePayload(TokenStream stream, String fieldName) {
+ private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) {
Integer val = previousMappings.get(fieldName);
if (val == null) {
switch(random.nextInt(3)) {
@@ -119,4 +134,21 @@ public final class MockAnalyzer extends
else
return new MockFixedLengthPayloadFilter(random, stream, val);
}
+
+ public void setPositionIncrementGap(int positionIncrementGap){
+ this.positionIncrementGap = positionIncrementGap;
+ }
+
+ @Override
+ public int getPositionIncrementGap(String fieldName){
+ return positionIncrementGap;
+ }
+
+ /**
+ * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+ * should leave this enabled.
+ */
+ public void setEnableChecks(boolean enableChecks) {
+ this.enableChecks = enableChecks;
+ }
}
Added: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java?rev=1102812&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (added)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java Fri May 13 16:57:20 2011
@@ -0,0 +1,177 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
+/**
+ * tokenizer for testing. Optionally lowercases.
+ */
+public class MockTokenizer extends Tokenizer {
+ /** Acts Similar to WhitespaceTokenizer */
+ public static final int WHITESPACE = 0;
+ /** Acts Similar to KeywordTokenizer.
+ * TODO: Keyword returns an "empty" token for an empty reader...
+ */
+ public static final int KEYWORD = 1;
+ /** Acts like LetterTokenizer. */
+ public static final int SIMPLE = 2;
+
+ private final int pattern;
+ private final boolean lowerCase;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ int off = 0;
+
+ // TODO: "register" with LuceneTestCase to ensure all streams are closed() ?
+ // currently, we can only check that the lifecycle is correct if someone is reusing,
+ // but not for "one-offs".
+ private static enum State {
+ SETREADER, // consumer set a reader input either via ctor or via reset(Reader)
+ RESET, // consumer has called reset()
+ INCREMENT, // consumer is consuming, has called incrementToken() == true
+ INCREMENT_FALSE, // consumer has called incrementToken() which returned false
+ END, // consumer has called end() to perform end of stream operations
+ CLOSE // consumer has called close() to release any resources
+ };
+
+ private State streamState = State.CLOSE;
+ private boolean enableChecks = true;
+
+ public MockTokenizer(AttributeFactory factory, Reader input, int pattern, boolean lowerCase) {
+ super(factory, input);
+ this.pattern = pattern;
+ this.lowerCase = lowerCase;
+ this.streamState = State.SETREADER;
+ }
+
+ public MockTokenizer(Reader input, int pattern, boolean lowerCase) {
+ super(input);
+ this.pattern = pattern;
+ this.lowerCase = lowerCase;
+ this.streamState = State.SETREADER;
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)
+ : "incrementToken() called while in wrong state: " + streamState;
+ clearAttributes();
+ for (;;) {
+ int startOffset = off;
+ int cp = readCodePoint();
+ if (cp < 0) {
+ break;
+ } else if (isTokenChar(cp)) {
+ int endOffset;
+ do {
+ char chars[] = Character.toChars(normalize(cp));
+ for (int i = 0; i < chars.length; i++)
+ termAtt.append(chars[i]);
+ endOffset = off;
+ cp = readCodePoint();
+ } while (cp >= 0 && isTokenChar(cp));
+ offsetAtt.setOffset(startOffset, endOffset);
+ streamState = State.INCREMENT;
+ return true;
+ }
+ }
+ streamState = State.INCREMENT_FALSE;
+ return false;
+ }
+
+ protected int readCodePoint() throws IOException {
+ int ch = input.read();
+ if (ch < 0) {
+ return ch;
+ } else {
+ assert !Character.isLowSurrogate((char) ch);
+ off++;
+ if (Character.isHighSurrogate((char) ch)) {
+ int ch2 = input.read();
+ if (ch2 >= 0) {
+ off++;
+ assert Character.isLowSurrogate((char) ch2);
+ return Character.toCodePoint((char) ch, (char) ch2);
+ }
+ }
+ return ch;
+ }
+ }
+
+ protected boolean isTokenChar(int c) {
+ switch(pattern) {
+ case WHITESPACE: return !Character.isWhitespace(c);
+ case KEYWORD: return true;
+ case SIMPLE: return Character.isLetter(c);
+ default: throw new RuntimeException("invalid pattern constant:" + pattern);
+ }
+ }
+
+ protected int normalize(int c) {
+ return lowerCase ? Character.toLowerCase(c) : c;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ off = 0;
+ assert !enableChecks || streamState != State.RESET : "double reset()";
+ streamState = State.RESET;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close()
+ // these tests should disable this check, by default we check the normal workflow.
+ // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this
+ assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState;
+ streamState = State.CLOSE;
+ }
+
+ @Override
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState;
+ streamState = State.SETREADER;
+ }
+
+ @Override
+ public void end() throws IOException {
+ int finalOffset = correctOffset(off);
+ offsetAtt.setOffset(finalOffset, finalOffset);
+ // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
+ // these tests should disable this check (in general you should consume the entire stream)
+ assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!";
+ streamState = State.END;
+ }
+
+ /**
+ * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+ * should leave this enabled.
+ */
+ public void setEnableChecks(boolean enableChecks) {
+ this.enableChecks = enableChecks;
+ }
+}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java Fri May 13 16:57:20 2011
@@ -25,6 +25,7 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -223,7 +224,7 @@ class DocHelper {
*/
public static SegmentInfo writeDoc(Random random, Directory dir, Document doc) throws IOException
{
- return writeDoc(random, dir, new MockAnalyzer(random, MockAnalyzer.WHITESPACE, false), null, doc);
+ return writeDoc(random, dir, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false), null, doc);
}
/**
Added: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1102812&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (added)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Fri May 13 16:57:20 2011
@@ -0,0 +1,90 @@
+package org.apache.lucene.analysis;
+
+import java.io.StringReader;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestMockAnalyzer extends BaseTokenStreamTestCase {
+
+ /** Test a configuration that behaves a lot like WhitespaceAnalyzer */
+ public void testWhitespace() throws Exception {
+ Analyzer a = new MockAnalyzer(random);
+ assertAnalyzesTo(a, "A bc defg hiJklmn opqrstuv wxy z ",
+ new String[] { "a", "bc", "defg", "hijklmn", "opqrstuv", "wxy", "z" });
+ assertAnalyzesToReuse(a, "aba cadaba shazam",
+ new String[] { "aba", "cadaba", "shazam" });
+ assertAnalyzesToReuse(a, "break on whitespace",
+ new String[] { "break", "on", "whitespace" });
+ }
+
+ /** Test a configuration that behaves a lot like SimpleAnalyzer */
+ public void testSimple() throws Exception {
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
+ assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ",
+ new String[] { "a", "bc", "defg", "hijklmn", "opqrstuv", "wxy", "z" });
+ assertAnalyzesToReuse(a, "aba4cadaba-Shazam",
+ new String[] { "aba", "cadaba", "shazam" });
+ assertAnalyzesToReuse(a, "break+on/Letters",
+ new String[] { "break", "on", "letters" });
+ }
+
+ /** Test a configuration that behaves a lot like KeywordAnalyzer */
+ public void testKeyword() throws Exception {
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
+ assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ",
+ new String[] { "a-bc123 defg+hijklmn567opqrstuv78wxy_z " });
+ assertAnalyzesToReuse(a, "aba4cadaba-Shazam",
+ new String[] { "aba4cadaba-Shazam" });
+ assertAnalyzesToReuse(a, "break+on/Nothing",
+ new String[] { "break+on/Nothing" });
+ }
+
+ /** Test a configuration that behaves a lot like StopAnalyzer */
+ public void testStop() throws Exception {
+ Analyzer a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, (CharArraySet) StopAnalyzer.ENGLISH_STOP_WORDS_SET, true);
+ assertAnalyzesTo(a, "the quick brown a fox",
+ new String[] { "quick", "brown", "fox" },
+ new int[] { 2, 1, 2 });
+
+ // disable positions
+ a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, (CharArraySet) StopAnalyzer.ENGLISH_STOP_WORDS_SET, false);
+ assertAnalyzesTo(a, "the quick brown a fox",
+ new String[] { "quick", "brown", "fox" },
+ new int[] { 1, 1, 1 });
+ }
+
+ public void testLUCENE_3042() throws Exception {
+ String testString = "t";
+
+ Analyzer analyzer = new MockAnalyzer(random);
+ TokenStream stream = analyzer.reusableTokenStream("dummy", new StringReader(testString));
+ stream.reset();
+ while (stream.incrementToken()) {
+ // consume
+ }
+ stream.end();
+ stream.close();
+
+ assertAnalyzesToReuse(analyzer, testString, new String[] { "t" });
+ }
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ checkRandomData(random, new MockAnalyzer(random), 10000*RANDOM_MULTIPLIER);
+ }
+}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1102812&r1=1102811&r2=1102812&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri May 13 16:57:20 2011
@@ -37,6 +37,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -158,6 +159,46 @@ public class TestIndexWriter extends Luc
}
}
+ private static class StringSplitAnalyzer extends Analyzer {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new StringSplitTokenizer(reader);
+ }
+ }
+
+ private static class StringSplitTokenizer extends Tokenizer {
+ private final String[] tokens;
+ private int upto = 0;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+
+ public StringSplitTokenizer(Reader r) {
+ try {
+ final StringBuilder b = new StringBuilder();
+ final char[] buffer = new char[1024];
+ int n;
+ while((n = r.read(buffer)) != -1) {
+ b.append(buffer, 0, n);
+ }
+ tokens = b.toString().split(" ");
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ clearAttributes();
+ if (upto < tokens.length) {
+ termAtt.setEmpty();
+ termAtt.append(tokens[upto]);
+ upto++;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
/**
* Make sure we skip wicked long terms.
*/
@@ -1804,7 +1845,7 @@ public class TestIndexWriter extends Luc
// LUCENE-510
public void testInvalidUTF16() throws Throwable {
Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new StringSplitAnalyzer()));
Document doc = new Document();
final int count = utf8Data.length/2;