You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/12 15:32:33 UTC
svn commit: r1102290 - in /lucene/dev/trunk/lucene:
contrib/highlighter/src/java/org/apache/lucene/search/highlight/
contrib/highlighter/src/test/org/apache/lucene/search/highlight/
contrib/queries/src/java/org/apache/lucene/search/ contrib/queries/src...
Author: rmuir
Date: Thu May 12 13:32:32 2011
New Revision: 1102290
URL: http://svn.apache.org/viewvc?rev=1102290&view=rev
Log:
LUCENE-3064: add checks to MockTokenizer to enforce proper consumption
Modified:
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Thu May 12 13:32:32 2011
@@ -355,6 +355,7 @@ public class Highlighter
{
try
{
+ tokenStream.end();
tokenStream.close();
}
catch (Exception e)
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Thu May 12 13:32:32 2011
@@ -1093,6 +1093,10 @@ public class HighlighterTest extends Bas
}
public void testMaxSizeHighlight() throws Exception {
+ final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
+ // we disable MockTokenizer checks because we will forcefully limit the
+ // tokenstream and call end() before incrementToken() returns false.
+ analyzer.setEnableChecks(false);
TestHighlightRunner helper = new TestHighlightRunner() {
@Override
@@ -1122,7 +1126,10 @@ public class HighlighterTest extends Bas
public void run() throws Exception {
String goodWord = "goodtoken";
CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("stoppedtoken"));
-
+ // we disable MockTokenizer checks because we will forcefully limit the
+ // tokenstream and call end() before incrementToken() returns false.
+ final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true);
+ analyzer.setEnableChecks(false);
TermQuery query = new TermQuery(new Term("data", goodWord));
String match;
@@ -1134,13 +1141,13 @@ public class HighlighterTest extends Bas
sb.append("stoppedtoken");
}
SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
- Highlighter hg = getHighlighter(query, "data", new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true).tokenStream(
+ Highlighter hg = getHighlighter(query, "data", analyzer.tokenStream(
"data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
// new
// QueryTermScorer(query));
hg.setTextFragmenter(new NullFragmenter());
hg.setMaxDocCharsToAnalyze(100);
- match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString());
+ match = hg.getBestFragment(analyzer, "data", sb.toString());
assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
.getMaxDocCharsToAnalyze());
@@ -1151,7 +1158,7 @@ public class HighlighterTest extends Bas
// + whitespace)
sb.append(" ");
sb.append(goodWord);
- match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString());
+ match = hg.getBestFragment(analyzer, "data", sb.toString());
assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
.getMaxDocCharsToAnalyze());
}
@@ -1726,6 +1733,11 @@ final class SynonymAnalyzer extends Anal
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
+ try {
+ stream.reset();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
return new SynonymTokenizer(stream, synonyms);
}
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java Thu May 12 13:32:32 2011
@@ -28,32 +28,38 @@ import org.apache.lucene.analysis.TokenS
public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
+ // we disable MockTokenizer checks because we will forcefully limit the
+ // tokenstream and call end() before incrementToken() returns false.
+ MockTokenizer stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 12);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
+ stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 30);
assertTokenStreamContents(filter, new String[] {"short", "toolong",
"evenmuchlongertext"});
-
+ // TODO: This is not actually testing reuse! (reusableTokenStream is not implemented)
checkOneTermReuse(new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new OffsetLimitTokenFilter(new MockTokenizer(reader,
- MockTokenizer.WHITESPACE, false), 10);
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false);
+ return new OffsetLimitTokenFilter(tokenizer, 10);
}
}, "llenges", "llenges");
}
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Thu May 12 13:32:32 2011
@@ -192,6 +192,7 @@ public class FuzzyLikeThisQuery extends
int corpusNumDocs=reader.numDocs();
Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet<String> processedTerms=new HashSet<String>();
+ ts.reset();
while (ts.incrementToken())
{
String term = termAtt.toString();
@@ -244,7 +245,9 @@ public class FuzzyLikeThisQuery extends
}
}
}
- }
+ }
+ ts.end();
+ ts.close();
}
@Override
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Thu May 12 13:32:32 2011
@@ -885,7 +885,7 @@ public final class MoreLikeThis {
int tokenCount=0;
// for every token
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
+ ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
tokenCount++;
@@ -906,6 +906,8 @@ public final class MoreLikeThis {
cnt.x++;
}
}
+ ts.end();
+ ts.close();
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Thu May 12 13:32:32 2011
@@ -110,6 +110,11 @@ public class AnalyzingQueryParser extend
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
int countTokens = 0;
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
while (true) {
try {
if (!source.incrementToken()) break;
@@ -126,6 +131,7 @@ public class AnalyzingQueryParser extend
}
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -191,7 +197,11 @@ public class AnalyzingQueryParser extend
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
List<String> tlist = new ArrayList<String>();
CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
-
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
while (true) {
try {
if (!source.incrementToken()) break;
@@ -202,6 +212,7 @@ public class AnalyzingQueryParser extend
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -242,6 +253,7 @@ public class AnalyzingQueryParser extend
boolean multipleTokens = false;
try {
+ source.reset();
if (source.incrementToken()) {
nextToken = termAtt.toString();
}
@@ -251,6 +263,7 @@ public class AnalyzingQueryParser extend
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -281,6 +294,7 @@ public class AnalyzingQueryParser extend
try {
source = getAnalyzer().tokenStream(field, new StringReader(part1));
termAtt = source.addAttribute(CharTermAttribute.class);
+ source.reset();
multipleTokens = false;
@@ -292,6 +306,7 @@ public class AnalyzingQueryParser extend
// ignore
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
@@ -308,6 +323,7 @@ public class AnalyzingQueryParser extend
termAtt = source.addAttribute(CharTermAttribute.class);
try {
+ source.reset();
if (source.incrementToken()) {
part2 = termAtt.toString();
}
@@ -316,6 +332,7 @@ public class AnalyzingQueryParser extend
// ignore
}
try {
+ source.end();
source.close();
} catch (IOException e) {
// ignore
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Thu May 12 13:32:32 2011
@@ -123,6 +123,11 @@ public class AnalyzerQueryNodeProcessor
TokenStream source = this.analyzer.tokenStream(field, new StringReader(
text));
+ try {
+ source.reset();
+ } catch (IOException e1) {
+ throw new RuntimeException(e1);
+ }
CachingTokenFilter buffer = new CachingTokenFilter(source);
PositionIncrementAttribute posIncrAtt = null;
Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Thu May 12 13:32:32 2011
@@ -118,12 +118,14 @@ public final class SynExpand {
// [1] Parse query into separate words so that when we expand we can avoid dups
TokenStream ts = a.tokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
-
+ ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
if ( already.add( word))
top.add( word);
}
+ ts.end();
+ ts.close();
final BooleanQuery tmp = new BooleanQuery();
// [2] form query
Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java Thu May 12 13:32:32 2011
@@ -111,7 +111,6 @@ public class TestSynonymTokenFilter exte
setPreviousTokenStream(streams);
} else {
streams.source.reset(reader);
- streams.result.reset(); // reset the SynonymTokenFilter
}
return streams.result;
}
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Thu May 12 13:32:32 2011
@@ -80,9 +80,12 @@ public class LikeThisQueryBuilder implem
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
+ ts.reset();
while(ts.incrementToken()) {
stopWordsSet.add(termAtt.toString());
}
+ ts.end();
+ ts.close();
}
catch(IOException ioe)
{
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Thu May 12 13:32:32 2011
@@ -59,11 +59,14 @@ public class SpanOrTermsBuilder extends
TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes)));
clausesList.add(stq);
}
+ ts.end();
+ ts.close();
SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f));
return soq;
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Thu May 12 13:32:32 2011
@@ -64,6 +64,7 @@ public class TermsFilterBuilder implemen
{
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
if (term == null)
@@ -76,6 +77,8 @@ public class TermsFilterBuilder implemen
}
tf.addTerm(term);
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Thu May 12 13:32:32 2011
@@ -61,6 +61,7 @@ public class TermsQueryBuilder implement
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
+ ts.reset();
while (ts.incrementToken()) {
termAtt.fillBytesRef();
if (term == null)
@@ -73,6 +74,8 @@ public class TermsQueryBuilder implement
}
bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
}
+ ts.end();
+ ts.close();
}
catch (IOException ioe)
{
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java Thu May 12 13:32:32 2011
@@ -806,6 +806,7 @@ public abstract class QueryParserBase {
}
try {
+ source.end();
source.close();
} catch (IOException ignored) {}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Thu May 12 13:32:32 2011
@@ -262,6 +262,7 @@ public abstract class BaseTokenStreamTes
tokens.add(termAtt.toString());
// TODO: we could collect offsets etc here for better checking that reset() really works.
}
+ ts.end();
ts.close();
// verify reusing is "reproducable" and also get the normal tokenstream sanity checks
if (!tokens.isEmpty())
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java Thu May 12 13:32:32 2011
@@ -36,6 +36,7 @@ public final class MockAnalyzer extends
private int positionIncrementGap;
private final Random random;
private Map<String,Integer> previousMappings = new HashMap<String,Integer>();
+ private boolean enableChecks = true;
/**
* Creates a new MockAnalyzer.
@@ -75,6 +76,7 @@ public final class MockAnalyzer extends
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ tokenizer.setEnableChecks(enableChecks);
TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
filt = maybePayload(filt, fieldName);
return filt;
@@ -98,13 +100,13 @@ public final class MockAnalyzer extends
if (saved == null) {
saved = new SavedStreams();
saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
+ saved.tokenizer.setEnableChecks(enableChecks);
saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
saved.filter = maybePayload(saved.filter, fieldName);
map.put(fieldName, saved);
return saved.filter;
} else {
saved.tokenizer.reset(reader);
- saved.filter.reset();
return saved.filter;
}
}
@@ -139,4 +141,12 @@ public final class MockAnalyzer extends
public int getPositionIncrementGap(String fieldName){
return positionIncrementGap;
}
+
+ /**
+ * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+ * should leave this enabled.
+ */
+ public void setEnableChecks(boolean enableChecks) {
+ this.enableChecks = enableChecks;
+ }
}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java Thu May 12 13:32:32 2011
@@ -86,6 +86,7 @@ final class MockPayloadFilter extends To
@Override
public void reset() throws IOException {
+ super.reset();
i = 0;
pos = 0;
}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java Thu May 12 13:32:32 2011
@@ -50,11 +50,27 @@ public class MockTokenizer extends Token
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
int off = 0;
+ // TODO: "register" with LuceneTestCase to ensure all streams are closed() ?
+ // currently, we can only check that the lifecycle is correct if someone is reusing,
+ // but not for "one-offs".
+ private static enum State {
+ SETREADER, // consumer set a reader input either via ctor or via reset(Reader)
+ RESET, // consumer has called reset()
+ INCREMENT, // consumer is consuming, has called incrementToken() == true
+ INCREMENT_FALSE, // consumer has called incrementToken() which returned false
+ END, // consumer has called end() to perform end of stream operations
+ CLOSE // consumer has called close() to release any resources
+ };
+
+ private State streamState = State.CLOSE;
+ private boolean enableChecks = true;
+
public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
super(factory, input);
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
this.state = runAutomaton.getInitialState();
+ this.streamState = State.SETREADER;
}
public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
@@ -62,10 +78,13 @@ public class MockTokenizer extends Token
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
this.state = runAutomaton.getInitialState();
+ this.streamState = State.SETREADER;
}
@Override
public final boolean incrementToken() throws IOException {
+ assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)
+ : "incrementToken() called while in wrong state: " + streamState;
clearAttributes();
for (;;) {
int startOffset = off;
@@ -82,9 +101,11 @@ public class MockTokenizer extends Token
cp = readCodePoint();
} while (cp >= 0 && isTokenChar(cp));
offsetAtt.setOffset(startOffset, endOffset);
+ streamState = State.INCREMENT;
return true;
}
}
+ streamState = State.INCREMENT_FALSE;
return false;
}
@@ -126,11 +147,42 @@ public class MockTokenizer extends Token
super.reset();
state = runAutomaton.getInitialState();
off = 0;
+ assert !enableChecks || streamState != State.RESET : "double reset()";
+ streamState = State.RESET;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close()
+ // these tests should disable this check, by default we check the normal workflow.
+ // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this
+ assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState;
+ streamState = State.CLOSE;
+ }
+
+ @Override
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState;
+ streamState = State.SETREADER;
}
@Override
public void end() throws IOException {
int finalOffset = correctOffset(off);
offsetAtt.setOffset(finalOffset, finalOffset);
+ // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
+ // these tests should disable this check (in general you should consume the entire stream)
+ assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!";
+ streamState = State.END;
+ }
+
+ /**
+ * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+ * should leave this enabled.
+ */
+ public void setEnableChecks(boolean enableChecks) {
+ this.enableChecks = enableChecks;
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Thu May 12 13:32:32 2011
@@ -107,6 +107,7 @@ public class TestMockAnalyzer extends Ba
// consume
}
stream.end();
+ stream.close();
assertAnalyzesToReuse(analyzer, testString, new String[] { "t" });
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu May 12 13:32:32 2011
@@ -155,7 +155,9 @@ public class TestIndexWriterExceptions e
}
MockDirectoryWrapper dir = newDirectory();
- MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ MockAnalyzer analyzer = new MockAnalyzer(random);
+ analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
.setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler()));
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
//writer.setMaxBufferedDocs(10);
@@ -201,7 +203,9 @@ public class TestIndexWriterExceptions e
public void testRandomExceptionsThreads() throws Throwable {
MockDirectoryWrapper dir = newDirectory();
- MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ MockAnalyzer analyzer = new MockAnalyzer(random);
+ analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
.setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler()));
((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions();
//writer.setMaxBufferedDocs(10);
@@ -321,7 +325,9 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
}
};
@@ -390,7 +396,9 @@ public class TestIndexWriterExceptions e
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new TokenFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)) {
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new TokenFilter(tokenizer) {
private int count = 0;
@Override
@@ -522,7 +530,9 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
}
};
@@ -621,7 +631,9 @@ public class TestIndexWriterExceptions e
Analyzer analyzer = new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
+ return new CrashingFilter(fieldName, tokenizer);
}
};
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Thu May 12 13:32:32 2011
@@ -49,6 +49,7 @@ public class TestLongPostings extends Lu
final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class);
final BytesRef termBytes = termAtt.getBytesRef();
int count = 0;
+ ts.reset();
while(ts.incrementToken()) {
termAtt.fillBytesRef();
if (count == 0 && !termBytes.utf8ToString().equals(s)) {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java Thu May 12 13:32:32 2011
@@ -121,7 +121,9 @@ public class TestTermVectorsWriter exten
Analyzer analyzer = new MockAnalyzer(random);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd ")));
+ TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd "));
+ stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct?
+ stream = new CachingTokenFilter(stream);
Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.add(f);
doc.add(f);
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1102290&r1=1102289&r2=1102290&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java Thu May 12 13:32:32 2011
@@ -626,11 +626,14 @@ public class TestPhraseQuery extends Luc
}
TokenStream ts = analyzer.reusableTokenStream("ignore", new StringReader(term));
CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class);
+ ts.reset();
while(ts.incrementToken()) {
String text = termAttr.toString();
doc.add(text);
sb.append(text).append(' ');
}
+ ts.end();
+ ts.close();
} else {
// pick existing sub-phrase
List<String> lastDoc = docs.get(r.nextInt(docs.size()));