You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/13 19:13:19 UTC
svn commit: r1102817 - in /lucene/dev/trunk/lucene: contrib/
contrib/highlighter/src/java/org/apache/lucene/search/highlight/
contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
contrib/memory/src/java/org/apache/lucene/index/memory/ co...
Author: rmuir
Date: Fri May 13 17:13:19 2011
New Revision: 1102817
URL: http://svn.apache.org/viewvc?rev=1102817&view=rev
Log:
LUCENE-3040: analysis consumers should use reusable tokenstreams
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Fri May 13 17:13:19 2011
@@ -79,6 +79,11 @@ New Features
* LUCENE-3016: Add analyzer for Latvian. (Robert Muir)
+Optimizations
+
+ * LUCENE-3040: Switch all analysis consumers (highlighter, morelikethis, memory, ...)
+ over to reusableTokenStream(). (Robert Muir)
+
======================= Lucene 3.1.0 =======================
Changes in backwards compatibility policy
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Fri May 13 17:13:19 2011
@@ -286,7 +286,11 @@ public class TokenSources {
// convenience method
public static TokenStream getTokenStream(String field, String contents,
Analyzer analyzer) {
- return analyzer.tokenStream(field, new StringReader(contents));
+ try {
+ return analyzer.reusableTokenStream(field, new StringReader(contents));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
}
}
Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Fri May 13 17:13:19 2011
@@ -532,7 +532,7 @@ public class InstantiatedIndexWriter imp
if (field.tokenStreamValue() != null) {
tokenStream = field.tokenStreamValue();
} else {
- tokenStream = analyzer.tokenStream(field.name(), new StringReader(field.stringValue()));
+ tokenStream = analyzer.reusableTokenStream(field.name(), new StringReader(field.stringValue()));
}
// reset the TokenStream to the first token
Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri May 13 17:13:19 2011
@@ -261,8 +261,12 @@ public class MemoryIndex {
if (analyzer == null)
throw new IllegalArgumentException("analyzer must not be null");
- TokenStream stream = analyzer.tokenStream(fieldName,
- new StringReader(text));
+ TokenStream stream;
+ try {
+ stream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
addField(fieldName, stream);
}
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri May 13 17:13:19 2011
@@ -186,7 +186,7 @@ public class FuzzyLikeThisQuery extends
private void addTerms(IndexReader reader,FieldVals f) throws IOException
{
if(f.queryString==null) return;
- TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString));
+ TokenStream ts=analyzer.reusableTokenStream(f.fieldName,new StringReader(f.queryString));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
int corpusNumDocs=reader.numDocs();
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri May 13 17:13:19 2011
@@ -881,7 +881,7 @@ public final class MoreLikeThis {
throw new UnsupportedOperationException("To use MoreLikeThis without " +
"term vectors, you must provide an Analyzer");
}
- TokenStream ts = analyzer.tokenStream(fieldName, r);
+ TokenStream ts = analyzer.reusableTokenStream(fieldName, r);
int tokenCount=0;
// for every token
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java Fri May 13 17:13:19 2011
@@ -85,7 +85,7 @@ public final class SimilarityQueries
Set<?> stop)
throws IOException
{
- TokenStream ts = a.tokenStream( field, new StringReader( body));
+ TokenStream ts = a.reusableTokenStream( field, new StringReader( body));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
BooleanQuery tmp = new BooleanQuery();
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Fri May 13 17:13:19 2011
@@ -106,15 +106,16 @@ public class AnalyzingQueryParser extend
}
// get Analyzer from superclass and tokenize the term
- TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
+ TokenStream source;
int countTokens = 0;
try {
+ source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr));
source.reset();
} catch (IOException e1) {
throw new RuntimeException(e1);
}
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
while (true) {
try {
if (!source.incrementToken()) break;
@@ -194,14 +195,15 @@ public class AnalyzingQueryParser extend
@Override
protected Query getPrefixQuery(String field, String termStr) throws ParseException {
// get Analyzer from superclass and tokenize the term
- TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
+ TokenStream source;
List<String> tlist = new ArrayList<String>();
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
try {
+ source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr));
source.reset();
} catch (IOException e1) {
throw new RuntimeException(e1);
}
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
while (true) {
try {
if (!source.incrementToken()) break;
@@ -247,12 +249,13 @@ public class AnalyzingQueryParser extend
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity)
throws ParseException {
// get Analyzer from superclass and tokenize the term
- TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
+ TokenStream source = null;
String nextToken = null;
boolean multipleTokens = false;
try {
+ source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr));
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
source.reset();
if (source.incrementToken()) {
nextToken = termAtt.toString();
@@ -292,7 +295,7 @@ public class AnalyzingQueryParser extend
if (part1 != null) {
// part1
try {
- source = getAnalyzer().tokenStream(field, new StringReader(part1));
+ source = getAnalyzer().reusableTokenStream(field, new StringReader(part1));
termAtt = source.addAttribute(CharTermAttribute.class);
source.reset();
multipleTokens = false;
@@ -318,11 +321,10 @@ public class AnalyzingQueryParser extend
}
if (part2 != null) {
- // part2
- source = getAnalyzer().tokenStream(field, new StringReader(part2));
- termAtt = source.addAttribute(CharTermAttribute.class);
-
try {
+ // part2
+ source = getAnalyzer().reusableTokenStream(field, new StringReader(part2));
+ termAtt = source.addAttribute(CharTermAttribute.class);
source.reset();
if (source.incrementToken()) {
part2 = termAtt.toString();
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Fri May 13 17:13:19 2011
@@ -121,9 +121,9 @@ public class AnalyzerQueryNodeProcessor
String text = fieldNode.getTextAsString();
String field = fieldNode.getFieldAsString();
- TokenStream source = this.analyzer.tokenStream(field, new StringReader(
- text));
+ TokenStream source;
try {
+ source = this.analyzer.reusableTokenStream(field, new StringReader(text));
source.reset();
} catch (IOException e1) {
throw new RuntimeException(e1);
Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java Fri May 13 17:13:19 2011
@@ -116,7 +116,7 @@ public final class SynExpand {
if ( a == null) a = new StandardAnalyzer(Version.LUCENE_CURRENT);
// [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.tokenStream( field, new StringReader( query));
+ TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (original)
+++ lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java Fri May 13 17:13:19 2011
@@ -124,7 +124,7 @@ public class SynLookup {
List<String> top = new LinkedList<String>(); // needs to be separately listed..
// [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.tokenStream( field, new StringReader( query));
+ TokenStream ts = a.reusableTokenStream( field, new StringReader( query));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java Fri May 13 17:13:19 2011
@@ -76,10 +76,10 @@ public class LikeThisQueryBuilder implem
stopWordsSet=new HashSet<String>();
for (int i = 0; i < fields.length; i++)
{
- TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords));
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fields[i],new StringReader(stopWords));
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while(ts.incrementToken()) {
stopWordsSet.add(termAtt.toString());
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Fri May 13 17:13:19 2011
@@ -56,7 +56,7 @@ public class SpanOrTermsBuilder extends
try
{
ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
- TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
+ TokenStream ts=analyzer.reusableTokenStream(fieldName,new StringReader(value));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Fri May 13 17:13:19 2011
@@ -57,11 +57,11 @@ public class TermsFilterBuilder implemen
TermsFilter tf = new TermsFilter();
String text = DOMUtils.getNonBlankTextOrFail(e);
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
- TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Fri May 13 17:13:19 2011
@@ -55,9 +55,9 @@ public class TermsQueryBuilder implement
BooleanQuery bq=new BooleanQuery(DOMUtils.getAttribute(e,"disableCoord",false));
bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e,"minimumNumberShouldMatch",0));
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
try
{
+ TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text));
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java?rev=1102817&r1=1102816&r2=1102817&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/QueryTermVector.java Fri May 13 17:13:19 2011
@@ -55,7 +55,12 @@ public class QueryTermVector implements
public QueryTermVector(String queryString, Analyzer analyzer) {
if (analyzer != null)
{
- TokenStream stream = analyzer.tokenStream("", new StringReader(queryString));
+ TokenStream stream;
+ try {
+ stream = analyzer.reusableTokenStream("", new StringReader(queryString));
+ } catch (IOException e1) {
+ stream = null;
+ }
if (stream != null)
{
List<BytesRef> terms = new ArrayList<BytesRef>();