You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/07/21 16:37:09 UTC
svn commit: r966254 - in /lucene/dev/trunk/lucene:
contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/
contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/
src/java/org/apache/lucene/queryParser/
Author: rmuir
Date: Wed Jul 21 14:37:09 2010
New Revision: 966254
URL: http://svn.apache.org/viewvc?rev=966254&view=rev
Log:
LUCENE-2514: consume tokenstreams in QP like the indexer: dont create intermediate string
Modified:
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj
lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Wed Jul 21 14:37:09 2010
@@ -24,6 +24,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.AttributeSource;
/**
@@ -307,7 +308,7 @@ public class PrecedenceQueryParser imple
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
+ TermToBytesRefAttribute termAtt = source.addAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@@ -328,7 +329,9 @@ public class PrecedenceQueryParser imple
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
- return new TermQuery(new Term(field, termAtt.toString()));
+ BytesRef term = new BytesRef();
+ termAtt.toBytesRef(term);
+ return new TermQuery(new Term(field, term));
} else {
if (severalTokensAtSamePosition || !quoted) {
if (positionCount == 1 || !quoted) {
@@ -339,9 +342,11 @@ public class PrecedenceQueryParser imple
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
+ termAtt.toBytesRef(term);
TermQuery currentQuery = new TermQuery(
- new Term(field, termAtt.toString()));
+ new Term(field, term));
q.add(currentQuery, occur);
}
return q;
@@ -351,12 +356,14 @@ public class PrecedenceQueryParser imple
MultiPhraseQuery mpq = new MultiPhraseQuery();
List<Term> multiTerms = new ArrayList<Term>();
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) {
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
- multiTerms.add(new Term(field, termAtt.toString()));
+ termAtt.toBytesRef(term);
+ multiTerms.add(new Term(field, term));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@@ -366,8 +373,10 @@ public class PrecedenceQueryParser imple
PhraseQuery q = new PhraseQuery();
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
- q.add(new Term(field, termAtt.toString()));
+ termAtt.toBytesRef(term);
+ q.add(new Term(field, term));
}
return q;
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj Wed Jul 21 14:37:09 2010
@@ -48,6 +48,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.AttributeSource;
/**
@@ -331,7 +332,7 @@ public class PrecedenceQueryParser {
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
- CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
+ TermToBytesRefAttribute termAtt = source.addAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@@ -352,7 +353,9 @@ public class PrecedenceQueryParser {
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
- return new TermQuery(new Term(field, termAtt.toString()));
+ BytesRef term = new BytesRef();
+ termAtt.toBytesRef(term);
+ return new TermQuery(new Term(field, term));
} else {
if (severalTokensAtSamePosition || !quoted) {
if (positionCount == 1 || !quoted) {
@@ -363,9 +366,11 @@ public class PrecedenceQueryParser {
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
+ termAtt.toBytesRef(term);
TermQuery currentQuery = new TermQuery(
- new Term(field, termAtt.toString()));
+ new Term(field, term));
q.add(currentQuery, occur);
}
return q;
@@ -375,12 +380,14 @@ public class PrecedenceQueryParser {
MultiPhraseQuery mpq = new MultiPhraseQuery();
List<Term> multiTerms = new ArrayList<Term>();
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
if (posincrAtt.getPositionIncrement() == 1 && multiTerms.size() > 0) {
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
- multiTerms.add(new Term(field, termAtt.toString()));
+ termAtt.toBytesRef(term);
+ multiTerms.add(new Term(field, term));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@@ -390,8 +397,10 @@ public class PrecedenceQueryParser {
PhraseQuery q = new PhraseQuery();
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
+ BytesRef term = new BytesRef();
source.restoreState(list.get(i));
- q.add(new Term(field, termAtt.toString()));
+ termAtt.toBytesRef(term);
+ q.add(new Term(field, term));
}
return q;
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParserTokenManager.java Wed Jul 21 14:37:09 2010
@@ -22,6 +22,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.AttributeSource;
/** Token Manager. */
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java Wed Jul 21 14:37:09 2010
@@ -6,11 +6,12 @@ import java.util.ArrayList;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spans.SpanOrQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.xmlparser.DOMUtils;
import org.apache.lucene.xmlparser.ParserException;
import org.w3c.dom.Element;
@@ -56,10 +57,12 @@ public class SpanOrTermsBuilder extends
{
ArrayList<SpanQuery> clausesList=new ArrayList<SpanQuery>();
TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value));
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
while (ts.incrementToken()) {
- SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, termAtt.toString()));
+ BytesRef term = new BytesRef();
+ termAtt.toBytesRef(term);
+ SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, term));
clausesList.add(stq);
}
SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()]));
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java Wed Jul 21 14:37:09 2010
@@ -5,10 +5,11 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.TermsFilter;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.xmlparser.DOMUtils;
import org.apache.lucene.xmlparser.FilterBuilder;
import org.apache.lucene.xmlparser.ParserException;
@@ -57,19 +58,21 @@ public class TermsFilterBuilder implemen
String text = DOMUtils.getNonBlankTextOrFail(e);
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
try
{
Term term = null;
while (ts.incrementToken()) {
+ BytesRef bytes = new BytesRef();
+ termAtt.toBytesRef(bytes);
if (term == null)
{
- term = new Term(fieldName, termAtt.toString());
+ term = new Term(fieldName, bytes);
} else
{
// create from previous to save fieldName.intern overhead
- term = term.createTerm(termAtt.toString());
+ term = term.createTerm(bytes);
}
tf.addTerm(term);
}
Modified: lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java Wed Jul 21 14:37:09 2010
@@ -5,12 +5,13 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.xmlparser.DOMUtils;
import org.apache.lucene.xmlparser.ParserException;
import org.apache.lucene.xmlparser.QueryBuilder;
@@ -57,16 +58,18 @@ public class TermsQueryBuilder implement
TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
try
{
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
while (ts.incrementToken()) {
+ BytesRef bytes = new BytesRef();
+ termAtt.toBytesRef(bytes);
if (term == null)
{
- term = new Term(fieldName, termAtt.toString());
+ term = new Term(fieldName, bytes);
} else
{
// create from previous to save fieldName.intern overhead
- term = term.createTerm(termAtt.toString());
+ term = term.createTerm(bytes);
}
bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD));
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.java Wed Jul 21 14:37:09 2010
@@ -17,7 +17,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.index.Term;
@@ -33,6 +33,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.VirtualMethod;
@@ -574,7 +575,7 @@ public class QueryParser implements Quer
source = analyzer.tokenStream(field, new StringReader(queryText));
}
CachingTokenFilter buffer = new CachingTokenFilter(source);
- CharTermAttribute termAtt = null;
+ TermToBytesRefAttribute termAtt = null;
PositionIncrementAttribute posIncrAtt = null;
int numTokens = 0;
@@ -586,8 +587,8 @@ public class QueryParser implements Quer
// success==false if we hit an exception
}
if (success) {
- if (buffer.hasAttribute(CharTermAttribute.class)) {
- termAtt = buffer.getAttribute(CharTermAttribute.class);
+ if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
+ termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
}
if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
@@ -629,11 +630,11 @@ public class QueryParser implements Quer
if (numTokens == 0)
return null;
else if (numTokens == 1) {
- String term = null;
+ BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
@@ -648,11 +649,11 @@ public class QueryParser implements Quer
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
@@ -670,12 +671,12 @@ public class QueryParser implements Quer
List<Term> multiTerms = new ArrayList<Term>();
int position = -1;
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -709,13 +710,13 @@ public class QueryParser implements Quer
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -1568,12 +1569,6 @@ public class QueryParser implements Quer
finally { jj_save(0, xla); }
}
- private boolean jj_3R_2() {
- if (jj_scan_token(TERM)) return true;
- if (jj_scan_token(COLON)) return true;
- return false;
- }
-
private boolean jj_3_1() {
Token xsp;
xsp = jj_scanpos;
@@ -1590,6 +1585,12 @@ public class QueryParser implements Quer
return false;
}
+ private boolean jj_3R_2() {
+ if (jj_scan_token(TERM)) return true;
+ if (jj_scan_token(COLON)) return true;
+ return false;
+ }
+
/** Generated Token Manager. */
public QueryParserTokenManager token_source;
/** Current token. */
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParser.jj Wed Jul 21 14:37:09 2010
@@ -41,7 +41,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.index.Term;
@@ -57,6 +57,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.VirtualMethod;
@@ -598,7 +599,7 @@ public class QueryParser {
source = analyzer.tokenStream(field, new StringReader(queryText));
}
CachingTokenFilter buffer = new CachingTokenFilter(source);
- CharTermAttribute termAtt = null;
+ TermToBytesRefAttribute termAtt = null;
PositionIncrementAttribute posIncrAtt = null;
int numTokens = 0;
@@ -610,8 +611,8 @@ public class QueryParser {
// success==false if we hit an exception
}
if (success) {
- if (buffer.hasAttribute(CharTermAttribute.class)) {
- termAtt = buffer.getAttribute(CharTermAttribute.class);
+ if (buffer.hasAttribute(TermToBytesRefAttribute.class)) {
+ termAtt = buffer.getAttribute(TermToBytesRefAttribute.class);
}
if (buffer.hasAttribute(PositionIncrementAttribute.class)) {
posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class);
@@ -653,11 +654,11 @@ public class QueryParser {
if (numTokens == 0)
return null;
else if (numTokens == 1) {
- String term = null;
+ BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
@@ -672,11 +673,11 @@ public class QueryParser {
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
@@ -694,12 +695,12 @@ public class QueryParser {
List<Term> multiTerms = new ArrayList<Term>();
int position = -1;
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -733,13 +734,13 @@ public class QueryParser {
for (int i = 0; i < numTokens; i++) {
- String term = null;
+ BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.toString();
+ termAtt.toBytesRef(term);
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java?rev=966254&r1=966253&r2=966254&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java Wed Jul 21 14:37:09 2010
@@ -15,7 +15,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.index.Term;
@@ -31,6 +31,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.util.VirtualMethod;