You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/06/01 12:35:16 UTC
svn commit: r950008 [1/4] - in /lucene/dev/trunk:
lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/src/test/org/apache/lucene/search...
Author: rmuir
Date: Tue Jun 1 10:35:13 2010
New Revision: 950008
URL: http://svn.apache.org/viewvc?rev=950008&view=rev
Log:
LUCENE-2372: switch over remaining uses of TermAttribute
Modified:
lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java
lucene/dev/trunk/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynonymTokenFilter.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java
lucene/dev/trunk/lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilter.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/fr/TestElision.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestSingleTokenTokenFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
lucene/dev/trunk/modules/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java
lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/BufferedTokenStream.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java
lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java
lucene/dev/trunk/solr/src/java/org/apache/solr/spelling/SpellingQueryConverter.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestBufferedTokenStream.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestRemoveDuplicatesTokenFilterFactory.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestSynonymMap.java
lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SimpleQueryConverter.java
lucene/dev/trunk/solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java
lucene/dev/trunk/solr/src/webapp/web/admin/analysis.jsp
Modified: lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/trunk/lucene/contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Jun 1 10:35:13 2010
@@ -26,9 +26,10 @@ import java.util.List;
import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.benchmark.BenchmarkTestCase;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
import org.apache.lucene.benchmark.byTask.feeds.ReutersQueryMaker;
@@ -918,11 +919,11 @@ public class TestPerfTasksLogic extends
TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text));
ts1.reset();
ts2.reset();
- TermAttribute termAtt1 = ts1.addAttribute(TermAttribute.class);
- TermAttribute termAtt2 = ts2.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt1 = ts1.addAttribute(CharTermAttribute.class);
+ CharTermAttribute termAtt2 = ts2.addAttribute(CharTermAttribute.class);
assertTrue(ts1.incrementToken());
assertTrue(ts2.incrementToken());
- assertEquals(termAtt1.term(), termAtt2.term());
+ assertEquals(termAtt1.toString(), termAtt2.toString());
assertFalse(ts1.incrementToken());
assertFalse(ts2.incrementToken());
ts1.close();
@@ -994,21 +995,7 @@ public class TestPerfTasksLogic extends
private void assertEqualShingle
(Analyzer analyzer, String text, String[] expected) throws Exception {
- TokenStream stream = analyzer.tokenStream("bogus", new StringReader(text));
- stream.reset();
- TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
- int termNum = 0;
- while (stream.incrementToken()) {
- assertTrue("Extra output term(s), starting with '"
- + new String(termAtt.termBuffer(), 0, termAtt.termLength()) + "'",
- termNum < expected.length);
- assertEquals("Mismatch in output term # " + termNum + " - ",
- expected[termNum],
- new String(termAtt.termBuffer(), 0, termAtt.termLength()));
- ++termNum;
- }
- assertEquals("Too few output terms", expected.length, termNum);
- stream.close();
+ BaseTokenStreamTestCase.assertAnalyzesTo(analyzer, text, expected);
}
private String[] getShingleConfig(String params) {
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java Tue Jun 1 10:35:13 2010
@@ -23,9 +23,9 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.PriorityQueue;
/**
@@ -191,7 +191,7 @@ public class Highlighter
ArrayList<TextFragment> docFrags = new ArrayList<TextFragment>();
StringBuilder newText=new StringBuilder();
- TermAttribute termAtt = tokenStream.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
tokenStream.addAttribute(PositionIncrementAttribute.class);
tokenStream.reset();
@@ -225,7 +225,7 @@ public class Highlighter
(offsetAtt.startOffset()>text.length())
)
{
- throw new InvalidTokenOffsetsException("Token "+ termAtt.term()
+ throw new InvalidTokenOffsetsException("Token "+ termAtt.toString()
+" exceeds length of provided text sized "+text.length());
}
if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct()))
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java Tue Jun 1 10:35:13 2010
@@ -25,8 +25,8 @@ import java.util.Set;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.lucene.search.Query;
@@ -46,7 +46,7 @@ public class QueryScorer implements Scor
private float maxTermWeight;
private int position = -1;
private String defaultField;
- private TermAttribute termAtt;
+ private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncAtt;
private boolean expandMultiTermQuery = true;
private Query query;
@@ -145,7 +145,7 @@ public class QueryScorer implements Scor
*/
public float getTokenScore() {
position += posIncAtt.getPositionIncrement();
- String termText = termAtt.term();
+ String termText = termAtt.toString();
WeightedSpanTerm weightedSpanTerm;
@@ -175,7 +175,7 @@ public class QueryScorer implements Scor
*/
public TokenStream init(TokenStream tokenStream) throws IOException {
position = -1;
- termAtt = tokenStream.addAttribute(TermAttribute.class);
+ termAtt = tokenStream.addAttribute(CharTermAttribute.class);
posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
if(!skipInitExtractor) {
if(fieldWeightedSpanTerms != null) {
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java Tue Jun 1 10:35:13 2010
@@ -21,7 +21,7 @@ import java.util.HashMap;
import java.util.HashSet;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
@@ -41,7 +41,7 @@ public class QueryTermScorer implements
float maxTermWeight = 0;
private HashMap<String,WeightedTerm> termsToFind;
- private TermAttribute termAtt;
+ private CharTermAttribute termAtt;
/**
*
@@ -95,7 +95,7 @@ public class QueryTermScorer implements
* @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
*/
public TokenStream init(TokenStream tokenStream) {
- termAtt = tokenStream.addAttribute(TermAttribute.class);
+ termAtt = tokenStream.addAttribute(CharTermAttribute.class);
return null;
}
@@ -118,7 +118,7 @@ public class QueryTermScorer implements
* @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
*/
public float getTokenScore() {
- String termText = termAtt.term();
+ String termText = termAtt.toString();
WeightedTerm queryTerm = termsToFind.get(termText);
if (queryTerm == null) {
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java Tue Jun 1 10:35:13 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.search.highlig
import java.util.List;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.search.spans.Spans;
@@ -38,7 +38,7 @@ public class SimpleSpanFragmenter implem
private QueryScorer queryScorer;
private int waitForPos = -1;
private int textSize;
- private TermAttribute termAtt;
+ private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncAtt;
private OffsetAttribute offsetAtt;
@@ -70,7 +70,7 @@ public class SimpleSpanFragmenter implem
return false;
}
- WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.term());
+ WeightedSpanTerm wSpanTerm = queryScorer.getWeightedSpanTerm(termAtt.toString());
if (wSpanTerm != null) {
List<PositionSpan> positionSpans = wSpanTerm.getPositionSpans();
@@ -101,7 +101,7 @@ public class SimpleSpanFragmenter implem
position = -1;
currentNumFrags = 1;
textSize = originalText.length();
- termAtt = tokenStream.addAttribute(TermAttribute.class);
+ termAtt = tokenStream.addAttribute(CharTermAttribute.class);
posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java Tue Jun 1 10:35:13 2010
@@ -19,8 +19,8 @@ package org.apache.lucene.search.highlig
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
/**
* One, or several overlapping tokens, along with the score(s) and the scope of
@@ -38,11 +38,11 @@ public class TokenGroup {
int matchStartOffset, matchEndOffset;
private OffsetAttribute offsetAtt;
- private TermAttribute termAtt;
+ private CharTermAttribute termAtt;
public TokenGroup(TokenStream tokenStream) {
offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
- termAtt = tokenStream.addAttribute(TermAttribute.class);
+ termAtt = tokenStream.addAttribute(CharTermAttribute.class);
}
void addToken(float score) {
@@ -68,7 +68,7 @@ public class TokenGroup {
}
}
Token token = new Token(termStartOffset, termEndOffset);
- token.setTermBuffer(termAtt.term());
+ token.setEmpty().append(termAtt);
tokens[numTokens] = token;
scores[numTokens] = score;
numTokens++;
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java Tue Jun 1 10:35:13 2010
@@ -29,8 +29,8 @@ import java.util.Comparator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermFreqVector;
@@ -153,13 +153,13 @@ public class TokenSources {
int currentToken = 0;
- TermAttribute termAtt;
+ CharTermAttribute termAtt;
OffsetAttribute offsetAtt;
StoredTokenStream(Token tokens[]) {
this.tokens = tokens;
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@@ -170,7 +170,7 @@ public class TokenSources {
}
Token token = tokens[currentToken++];
clearAttributes();
- termAtt.setTermBuffer(token.term());
+ termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
}
@@ -204,9 +204,8 @@ public class TokenSources {
unsortedTokens = new ArrayList<Token>();
}
for (int tp = 0; tp < offsets.length; tp++) {
- Token token = new Token(offsets[tp].getStartOffset(), offsets[tp]
+ Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp]
.getEndOffset());
- token.setTermBuffer(terms[t]);
unsortedTokens.add(token);
}
} else {
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Tue Jun 1 10:35:13 2010
@@ -25,9 +25,9 @@ import java.util.List;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorOffsetInfo;
@@ -37,7 +37,7 @@ public final class TokenStreamFromTermPo
private Iterator<Token> tokensAtCurrentPosition;
- private TermAttribute termAttribute;
+ private CharTermAttribute termAttribute;
private PositionIncrementAttribute positionIncrementAttribute;
@@ -51,7 +51,7 @@ public final class TokenStreamFromTermPo
*/
public TokenStreamFromTermPositionVector(
final TermPositionVector termPositionVector) {
- termAttribute = addAttribute(TermAttribute.class);
+ termAttribute = addAttribute(CharTermAttribute.class);
positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
final String[] terms = termPositionVector.getTerms();
@@ -65,7 +65,7 @@ public final class TokenStreamFromTermPo
offsets[j].getStartOffset(), offsets[j].getEndOffset());
} else {
token = new Token();
- token.setTermBuffer(terms[i]);
+ token.setEmpty().append(terms[i]);
}
// Yes - this is the position, not the increment! This is for
// sorting. This value
@@ -100,7 +100,7 @@ public final class TokenStreamFromTermPo
if (this.tokensAtCurrentPosition.hasNext()) {
final Token next = this.tokensAtCurrentPosition.next();
clearAttributes();
- termAttribute.setTermBuffer(next.term());
+ termAttribute.setEmpty().append(next);
positionIncrementAttribute.setPositionIncrement(next
.getPositionIncrement());
offsetAttribute.setOffset(next.startOffset(), next.endOffset());
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Tue Jun 1 10:35:13 2010
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
@@ -296,16 +296,11 @@ public class HighlighterPhraseTest exten
private int i = -1;
- private TermAttribute termAttribute;
-
- private OffsetAttribute offsetAttribute;
-
- private PositionIncrementAttribute positionIncrementAttribute;
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+ private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
public TokenStreamSparse() {
- termAttribute = addAttribute(TermAttribute.class);
- offsetAttribute = addAttribute(OffsetAttribute.class);
- positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
reset();
}
@@ -316,8 +311,7 @@ public class HighlighterPhraseTest exten
return false;
}
clearAttributes();
- termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
- .term().length());
+ termAttribute.setEmpty().append(this.tokens[i]);
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
.endOffset());
positionIncrementAttribute.setPositionIncrement(this.tokens[i]
@@ -342,16 +336,11 @@ public class HighlighterPhraseTest exten
private int i = -1;
- private TermAttribute termAttribute;
-
- private OffsetAttribute offsetAttribute;
-
- private PositionIncrementAttribute positionIncrementAttribute;
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+ private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
public TokenStreamConcurrent() {
- termAttribute = addAttribute(TermAttribute.class);
- offsetAttribute = addAttribute(OffsetAttribute.class);
- positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class);
reset();
}
@@ -362,8 +351,7 @@ public class HighlighterPhraseTest exten
return false;
}
clearAttributes();
- termAttribute.setTermBuffer(this.tokens[i].term(), 0, this.tokens[i]
- .term().length());
+ termAttribute.setEmpty().append(this.tokens[i]);
offsetAttribute.setOffset(this.tokens[i].startOffset(), this.tokens[i]
.endOffset());
positionIncrementAttribute.setPositionIncrement(this.tokens[i]
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Tue Jun 1 10:35:13 2010
@@ -41,7 +41,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericField;
@@ -1424,13 +1424,10 @@ public class HighlighterTest extends Bas
return new TokenStream() {
Iterator<Token> iter;
List<Token> lst;
- private TermAttribute termAtt;
- private PositionIncrementAttribute posIncrAtt;
- private OffsetAttribute offsetAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
{
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- offsetAtt = addAttribute(OffsetAttribute.class);
lst = new ArrayList<Token>();
Token t;
t = createToken("hi", 0, 2);
@@ -1456,7 +1453,7 @@ public class HighlighterTest extends Bas
if(iter.hasNext()) {
Token token = iter.next();
clearAttributes();
- termAtt.setTermBuffer(token.term());
+ termAtt.setEmpty().append(token);
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
@@ -1473,13 +1470,10 @@ public class HighlighterTest extends Bas
return new TokenStream() {
Iterator<Token> iter;
List<Token> lst;
- private TermAttribute termAtt;
- private PositionIncrementAttribute posIncrAtt;
- private OffsetAttribute offsetAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
{
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- offsetAtt = addAttribute(OffsetAttribute.class);
lst = new ArrayList<Token>();
Token t;
t = createToken("hispeed", 0, 8);
@@ -1505,7 +1499,7 @@ public class HighlighterTest extends Bas
if(iter.hasNext()) {
Token token = iter.next();
clearAttributes();
- termAtt.setTermBuffer(token.term());
+ termAtt.setEmpty().append(token);
posIncrAtt.setPositionIncrement(token.getPositionIncrement());
offsetAtt.setOffset(token.startOffset(), token.endOffset());
return true;
@@ -1762,9 +1756,7 @@ public class HighlighterTest extends Bas
private static Token createToken(String term, int start, int offset)
{
- Token token = new Token(start, offset);
- token.setTermBuffer(term);
- return token;
+ return new Token(term, start, offset);
}
}
@@ -1795,7 +1787,7 @@ final class SynonymAnalyzer extends Anal
@Override
public TokenStream tokenStream(String arg0, Reader arg1) {
Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true);
- stream.addAttribute(TermAttribute.class);
+ stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
return new SynonymTokenizer(stream, synonyms);
@@ -1811,21 +1803,21 @@ final class SynonymTokenizer extends Tok
private Token currentRealToken = null;
private Map<String,String> synonyms;
StringTokenizer st = null;
- private TermAttribute realTermAtt;
+ private CharTermAttribute realTermAtt;
private PositionIncrementAttribute realPosIncrAtt;
private OffsetAttribute realOffsetAtt;
- private TermAttribute termAtt;
+ private CharTermAttribute termAtt;
private PositionIncrementAttribute posIncrAtt;
private OffsetAttribute offsetAtt;
public SynonymTokenizer(TokenStream realStream, Map<String,String> synonyms) {
this.realStream = realStream;
this.synonyms = synonyms;
- realTermAtt = realStream.addAttribute(TermAttribute.class);
+ realTermAtt = realStream.addAttribute(CharTermAttribute.class);
realPosIncrAtt = realStream.addAttribute(PositionIncrementAttribute.class);
realOffsetAtt = realStream.addAttribute(OffsetAttribute.class);
- termAtt = addAttribute(TermAttribute.class);
+ termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
}
@@ -1840,25 +1832,25 @@ final class SynonymTokenizer extends Tok
}
//Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
clearAttributes();
- termAtt.setTermBuffer(realTermAtt.term());
+ termAtt.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
- String expansions = synonyms.get(realTermAtt.term());
+ String expansions = synonyms.get(realTermAtt.toString());
if (expansions == null) {
return true;
}
st = new StringTokenizer(expansions, ",");
if (st.hasMoreTokens()) {
currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
- currentRealToken.setTermBuffer(realTermAtt.term());
+ currentRealToken.copyBuffer(realTermAtt.buffer(), 0, realTermAtt.length());
}
return true;
} else {
String tok = st.nextToken();
clearAttributes();
- termAtt.setTermBuffer(tok);
+ termAtt.setEmpty().append(tok);
offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
posIncrAtt.setPositionIncrement(0);
if (!st.hasMoreTokens()) {
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Tue Jun 1 10:35:13 2010
@@ -26,8 +26,8 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
@@ -221,14 +221,14 @@ public abstract class AbstractTestCase e
ch = 0;
}
- TermAttribute termAtt = addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
public boolean incrementToken() throws IOException {
if( !getNextPartialSnippet() )
return false;
clearAttributes();
- termAtt.setTermBuffer(snippet, startTerm, lenTerm);
+ termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm);
offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
return true;
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Tue Jun 1 10:35:13 2010
@@ -25,7 +25,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.AttributeImpl;
@@ -301,7 +301,7 @@ public class IndexTimeSynonymTest extend
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
- final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
+ final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
int p = 0;
@Override
Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Tue Jun 1 10:35:13 2010
@@ -561,7 +561,7 @@ public class InstantiatedIndexWriter imp
// untokenized
String fieldVal = field.stringValue();
Token token = new Token(0, fieldVal.length(), "untokenized");
- token.setTermBuffer(fieldVal);
+ token.setEmpty().append(fieldVal);
tokens.add(token);
fieldSetting.fieldLength++;
}
@@ -596,10 +596,10 @@ public class InstantiatedIndexWriter imp
for (Token token : eField_Tokens.getValue()) {
- TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.term());
+ TermDocumentInformationFactory termDocumentInformationFactory = termDocumentInformationFactoryByTermText.get(token.toString());
if (termDocumentInformationFactory == null) {
termDocumentInformationFactory = new TermDocumentInformationFactory();
- termDocumentInformationFactoryByTermText.put(token.term(), termDocumentInformationFactory);
+ termDocumentInformationFactoryByTermText.put(token.toString(), termDocumentInformationFactory);
}
//termDocumentInformationFactory.termFrequency++;
Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Tue Jun 1 10:35:13 2010
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -278,7 +278,7 @@ public class TestIndicesEquals extends L
tokens.add(t);
tokens.add(createToken("fin", 7, 9));
TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
- final AttributeImpl reusableToken = (AttributeImpl) addAttribute(TermAttribute.class);
+ final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
Iterator<Token> it = tokens.iterator();
@Override
@@ -601,16 +601,12 @@ public class TestIndicesEquals extends L
private static Token createToken(String term, int start, int offset)
{
- Token token = new Token(start, offset);
- token.setTermBuffer(term);
- return token;
+ return new Token(term, start, offset);
}
private static Token createToken(String term, int start, int offset, String type)
{
- Token token = new Token(start, offset, type);
- token.setTermBuffer(term);
- return token;
+ return new Token(term, start, offset, type);
}
Modified: lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java (original)
+++ lucene/dev/trunk/lucene/contrib/lucli/src/java/lucli/LuceneMethods.java Tue Jun 1 10:35:13 2010
@@ -36,8 +36,8 @@ import jline.ConsoleReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
@@ -303,14 +303,14 @@ class LuceneMethods {
int position = 0;
// Tokenize field and add to postingTable
TokenStream stream = analyzer.tokenStream(fieldName, reader);
- TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
try {
while (stream.incrementToken()) {
position += (posIncrAtt.getPositionIncrement() - 1);
position++;
- String name = termAtt.term();
+ String name = termAtt.toString();
Integer Count = tokenMap.get(name);
if (Count == null) { // not in there yet
tokenMap.put(name, Integer.valueOf(1)); //first one
Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Tue Jun 1 10:35:13 2010
@@ -30,9 +30,10 @@ import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.IndexReader;
@@ -51,6 +52,7 @@ import org.apache.lucene.search.Searcher
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.store.RAMDirectory; // for javadocs
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Constants; // for javadocs
/**
@@ -276,8 +278,8 @@ public class MemoryIndex implements Seri
return new TokenStream() {
private Iterator<T> iter = keywords.iterator();
private int start = 0;
- private TermAttribute termAtt = addAttribute(TermAttribute.class);
- private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
public boolean incrementToken() {
@@ -289,8 +291,8 @@ public class MemoryIndex implements Seri
String term = obj.toString();
clearAttributes();
- termAtt.setTermBuffer(term);
- offsetAtt.setOffset(start, start+termAtt.termLength());
+ termAtt.setEmpty().append(term);
+ offsetAtt.setOffset(start, start+termAtt.length());
start += term.length() + 1; // separate words by 1 (blank) character
return true;
}
@@ -340,13 +342,15 @@ public class MemoryIndex implements Seri
int numOverlapTokens = 0;
int pos = -1;
- TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
+ TermToBytesRefAttribute termAtt = stream.addAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
OffsetAttribute offsetAtt = stream.addAttribute(OffsetAttribute.class);
-
+ BytesRef ref = new BytesRef(10);
stream.reset();
while (stream.incrementToken()) {
- String term = termAtt.term();
+ termAtt.toBytesRef(ref);
+ // TODO: support non-UTF8 strings (like numerics) here
+ String term = ref.utf8ToString();
if (term.length() == 0) continue; // nothing to do
// if (DEBUG) System.err.println("token='" + term + "'");
numTokens++;
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Tue Jun 1 10:35:13 2010
@@ -26,7 +26,7 @@ import java.util.Iterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.BytesRef;
@@ -185,14 +185,14 @@ public class FuzzyLikeThisQuery extends
{
if(f.queryString==null) return;
TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString));
- TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
int corpusNumDocs=reader.numDocs();
Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet<String> processedTerms=new HashSet<String>();
while (ts.incrementToken())
{
- String term = termAtt.term();
+ String term = termAtt.toString();
if(!processedTerms.contains(term))
{
processedTerms.add(term);
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Tue Jun 1 10:35:13 2010
@@ -32,7 +32,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@@ -884,10 +884,10 @@ public final class MoreLikeThis {
TokenStream ts = analyzer.tokenStream(fieldName, r);
int tokenCount=0;
// for every token
- TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while (ts.incrementToken()) {
- String word = termAtt.term();
+ String word = termAtt.toString();
tokenCount++;
if(tokenCount>maxNumTokensParsed)
{
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java Tue Jun 1 10:35:13 2010
@@ -22,7 +22,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
@@ -86,12 +86,12 @@ public final class SimilarityQueries
throws IOException
{
TokenStream ts = a.tokenStream( field, new StringReader( body));
- TermAttribute termAtt = ts.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
BooleanQuery tmp = new BooleanQuery();
Set<String> already = new HashSet<String>(); // ignore dups
while (ts.incrementToken()) {
- String word = termAtt.term();
+ String word = termAtt.toString();
// ignore opt stop words
if ( stop != null &&
stop.contains( word)) continue;
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java Tue Jun 1 10:35:13 2010
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.Version;
@@ -107,7 +107,7 @@ public class AnalyzingQueryParser extend
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
int countTokens = 0;
while (true) {
@@ -116,7 +116,7 @@ public class AnalyzingQueryParser extend
} catch (IOException e) {
break;
}
- String term = termAtt.term();
+ String term = termAtt.toString();
if (!"".equals(term)) {
try {
tlist.set(countTokens++, term);
@@ -190,7 +190,7 @@ public class AnalyzingQueryParser extend
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
List<String> tlist = new ArrayList<String>();
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
while (true) {
try {
@@ -198,7 +198,7 @@ public class AnalyzingQueryParser extend
} catch (IOException e) {
break;
}
- tlist.add(termAtt.term());
+ tlist.add(termAtt.toString());
}
try {
@@ -237,13 +237,13 @@ public class AnalyzingQueryParser extend
throws ParseException {
// get Analyzer from superclass and tokenize the term
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr));
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
String nextToken = null;
boolean multipleTokens = false;
try {
if (source.incrementToken()) {
- nextToken = termAtt.term();
+ nextToken = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {
@@ -273,13 +273,13 @@ public class AnalyzingQueryParser extend
throws ParseException {
// get Analyzer from superclass and tokenize the terms
TokenStream source = getAnalyzer().tokenStream(field, new StringReader(part1));
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
boolean multipleTokens = false;
// part1
try {
if (source.incrementToken()) {
- part1 = termAtt.term();
+ part1 = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {
@@ -297,11 +297,11 @@ public class AnalyzingQueryParser extend
// part2
source = getAnalyzer().tokenStream(field, new StringReader(part2));
- termAtt = source.addAttribute(TermAttribute.class);
+ termAtt = source.addAttribute(CharTermAttribute.class);
try {
if (source.incrementToken()) {
- part2 = termAtt.term();
+ part2 = termAtt.toString();
}
multipleTokens = source.incrementToken();
} catch (IOException e) {
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.java Tue Jun 1 10:35:13 2010
@@ -307,7 +307,7 @@ public class PrecedenceQueryParser imple
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@@ -328,7 +328,7 @@ public class PrecedenceQueryParser imple
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
- return new TermQuery(new Term(field, termAtt.term()));
+ return new TermQuery(new Term(field, termAtt.toString()));
} else {
if (severalTokensAtSamePosition) {
if (positionCount == 1) {
@@ -337,7 +337,7 @@ public class PrecedenceQueryParser imple
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
TermQuery currentQuery = new TermQuery(
- new Term(field, termAtt.term()));
+ new Term(field, termAtt.toString()));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
return q;
@@ -352,7 +352,7 @@ public class PrecedenceQueryParser imple
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
- multiTerms.add(new Term(field, termAtt.term()));
+ multiTerms.add(new Term(field, termAtt.toString()));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@@ -363,7 +363,7 @@ public class PrecedenceQueryParser imple
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
- q.add(new Term(field, termAtt.term()));
+ q.add(new Term(field, termAtt.toString()));
}
return q;
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/precedence/PrecedenceQueryParser.jj Tue Jun 1 10:35:13 2010
@@ -331,7 +331,7 @@ public class PrecedenceQueryParser {
List<AttributeSource.State> list = new ArrayList<AttributeSource.State>();
int positionCount = 0;
boolean severalTokensAtSamePosition = false;
- TermAttribute termAtt = source.addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posincrAtt = source.addAttribute(PositionIncrementAttribute.class);
try {
@@ -352,7 +352,7 @@ public class PrecedenceQueryParser {
return null;
else if (list.size() == 1) {
source.restoreState(list.get(0));
- return new TermQuery(new Term(field, termAtt.term()));
+ return new TermQuery(new Term(field, termAtt.toString()));
} else {
if (severalTokensAtSamePosition) {
if (positionCount == 1) {
@@ -361,7 +361,7 @@ public class PrecedenceQueryParser {
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
TermQuery currentQuery = new TermQuery(
- new Term(field, termAtt.term()));
+ new Term(field, termAtt.toString()));
q.add(currentQuery, BooleanClause.Occur.SHOULD);
}
return q;
@@ -376,7 +376,7 @@ public class PrecedenceQueryParser {
mpq.add(multiTerms.toArray(new Term[0]));
multiTerms.clear();
}
- multiTerms.add(new Term(field, termAtt.term()));
+ multiTerms.add(new Term(field, termAtt.toString()));
}
mpq.add(multiTerms.toArray(new Term[0]));
return mpq;
@@ -387,7 +387,7 @@ public class PrecedenceQueryParser {
q.setSlop(phraseSlop);
for (int i = 0; i < list.size(); i++) {
source.restoreState(list.get(i));
- q.add(new Term(field, termAtt.term()));
+ q.add(new Term(field, termAtt.toString()));
}
return q;
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java Tue Jun 1 10:35:13 2010
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.queryParser.core.QueryNodeException;
import org.apache.lucene.queryParser.core.config.QueryConfigHandler;
import org.apache.lucene.queryParser.core.nodes.FieldQueryNode;
@@ -162,11 +162,11 @@ public class AnalyzerQueryNodeProcessor
// ignore
}
- if (!buffer.hasAttribute(TermAttribute.class)) {
+ if (!buffer.hasAttribute(CharTermAttribute.class)) {
return new NoTokenFoundQueryNode();
}
- TermAttribute termAtt = buffer.getAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class);
if (numTokens == 0) {
return new NoTokenFoundQueryNode();
@@ -177,7 +177,7 @@ public class AnalyzerQueryNodeProcessor
boolean hasNext;
hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.term();
+ term = termAtt.toString();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
@@ -197,7 +197,7 @@ public class AnalyzerQueryNodeProcessor
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.term();
+ term = termAtt.toString();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
@@ -224,7 +224,7 @@ public class AnalyzerQueryNodeProcessor
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.term();
+ term = termAtt.toString();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -290,7 +290,7 @@ public class AnalyzerQueryNodeProcessor
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- term = termAtt.term();
+ term = termAtt.toString();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java Tue Jun 1 10:35:13 2010
@@ -23,8 +23,8 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FuzzyQuery;
@@ -68,7 +68,7 @@ public class TestPrecedenceQueryParser e
boolean inPhrase = false;
int savedStart = 0, savedEnd = 0;
- TermAttribute termAtt = addAttribute(TermAttribute.class);
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@Override
@@ -76,19 +76,19 @@ public class TestPrecedenceQueryParser e
clearAttributes();
if (inPhrase) {
inPhrase = false;
- termAtt.setTermBuffer("phrase2");
+ termAtt.setEmpty().append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while(input.incrementToken())
- if (termAtt.term().equals("phrase")) {
+ if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
- termAtt.setTermBuffer("phrase1");
+ termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
- } else if (!termAtt.term().equals("stop"))
+ } else if (!termAtt.toString().equals("stop"))
return true;
return false;
}
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java Tue Jun 1 10:35:13 2010
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.queryParser.core.QueryNodeException;
import org.apache.lucene.queryParser.standard.config.DefaultOperatorAttribute.Operator;
@@ -163,24 +163,19 @@ public class TestMultiAnalyzerQPHelper e
private int prevStartOffset;
private int prevEndOffset;
- TermAttribute termAtt;
- PositionIncrementAttribute posIncrAtt;
- OffsetAttribute offsetAtt;
- TypeAttribute typeAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public TestFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- offsetAtt = addAttribute(OffsetAttribute.class);
- typeAtt = addAttribute(TypeAttribute.class);
-
}
@Override
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
- termAtt.setTermBuffer("multi" + (multiToken + 1));
+ termAtt.setEmpty().append("multi" + (multiToken + 1));
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
typeAtt.setType(prevType);
posIncrAtt.setPositionIncrement(0);
@@ -194,7 +189,7 @@ public class TestMultiAnalyzerQPHelper e
prevType = typeAtt.type();
prevStartOffset = offsetAtt.startOffset();
prevEndOffset = offsetAtt.endOffset();
- String text = termAtt.term();
+ String text = termAtt.toString();
if (text.equals("triplemulti")) {
multiToken = 2;
return true;
@@ -228,21 +223,19 @@ public class TestMultiAnalyzerQPHelper e
private class TestPosIncrementFilter extends TokenFilter {
- TermAttribute termAtt;
- PositionIncrementAttribute posIncrAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
public TestPosIncrementFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
- if (termAtt.term().equals("the")) {
+ if (termAtt.toString().equals("the")) {
// stopword, do nothing
- } else if (termAtt.term().equals("quick")) {
+ } else if (termAtt.toString().equals("quick")) {
posIncrAtt.setPositionIncrement(2);
return true;
} else {
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java Tue Jun 1 10:35:13 2010
@@ -23,9 +23,9 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.util.LuceneTestCase;
@@ -157,24 +157,19 @@ public class TestMultiAnalyzerWrapper ex
private int prevStartOffset;
private int prevEndOffset;
- TermAttribute termAtt;
- PositionIncrementAttribute posIncrAtt;
- OffsetAttribute offsetAtt;
- TypeAttribute typeAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
public TestFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- offsetAtt = addAttribute(OffsetAttribute.class);
- typeAtt = addAttribute(TypeAttribute.class);
-
}
@Override
public final boolean incrementToken() throws java.io.IOException {
if (multiToken > 0) {
- termAtt.setTermBuffer("multi" + (multiToken + 1));
+ termAtt.setEmpty().append("multi" + (multiToken + 1));
offsetAtt.setOffset(prevStartOffset, prevEndOffset);
typeAtt.setType(prevType);
posIncrAtt.setPositionIncrement(0);
@@ -188,7 +183,7 @@ public class TestMultiAnalyzerWrapper ex
prevType = typeAtt.type();
prevStartOffset = offsetAtt.startOffset();
prevEndOffset = offsetAtt.endOffset();
- String text = termAtt.term();
+ String text = termAtt.toString();
if (text.equals("triplemulti")) {
multiToken = 2;
return true;
@@ -222,21 +217,19 @@ public class TestMultiAnalyzerWrapper ex
private class TestPosIncrementFilter extends TokenFilter {
- TermAttribute termAtt;
- PositionIncrementAttribute posIncrAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
public TestPosIncrementFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
- posIncrAtt = addAttribute(PositionIncrementAttribute.class);
}
@Override
public final boolean incrementToken() throws java.io.IOException {
while (input.incrementToken()) {
- if (termAtt.term().equals("the")) {
+ if (termAtt.toString().equals("the")) {
// stopword, do nothing
- } else if (termAtt.term().equals("quick")) {
+ } else if (termAtt.toString().equals("quick")) {
posIncrAtt.setPositionIncrement(2);
return true;
} else {
Modified: lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java?rev=950008&r1=950007&r2=950008&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (original)
+++ lucene/dev/trunk/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java Tue Jun 1 10:35:13 2010
@@ -37,8 +37,8 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
@@ -96,8 +96,8 @@ public class TestQPHelper extends Locali
public static Analyzer qpAnalyzer = new QPTestAnalyzer();
public static final class QPTestFilter extends TokenFilter {
- TermAttribute termAtt;
- OffsetAttribute offsetAtt;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
/**
* Filter which discards the token 'stop' and which expands the token
@@ -105,8 +105,6 @@ public class TestQPHelper extends Locali
*/
public QPTestFilter(TokenStream in) {
super(in);
- termAtt = addAttribute(TermAttribute.class);
- offsetAtt = addAttribute(OffsetAttribute.class);
}
boolean inPhrase = false;
@@ -117,19 +115,19 @@ public class TestQPHelper extends Locali
if (inPhrase) {
inPhrase = false;
clearAttributes();
- termAtt.setTermBuffer("phrase2");
+ termAtt.setEmpty().append("phrase2");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
} else
while (input.incrementToken()) {
- if (termAtt.term().equals("phrase")) {
+ if (termAtt.toString().equals("phrase")) {
inPhrase = true;
savedStart = offsetAtt.startOffset();
savedEnd = offsetAtt.endOffset();
- termAtt.setTermBuffer("phrase1");
+ termAtt.setEmpty().append("phrase1");
offsetAtt.setOffset(savedStart, savedEnd);
return true;
- } else if (!termAtt.term().equals("stop"))
+ } else if (!termAtt.toString().equals("stop"))
return true;
}
return false;
@@ -1158,7 +1156,7 @@ public class TestQPHelper extends Locali
private class CannedTokenStream extends TokenStream {
private int upto = 0;
final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
- final TermAttribute term = addAttribute(TermAttribute.class);
+ final CharTermAttribute term = addAttribute(CharTermAttribute.class);
@Override
public boolean incrementToken() {
clearAttributes();
@@ -1167,16 +1165,16 @@ public class TestQPHelper extends Locali
}
if (upto == 0) {
posIncr.setPositionIncrement(1);
- term.setTermBuffer("a");
+ term.setEmpty().append("a");
} else if (upto == 1) {
posIncr.setPositionIncrement(1);
- term.setTermBuffer("b");
+ term.setEmpty().append("b");
} else if (upto == 2) {
posIncr.setPositionIncrement(0);
- term.setTermBuffer("c");
+ term.setEmpty().append("c");
} else {
posIncr.setPositionIncrement(0);
- term.setTermBuffer("d");
+ term.setEmpty().append("d");
}
upto++;
return true;