You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ch...@apache.org on 2011/09/12 07:50:28 UTC
svn commit: r1169607 [1/2] - in /lucene/dev/trunk: lucene/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/
lucene/src/java/org/apache/lucene/analysis/...
Author: chrism
Date: Mon Sep 12 05:50:26 2011
New Revision: 1169607
URL: http://svn.apache.org/viewvc?rev=1169607&view=rev
Log:
LUCENE-3396: Converted most Analyzers over to using ReusableAnalyzerBase
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestAssertions.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
lucene/dev/trunk/modules/analysis/CHANGES.txt
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java
lucene/dev/trunk/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
lucene/dev/trunk/modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
lucene/dev/trunk/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyFilter.java
lucene/dev/trunk/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
lucene/dev/trunk/modules/facet/src/test/org/apache/lucene/facet/search/CategoryListIteratorTest.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
lucene/dev/trunk/modules/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Sep 12 05:50:26 2011
@@ -171,6 +171,9 @@ Changes in backwards compatibility polic
IndexableFieldType. See MIGRATE.txt for more details.
(Nikola Tankovic, Mike McCandless, Chris Male)
+* LUCENE-3396: ReusableAnalyzerBase.TokenStreamComponents.reset(Reader) now returns void instead
+ of boolean. If a Component cannot be reset, it should throw an Exception.
+
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
@@ -523,6 +526,12 @@ New features
(David Mark Nemeskey via Robert Muir)
+* LUCENE-3396: ReusableAnalyzerBase now provides a ReuseStrategy abstraction which
+ controls how TokenStreamComponents are reused per request. Two implementations are
+ provided - GlobalReuseStrategy which implements the current behavior of sharing
+ components between all fields, and PerFieldReuseStrategy which shares per field.
+ (Chris Male)
+
Optimizations
* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Mon Sep 12 05:50:26 2011
@@ -1802,7 +1802,7 @@ public class HighlighterTest extends Bas
// behaviour to synonyms
// ===================================================================
-final class SynonymAnalyzer extends Analyzer {
+final class SynonymAnalyzer extends ReusableAnalyzerBase {
private Map<String,String> synonyms;
public SynonymAnalyzer(Map<String,String> synonyms) {
@@ -1816,12 +1816,12 @@ final class SynonymAnalyzer extends Anal
* java.io.Reader)
*/
@Override
- public TokenStream tokenStream(String arg0, Reader arg1) {
+ public TokenStreamComponents createComponents(String arg0, Reader arg1) {
Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true);
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
- return new SynonymTokenizer(stream, synonyms);
+ return new TokenStreamComponents(stream, new SynonymTokenizer(stream, synonyms));
}
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java Mon Sep 12 05:50:26 2011
@@ -20,10 +20,7 @@ package org.apache.lucene.search.highlig
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase {
@@ -52,15 +49,14 @@ public class OffsetLimitTokenFilterTest
assertTokenStreamContents(filter, new String[] {"short", "toolong",
"evenmuchlongertext"});
- // TODO: This is not actually testing reuse! (reusableTokenStream is not implemented)
- checkOneTermReuse(new Analyzer() {
+ checkOneTermReuse(new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false);
- return new OffsetLimitTokenFilter(tokenizer, 10);
+ return new TokenStreamComponents(tokenizer, new OffsetLimitTokenFilter(tokenizer, 10));
}
}, "llenges", "llenges");
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java Mon Sep 12 05:50:26 2011
@@ -20,9 +20,7 @@ package org.apache.lucene.search.highlig
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -50,15 +48,15 @@ import org.apache.lucene.util.LuceneTest
public class TokenSourcesTest extends LuceneTestCase {
private static final String FIELD = "text";
- private static final class OverlapAnalyzer extends Analyzer {
+ private static final class OverlapAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new TokenStreamOverlap();
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new TokenStreamOverlap());
}
}
- private static final class TokenStreamOverlap extends TokenStream {
+ private static final class TokenStreamOverlap extends Tokenizer {
private Token[] tokens;
private int i = -1;
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Mon Sep 12 05:50:26 2011
@@ -24,11 +24,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
@@ -198,10 +194,10 @@ public abstract class AbstractTestCase e
return phraseQuery;
}
- static final class BigramAnalyzer extends Analyzer {
+ static final class BigramAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new BasicNGramTokenizer( reader );
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new BasicNGramTokenizer(reader));
}
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java Mon Sep 12 05:50:26 2011
@@ -22,9 +22,7 @@ import java.io.Reader;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanClause.Occur;
@@ -292,15 +290,15 @@ public class IndexTimeSynonymTest extend
return token;
}
- public static final class TokenArrayAnalyzer extends Analyzer {
- Token[] tokens;
- public TokenArrayAnalyzer( Token... tokens ){
+ public static final class TokenArrayAnalyzer extends ReusableAnalyzerBase {
+ final Token[] tokens;
+ public TokenArrayAnalyzer(Token... tokens) {
this.tokens = tokens;
}
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
int p = 0;
@@ -318,7 +316,7 @@ public class IndexTimeSynonymTest extend
this.p = 0;
}
};
- return ts;
+ return new TokenStreamComponents(ts);
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/ReusableAnalyzerBase.java Mon Sep 12 05:50:26 2011
@@ -17,8 +17,13 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.CloseableThreadLocal;
+
import java.io.IOException;
import java.io.Reader;
+import java.util.HashMap;
+import java.util.Map;
/**
* An convenience subclass of Analyzer that makes it easy to implement
@@ -38,6 +43,16 @@ import java.io.Reader;
*/
public abstract class ReusableAnalyzerBase extends Analyzer {
+ private final ReuseStrategy reuseStrategy;
+
+ public ReusableAnalyzerBase() {
+ this(new GlobalReuseStrategy());
+ }
+
+ public ReusableAnalyzerBase(ReuseStrategy reuseStrategy) {
+ this.reuseStrategy = reuseStrategy;
+ }
+
/**
* Creates a new {@link TokenStreamComponents} instance for this analyzer.
*
@@ -66,14 +81,15 @@ public abstract class ReusableAnalyzerBa
@Override
public final TokenStream reusableTokenStream(final String fieldName,
final Reader reader) throws IOException {
- TokenStreamComponents streamChain = (TokenStreamComponents)
- getPreviousTokenStream();
+ TokenStreamComponents components = reuseStrategy.getReusableComponents(fieldName);
final Reader r = initReader(reader);
- if (streamChain == null || !streamChain.reset(r)) {
- streamChain = createComponents(fieldName, r);
- setPreviousTokenStream(streamChain);
+ if (components == null) {
+ components = createComponents(fieldName, r);
+ reuseStrategy.setReusableComponents(fieldName, components);
+ } else {
+ components.reset(r);
}
- return streamChain.getTokenStream();
+ return components.getTokenStream();
}
/**
@@ -98,7 +114,16 @@ public abstract class ReusableAnalyzerBa
protected Reader initReader(Reader reader) {
return reader;
}
-
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void close() {
+ super.close();
+ reuseStrategy.close();
+ }
+
/**
* This class encapsulates the outer components of a token stream. It provides
* access to the source ({@link Tokenizer}) and the outer end (sink), an
@@ -137,22 +162,16 @@ public abstract class ReusableAnalyzerBa
}
/**
- * Resets the encapsulated components with the given reader. This method by
- * default returns <code>true</code> indicating that the components have
- * been reset successfully. Subclasses of {@link ReusableAnalyzerBase} might use
- * their own {@link TokenStreamComponents} returning <code>false</code> if
- * the components cannot be reset.
+ * Resets the encapsulated components with the given reader. If the components
+ * cannot be reset, an Exception should be thrown.
*
* @param reader
* a reader to reset the source component
- * @return <code>true</code> if the components were reset, otherwise
- * <code>false</code>
* @throws IOException
* if the component's reset method throws an {@link IOException}
*/
- protected boolean reset(final Reader reader) throws IOException {
+ protected void reset(final Reader reader) throws IOException {
source.reset(reader);
- return true;
}
/**
@@ -166,4 +185,124 @@ public abstract class ReusableAnalyzerBa
}
+ /**
+ * Strategy defining how TokenStreamComponents are reused per call to
+ * {@link ReusableAnalyzerBase#tokenStream(String, java.io.Reader)}.
+ */
+ public static abstract class ReuseStrategy {
+
+ private CloseableThreadLocal<Object> storedValue = new CloseableThreadLocal<Object>();
+
+ /**
+ * Gets the reusable TokenStreamComponents for the field with the given name
+ *
+ * @param fieldName Name of the field whose reusable TokenStreamComponents
+ * are to be retrieved
+ * @return Reusable TokenStreamComponents for the field, or {@code null}
+ * if there was no previous components for the field
+ */
+ public abstract TokenStreamComponents getReusableComponents(String fieldName);
+
+ /**
+ * Stores the given TokenStreamComponents as the reusable components for the
+ * field with the give name
+ *
+ * @param fieldName Name of the field whose TokenStreamComponents are being set
+ * @param components TokenStreamComponents which are to be reused for the field
+ */
+ public abstract void setReusableComponents(String fieldName, TokenStreamComponents components);
+
+ /**
+ * Returns the currently stored value
+ *
+ * @return Currently stored value or {@code null} if no value is stored
+ */
+ protected final Object getStoredValue() {
+ try {
+ return storedValue.get();
+ } catch (NullPointerException npe) {
+ if (storedValue == null) {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ } else {
+ throw npe;
+ }
+ }
+ }
+
+ /**
+ * Sets the stored value
+ *
+ * @param storedValue Value to store
+ */
+ protected final void setStoredValue(Object storedValue) {
+ try {
+ this.storedValue.set(storedValue);
+ } catch (NullPointerException npe) {
+ if (storedValue == null) {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ } else {
+ throw npe;
+ }
+ }
+ }
+
+ /**
+ * Closes the ReuseStrategy, freeing any resources
+ */
+ public void close() {
+ storedValue.close();
+ storedValue = null;
+ }
+ }
+
+ /**
+ * Implementation of {@link ReuseStrategy} that reuses the same components for
+ * every field.
+ */
+ public final static class GlobalReuseStrategy extends ReuseStrategy {
+
+ /**
+ * {@inheritDoc}
+ */
+ public TokenStreamComponents getReusableComponents(String fieldName) {
+ return (TokenStreamComponents) getStoredValue();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void setReusableComponents(String fieldName, TokenStreamComponents components) {
+ setStoredValue(components);
+ }
+ }
+
+ /**
+ * Implementation of {@link ReuseStrategy} that reuses components per-field by
+ * maintaining a Map of TokenStreamComponent per field name.
+ */
+ public static class PerFieldReuseStrategy extends ReuseStrategy {
+
+ /**
+ * {@inheritDoc}
+ */
+ @SuppressWarnings("unchecked")
+ public TokenStreamComponents getReusableComponents(String fieldName) {
+ Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue();
+ return componentsPerField != null ? componentsPerField.get(fieldName) : null;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @SuppressWarnings("unchecked")
+ public void setReusableComponents(String fieldName, TokenStreamComponents components) {
+ Map<String, TokenStreamComponents> componentsPerField = (Map<String, TokenStreamComponents>) getStoredValue();
+ if (componentsPerField == null) {
+ componentsPerField = new HashMap<String, TokenStreamComponents>();
+ setStoredValue(componentsPerField);
+ }
+ componentsPerField.put(fieldName, components);
+ }
+ }
+
}
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java Mon Sep 12 05:50:26 2011
@@ -42,7 +42,7 @@ import org.apache.lucene.util.automaton.
* </ul>
* @see MockTokenizer
*/
-public final class MockAnalyzer extends Analyzer {
+public final class MockAnalyzer extends ReusableAnalyzerBase {
private final CharacterRunAutomaton runAutomaton;
private final boolean lowerCase;
private final CharacterRunAutomaton filter;
@@ -62,6 +62,7 @@ public final class MockAnalyzer extends
* @param enablePositionIncrements true if position increments should reflect filtered terms.
*/
public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) {
+ super(new PerFieldReuseStrategy());
this.random = random;
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
@@ -88,41 +89,11 @@ public final class MockAnalyzer extends
}
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
tokenizer.setEnableChecks(enableChecks);
TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements);
- filt = maybePayload(filt, fieldName);
- return filt;
- }
-
- private class SavedStreams {
- MockTokenizer tokenizer;
- TokenFilter filter;
- }
-
- @Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader)
- throws IOException {
- @SuppressWarnings("unchecked") Map<String,SavedStreams> map = (Map) getPreviousTokenStream();
- if (map == null) {
- map = new HashMap<String,SavedStreams>();
- setPreviousTokenStream(map);
- }
-
- SavedStreams saved = map.get(fieldName);
- if (saved == null) {
- saved = new SavedStreams();
- saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase);
- saved.tokenizer.setEnableChecks(enableChecks);
- saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements);
- saved.filter = maybePayload(saved.filter, fieldName);
- map.put(fieldName, saved);
- return saved.filter;
- } else {
- saved.tokenizer.reset(reader);
- return saved.filter;
- }
+ return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName));
}
private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) {
Modified: lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java Mon Sep 12 05:50:26 2011
@@ -30,16 +30,15 @@ import java.io.Reader;
*
*
**/
-public final class MockPayloadAnalyzer extends Analyzer {
+public final class MockPayloadAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
- return new MockPayloadFilter(result, fieldName);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName));
}
}
-
/**
*
*
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestAssertions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestAssertions.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestAssertions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/TestAssertions.java Mon Sep 12 05:50:26 2011
@@ -19,6 +19,7 @@ package org.apache.lucene;
import java.io.Reader;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
@@ -34,32 +35,36 @@ public class TestAssertions extends Luce
}
}
- static class TestAnalyzer1 extends Analyzer {
- @Override
- public final TokenStream tokenStream(String s, Reader r) { return null; }
+ static class TestAnalyzer1 extends ReusableAnalyzerBase {
+
@Override
- public final TokenStream reusableTokenStream(String s, Reader r) { return null; }
+ protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
+ return null;
+ }
}
- static final class TestAnalyzer2 extends Analyzer {
- @Override
- public TokenStream tokenStream(String s, Reader r) { return null; }
+ static final class TestAnalyzer2 extends ReusableAnalyzerBase {
+
@Override
- public TokenStream reusableTokenStream(String s, Reader r) { return null; }
+ protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
+ return null;
+ }
}
- static class TestAnalyzer3 extends Analyzer {
- @Override
- public TokenStream tokenStream(String s, Reader r) { return null; }
+ static class TestAnalyzer3 extends ReusableAnalyzerBase {
+
@Override
- public TokenStream reusableTokenStream(String s, Reader r) { return null; }
+ protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
+ return null;
+ }
}
- static class TestAnalyzer4 extends Analyzer {
- @Override
- public final TokenStream tokenStream(String s, Reader r) { return null; }
+ static class TestAnalyzer4 extends ReusableAnalyzerBase {
+
@Override
- public TokenStream reusableTokenStream(String s, Reader r) { return null; }
+ protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
+ return null;
+ }
}
static class TestTokenStream1 extends TokenStream {
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon Sep 12 05:50:26 2011
@@ -20,11 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -107,10 +103,10 @@ public class TestDocumentWriter extends
}
public void testPositionIncrementGap() throws IOException {
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
}
@Override
@@ -142,10 +138,11 @@ public class TestDocumentWriter extends
}
public void testTokenReuse() throws IOException {
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new TokenFilter(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new TokenFilter(tokenizer) {
boolean first = true;
AttributeSource.State state;
@@ -187,7 +184,7 @@ public class TestDocumentWriter extends
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- };
+ });
}
};
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Sep 12 05:50:26 2011
@@ -31,11 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.BinaryField;
@@ -1710,10 +1706,10 @@ public class TestIndexWriter extends Luc
dir.close();
}
- static final class StringSplitAnalyzer extends Analyzer {
+ static final class StringSplitAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new StringSplitTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new StringSplitTokenizer(reader));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Mon Sep 12 05:50:26 2011
@@ -23,11 +23,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
@@ -179,21 +175,20 @@ public class TestIndexWriterCommit exten
Analyzer analyzer;
if (random.nextBoolean()) {
// no payloads
- analyzer = new Analyzer() {
+ analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
}
};
} else {
// fixed length payloads
final int length = random.nextInt(200);
- analyzer = new Analyzer() {
+ analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockFixedLengthPayloadFilter(random,
- new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
- length);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ return new TokenStreamComponents(tokenizer, new MockFixedLengthPayloadFilter(random, tokenizer, length));
}
};
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Mon Sep 12 05:50:26 2011
@@ -26,10 +26,7 @@ import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
@@ -902,10 +899,10 @@ public class TestIndexWriterDelete exten
final Random r = random;
Directory dir = newDirectory();
// note this test explicitly disables payloads
- final Analyzer analyzer = new Analyzer() {
+ final Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
}
};
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Mon Sep 12 05:50:26 2011
@@ -27,11 +27,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Random;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -390,12 +386,12 @@ public class TestIndexWriterExceptions e
doc.add(newField("field", "a field", TextField.TYPE_STORED));
w.addDocument(doc);
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase(new ReusableAnalyzerBase.PerFieldReuseStrategy()) {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
+ return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
};
@@ -458,13 +454,13 @@ public class TestIndexWriterExceptions e
// LUCENE-1072
public void testExceptionFromTokenStream() throws IOException {
Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
+ IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new TokenFilter(tokenizer) {
+ return new TokenStreamComponents(tokenizer, new TokenFilter(tokenizer) {
private int count = 0;
@Override
@@ -480,7 +476,7 @@ public class TestIndexWriterExceptions e
super.reset();
this.count = 0;
}
- };
+ });
}
});
@@ -595,12 +591,12 @@ public class TestIndexWriterExceptions e
}
public void testDocumentsWriterExceptions() throws IOException {
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase(new ReusableAnalyzerBase.PerFieldReuseStrategy()) {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
+ return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
};
@@ -691,12 +687,12 @@ public class TestIndexWriterExceptions e
}
public void testDocumentsWriterExceptionThreads() throws Exception {
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase(new ReusableAnalyzerBase.PerFieldReuseStrategy()) {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
- return new CrashingFilter(fieldName, tokenizer);
+ return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
};
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Mon Sep 12 05:50:26 2011
@@ -20,10 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.codecs.CodecProvider;
@@ -71,10 +68,10 @@ public class TestLazyProxSkipping extend
private void createIndex(int numHits) throws IOException {
int numDocs = 500;
- final Analyzer analyzer = new Analyzer() {
+ final Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
}
};
Directory directory = new SeekCountingDirectory(new RAMDirectory());
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Mon Sep 12 05:50:26 2011
@@ -21,10 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
@@ -114,11 +111,12 @@ public class TestMultiLevelSkipList exte
assertEquals("Wrong payload for the target " + target + ": " + b.bytes[b.offset], (byte) target, b.bytes[b.offset]);
}
- private static class PayloadAnalyzer extends Analyzer {
+ private static class PayloadAnalyzer extends ReusableAnalyzerBase {
private final AtomicInteger payloadCount = new AtomicInteger(-1);
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new PayloadFilter(payloadCount, new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ return new TokenStreamComponents(tokenizer, new PayloadFilter(payloadCount, tokenizer));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestPayloads.java Mon Sep 12 05:50:26 2011
@@ -25,11 +25,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
@@ -105,12 +101,12 @@ public class TestPayloads extends Lucene
// so this field is used to check if the DocumentWriter correctly enables the payloads bit
// even if only some term positions have payloads
d.add(newField("f2", "This field has payloads in all docs", TextField.TYPE_UNSTORED));
- d.add(newField("f2", "This field has payloads in all docs", TextField.TYPE_UNSTORED));
+ d.add(newField("f2", "This field has payloads in all docs NO PAYLOAD", TextField.TYPE_UNSTORED));
// this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads
// enabled in only some documents
d.add(newField("f3", "This field has payloads in some docs", TextField.TYPE_UNSTORED));
// only add payload data for field f2
- analyzer.setPayloadData("f2", 1, "somedata".getBytes(), 0, 1);
+ analyzer.setPayloadData("f2", "somedata".getBytes(), 0, 1);
writer.addDocument(d);
// flush
writer.close();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java Mon Sep 12 05:50:26 2011
@@ -20,8 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -64,10 +63,10 @@ public class TestSameTokenSamePosition e
}
}
-final class BugReproAnalyzer extends Analyzer{
+final class BugReproAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String arg0, Reader arg1) {
- return new BugReproAnalyzerTokenizer();
+ public TokenStreamComponents createComponents(String arg0, Reader arg1) {
+ return new TokenStreamComponents(new BugReproAnalyzerTokenizer());
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java Mon Sep 12 05:50:26 2011
@@ -137,7 +137,7 @@ public class TestTermVectorsReader exten
super.tearDown();
}
- private class MyTokenStream extends TokenStream {
+ private class MyTokenStream extends Tokenizer {
private int tokenUpto;
private final CharTermAttribute termAtt;
@@ -175,10 +175,10 @@ public class TestTermVectorsReader exten
}
}
- private class MyAnalyzer extends Analyzer {
+ private class MyAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MyTokenStream();
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MyTokenStream());
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java Mon Sep 12 05:50:26 2011
@@ -22,7 +22,7 @@ import java.io.Reader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
@@ -77,10 +77,10 @@ public class TestTermdocPerf extends Luc
void addDocs(final Random random, Directory dir, final int ndocs, String field, final String val, final int maxTF, final float percentDocs) throws IOException {
final RepeatingTokenStream ts = new RepeatingTokenStream(val, random, percentDocs, maxTF);
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return ts;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(ts);
}
};
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Mon Sep 12 05:50:26 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -345,7 +346,7 @@ public class TestMultiPhraseQuery extend
}
}
- private static class CannedAnalyzer extends Analyzer {
+ private static class CannedAnalyzer extends ReusableAnalyzerBase {
private final TokenAndPos[] tokens;
public CannedAnalyzer(TokenAndPos[] tokens) {
@@ -353,8 +354,8 @@ public class TestMultiPhraseQuery extend
}
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new CannedTokenizer(tokens);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new CannedTokenizer(tokens));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java Mon Sep 12 05:50:26 2011
@@ -55,10 +55,10 @@ public class TestPhraseQuery extends Luc
@BeforeClass
public static void beforeClass() throws Exception {
directory = newDirectory();
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
}
@Override
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java Mon Sep 12 05:50:26 2011
@@ -56,10 +56,10 @@ public class TestPositionIncrement exten
final static boolean VERBOSE = false;
public void testSetPosition() throws Exception {
- Analyzer analyzer = new Analyzer() {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new TokenStream() {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new Tokenizer() {
private final String[] TOKENS = {"1", "2", "3", "4", "5"};
private final int[] INCREMENTS = {0, 2, 1, 0, 1};
private int i = 0;
@@ -85,7 +85,7 @@ public class TestPositionIncrement exten
super.reset();
this.i = 0;
}
- };
+ });
}
};
Directory store = newDirectory();
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestTermRangeQuery.java Mon Sep 12 05:50:26 2011
@@ -190,7 +190,7 @@ public class TestTermRangeQuery extends
assertFalse("queries with different inclusive are not equal", query.equals(other));
}
- private static class SingleCharAnalyzer extends Analyzer {
+ private static class SingleCharAnalyzer extends ReusableAnalyzerBase {
private static class SingleCharTokenizer extends Tokenizer {
char[] buffer = new char[1];
@@ -225,19 +225,8 @@ public class TestTermRangeQuery extends
}
@Override
- public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
- Tokenizer tokenizer = (Tokenizer) getPreviousTokenStream();
- if (tokenizer == null) {
- tokenizer = new SingleCharTokenizer(reader);
- setPreviousTokenStream(tokenizer);
- } else
- tokenizer.reset(reader);
- return tokenizer;
- }
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new SingleCharTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new SingleCharTokenizer(reader));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java Mon Sep 12 05:50:26 2011
@@ -55,14 +55,16 @@ public class PayloadHelper {
public IndexReader reader;
- public final class PayloadAnalyzer extends Analyzer {
+ public final class PayloadAnalyzer extends ReusableAnalyzerBase {
+ public PayloadAnalyzer() {
+ super(new PerFieldReuseStrategy());
+ }
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- result = new PayloadFilter(result, fieldName);
- return result;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Mon Sep 12 05:50:26 2011
@@ -55,12 +55,11 @@ public class TestPayloadNearQuery extend
private static byte[] payload2 = new byte[]{2};
private static byte[] payload4 = new byte[]{4};
- private static class PayloadAnalyzer extends Analyzer {
+ private static class PayloadAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- result = new PayloadFilter(result, fieldName);
- return result;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Mon Sep 12 05:50:26 2011
@@ -64,14 +64,16 @@ public class TestPayloadTermQuery extend
private static final byte[] payloadMultiField2 = new byte[]{4};
protected static Directory directory;
- private static class PayloadAnalyzer extends Analyzer {
+ private static class PayloadAnalyzer extends ReusableAnalyzerBase {
+ private PayloadAnalyzer() {
+ super(new PerFieldReuseStrategy());
+ }
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- result = new PayloadFilter(result, fieldName);
- return result;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestBasics.java Mon Sep 12 05:50:26 2011
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -70,14 +71,12 @@ public class TestBasics extends LuceneTe
private static Directory directory;
static final class SimplePayloadFilter extends TokenFilter {
- String fieldName;
int pos;
final PayloadAttribute payloadAttr;
final CharTermAttribute termAttr;
- public SimplePayloadFilter(TokenStream input, String fieldName) {
+ public SimplePayloadFilter(TokenStream input) {
super(input);
- this.fieldName = fieldName;
pos = 0;
payloadAttr = input.addAttribute(PayloadAttribute.class);
termAttr = input.addAttribute(CharTermAttribute.class);
@@ -105,7 +104,7 @@ public class TestBasics extends LuceneTe
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
- return new SimplePayloadFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true), fieldName);
+ return new SimplePayloadFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true));
}
};
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Mon Sep 12 05:50:26 2011
@@ -23,10 +23,7 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -479,18 +476,16 @@ public class TestPayloadSpans extends Lu
assertEquals(numSpans, cnt);
}
- final class PayloadAnalyzer extends Analyzer {
+ final class PayloadAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- result = new PayloadFilter(result, fieldName);
- return result;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result));
}
}
final class PayloadFilter extends TokenFilter {
- String fieldName;
Set<String> entities = new HashSet<String>();
Set<String> nopayload = new HashSet<String>();
int pos;
@@ -498,9 +493,8 @@ public class TestPayloadSpans extends Lu
CharTermAttribute termAtt;
PositionIncrementAttribute posIncrAtt;
- public PayloadFilter(TokenStream input, String fieldName) {
+ public PayloadFilter(TokenStream input) {
super(input);
- this.fieldName = fieldName;
pos = 0;
entities.add("xx");
entities.add("one");
@@ -536,13 +530,12 @@ public class TestPayloadSpans extends Lu
}
}
- public final class TestPayloadAnalyzer extends Analyzer {
+ public final class TestPayloadAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- result = new PayloadFilter(result, fieldName);
- return result;
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(result, new PayloadFilter(result));
}
}
}
Modified: lucene/dev/trunk/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/CHANGES.txt?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/trunk/modules/analysis/CHANGES.txt Mon Sep 12 05:50:26 2011
@@ -106,3 +106,5 @@ Build
* LUCENE-2413: All analyzers in contrib/analyzers and contrib/icu were moved to the
analysis module. The 'smartcn' and 'stempel' components now depend on 'common'.
(Robert Muir)
+
+ * LUCENE-3376: Moved ReusableAnalyzerBase into lucene core. (Chris Male)
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PatternAnalyzer.java Mon Sep 12 05:50:26 2011
@@ -27,6 +27,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.StopAnalyzer;
@@ -66,7 +67,7 @@ import org.apache.lucene.util.Version;
* @deprecated (4.0) use the pattern-based analysis in the analysis/pattern package instead.
*/
@Deprecated
-public final class PatternAnalyzer extends Analyzer {
+public final class PatternAnalyzer extends ReusableAnalyzerBase {
/** <code>"\\W+"</code>; Divides text at non-letters (NOT Character.isLetter(c)) */
public static final Pattern NON_WORD_PATTERN = Pattern.compile("\\W+");
@@ -187,25 +188,21 @@ public final class PatternAnalyzer exten
* the string to tokenize
* @return a new token stream
*/
- public TokenStream tokenStream(String fieldName, String text) {
+ public TokenStreamComponents createComponents(String fieldName, String text) {
// Ideally the Analyzer superclass should have a method with the same signature,
// with a default impl that simply delegates to the StringReader flavour.
if (text == null)
throw new IllegalArgumentException("text must not be null");
- TokenStream stream;
if (pattern == NON_WORD_PATTERN) { // fast path
- stream = new FastStringTokenizer(text, true, toLowerCase, stopWords);
+ return new TokenStreamComponents(new FastStringTokenizer(text, true, toLowerCase, stopWords));
+ } else if (pattern == WHITESPACE_PATTERN) { // fast path
+ return new TokenStreamComponents(new FastStringTokenizer(text, false, toLowerCase, stopWords));
}
- else if (pattern == WHITESPACE_PATTERN) { // fast path
- stream = new FastStringTokenizer(text, false, toLowerCase, stopWords);
- }
- else {
- stream = new PatternTokenizer(text, pattern, toLowerCase);
- if (stopWords != null) stream = new StopFilter(matchVersion, stream, stopWords);
- }
-
- return stream;
+
+ Tokenizer tokenizer = new PatternTokenizer(text, pattern, toLowerCase);
+ TokenStream result = (stopWords != null) ? new StopFilter(matchVersion, tokenizer, stopWords) : tokenizer;
+ return new TokenStreamComponents(tokenizer, result);
}
/**
@@ -220,10 +217,10 @@ public final class PatternAnalyzer exten
* @return a new token stream
*/
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
try {
String text = toString(reader);
- return tokenStream(fieldName, text);
+ return createComponents(fieldName, text);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Mon Sep 12 05:50:26 2011
@@ -123,9 +123,9 @@ public final class ClassicAnalyzer exten
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected boolean reset(final Reader reader) throws IOException {
+ protected void reset(final Reader reader) throws IOException {
src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- return super.reset(reader);
+ super.reset(reader);
}
};
}
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Mon Sep 12 05:50:26 2011
@@ -124,9 +124,9 @@ public final class StandardAnalyzer exte
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected boolean reset(final Reader reader) throws IOException {
+ protected void reset(final Reader reader) throws IOException {
src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
- return super.reset(reader);
+ super.reset(reader);
}
};
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java Mon Sep 12 05:50:26 2011
@@ -21,9 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.util.Version;
@@ -66,10 +64,10 @@ public class TestChineseTokenizer extend
* Analyzer that just uses ChineseTokenizer, not ChineseFilter.
* convenience to show the behavior of the tokenizer
*/
- private class JustChineseTokenizerAnalyzer extends Analyzer {
+ private class JustChineseTokenizerAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new ChineseTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return new TokenStreamComponents(new ChineseTokenizer(reader));
}
}
@@ -77,10 +75,11 @@ public class TestChineseTokenizer extend
* Analyzer that just uses ChineseFilter, not ChineseTokenizer.
* convenience to show the behavior of the filter.
*/
- private class JustChineseFilterAnalyzer extends Analyzer {
+ private class JustChineseFilterAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new ChineseFilter(new WhitespaceTokenizer(Version.LUCENE_CURRENT, reader));
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_CURRENT, reader);
+ return new TokenStreamComponents(tokenizer, new ChineseFilter(tokenizer));
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java Mon Sep 12 05:50:26 2011
@@ -19,11 +19,8 @@ package org.apache.lucene.analysis.commo
import java.io.Reader;
import java.io.StringReader;
import java.util.Arrays;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -87,11 +84,12 @@ public class CommonGramsFilterTest exten
* @return Map<String,String>
*/
public void testCommonGramsQueryFilter() throws Exception {
- Analyzer a = new Analyzer() {
+ Analyzer a = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String field, Reader in) {
- return new CommonGramsQueryFilter(new CommonGramsFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(in, MockTokenizer.WHITESPACE, false), commonWords));
+ public TokenStreamComponents createComponents(String field, Reader in) {
+ Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new CommonGramsQueryFilter(new CommonGramsFilter(TEST_VERSION_CURRENT,
+ tokenizer, commonWords)));
}
};
@@ -156,11 +154,12 @@ public class CommonGramsFilterTest exten
}
public void testCommonGramsFilter() throws Exception {
- Analyzer a = new Analyzer() {
+ Analyzer a = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String field, Reader in) {
- return new CommonGramsFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(in, MockTokenizer.WHITESPACE, false), commonWords);
+ public TokenStreamComponents createComponents(String field, Reader in) {
+ Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new CommonGramsFilter(TEST_VERSION_CURRENT,
+ tokenizer, commonWords));
}
};
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Mon Sep 12 05:50:26 2011
@@ -21,10 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
@@ -120,12 +117,12 @@ public class TestAnalyzers extends BaseT
String[] y = StandardTokenizer.TOKEN_TYPES;
}
- private static class LowerCaseWhitespaceAnalyzer extends Analyzer {
+ private static class LowerCaseWhitespaceAnalyzer extends ReusableAnalyzerBase {
@Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseFilter(TEST_VERSION_CURRENT,
- new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
+ return new TokenStreamComponents(tokenizer, new LowerCaseFilter(TEST_VERSION_CURRENT, tokenizer));
}
}
@@ -237,4 +234,4 @@ final class PayloadSetter extends TokenF
data[0]++;
return true;
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/PatternAnalyzerTest.java Mon Sep 12 05:50:26 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.misce
*/
import java.io.IOException;
+import java.io.StringReader;
import java.util.Arrays;
import java.util.regex.Pattern;
@@ -128,7 +129,7 @@ public class PatternAnalyzerTest extends
assertTokenStreamContents(ts, expected);
// analysis of a String, uses PatternAnalyzer.tokenStream(String, String)
- TokenStream ts2 = analyzer.tokenStream("dummy", document);
+ TokenStream ts2 = analyzer.tokenStream("dummy", new StringReader(document));
assertTokenStreamContents(ts2, expected);
}
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java?rev=1169607&r1=1169606&r2=1169607&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java Mon Sep 12 05:50:26 2011
@@ -17,12 +17,7 @@
package org.apache.lucene.analysis.miscellaneous;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -245,13 +240,13 @@ public class TestWordDelimiterFilter ext
new int[] { 1, 1, 1 });
/* analyzer that will consume tokens with large position increments */
- Analyzer a2 = new Analyzer() {
+ Analyzer a2 = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String field, Reader reader) {
- return new WordDelimiterFilter(
- new LargePosIncTokenFilter(
- new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)),
- flags, protWords);
+ public TokenStreamComponents createComponents(String field, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(
+ new LargePosIncTokenFilter(tokenizer),
+ flags, protWords));
}
};
@@ -278,13 +273,14 @@ public class TestWordDelimiterFilter ext
new int[] { 6, 14, 19 },
new int[] { 1, 11, 1 });
- Analyzer a3 = new Analyzer() {
+ Analyzer a3 = new ReusableAnalyzerBase() {
@Override
- public TokenStream tokenStream(String field, Reader reader) {
+ public TokenStreamComponents createComponents(String field, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
StopFilter filter = new StopFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), StandardAnalyzer.STOP_WORDS_SET);
+ tokenizer, StandardAnalyzer.STOP_WORDS_SET);
filter.setEnablePositionIncrements(true);
- return new WordDelimiterFilter(filter, flags, protWords);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(filter, flags, protWords));
}
};