You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2012/11/21 08:22:19 UTC
svn commit: r1411997 - in /lucene/dev/branches/branch_4x: ./ dev-tools/
lucene/ lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/
lucene/...
Author: hossman
Date: Wed Nov 21 07:22:16 2012
New Revision: 1411997
URL: http://svn.apache.org/viewvc?rev=1411997&view=rev
Log:
LUCENE-4489: Added consumeAllTokens option to LimitTokenCountFilter (merge r1411996)
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/BUILD.txt (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/JRE_VERSION_MIGRATION.txt (props changed)
lucene/dev/branches/branch_4x/lucene/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/README.txt (props changed)
lucene/dev/branches/branch_4x/lucene/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyFilterFactory.java (props changed)
lucene/dev/branches/branch_4x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.cfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/index.40.optimized.nocfs.zip (props changed)
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/licenses/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/site/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/LICENSE.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/README.txt (props changed)
lucene/dev/branches/branch_4x/solr/SYSTEM_REQUIREMENTS.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpclient-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpcore-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-LICENSE-ASL.txt (props changed)
lucene/dev/branches/branch_4x/solr/licenses/httpmime-NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/scripts/ (props changed)
lucene/dev/branches/branch_4x/solr/site/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1411997&r1=1411996&r2=1411997&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Nov 21 07:22:16 2012
@@ -63,6 +63,9 @@ New Features
you only need a few bits per-document, or where you might want exact
document length, and so on. (Robert Muir)
+* LUCENE-4489: Added consumeAllTokens option to LimitTokenCountFilter
+ (hossman, Robert Muir)
+
API Changes
* LUCENE-4399: Deprecated AppendingCodec. Lucene's term dictionaries
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java?rev=1411997&r1=1411996&r2=1411997&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java Wed Nov 21 07:22:16 2012
@@ -23,17 +23,32 @@ import org.apache.lucene.analysis.Analyz
/**
* This Analyzer limits the number of tokens while indexing. It is
* a replacement for the maximum field length setting inside {@link org.apache.lucene.index.IndexWriter}.
+ * @see LimitTokenCountFilter
*/
public final class LimitTokenCountAnalyzer extends AnalyzerWrapper {
private final Analyzer delegate;
private final int maxTokenCount;
+ private final boolean consumeAllTokens;
/**
* Build an analyzer that limits the maximum number of tokens per field.
+ * This analyzer will not consume any tokens beyond the maxTokenCount limit
+ *
+ * @see #LimitTokenCountAnalyzer(Analyzer,int,boolean)
*/
public LimitTokenCountAnalyzer(Analyzer delegate, int maxTokenCount) {
+ this(delegate, maxTokenCount, false);
+ }
+ /**
+ * Build an analyzer that limits the maximum number of tokens per field.
+ * @param delegate the analyzer to wrap
+ * @param maxTokenCount max number of tokens to produce
+ * @param consumeAllTokens whether all tokens from the delegate should be consumed even if maxTokenCount is reached.
+ */
+ public LimitTokenCountAnalyzer(Analyzer delegate, int maxTokenCount, boolean consumeAllTokens) {
this.delegate = delegate;
this.maxTokenCount = maxTokenCount;
+ this.consumeAllTokens = consumeAllTokens;
}
@Override
@@ -44,11 +59,11 @@ public final class LimitTokenCountAnalyz
@Override
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
return new TokenStreamComponents(components.getTokenizer(),
- new LimitTokenCountFilter(components.getTokenStream(), maxTokenCount));
+ new LimitTokenCountFilter(components.getTokenStream(), maxTokenCount, consumeAllTokens));
}
@Override
public String toString() {
- return "LimitTokenCountAnalyzer(" + delegate.toString() + ", maxTokenCount=" + maxTokenCount + ")";
+ return "LimitTokenCountAnalyzer(" + delegate.toString() + ", maxTokenCount=" + maxTokenCount + ", consumeAllTokens=" + consumeAllTokens + ")";
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java?rev=1411997&r1=1411996&r2=1411997&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilter.java Wed Nov 21 07:22:16 2012
@@ -25,32 +25,68 @@ import java.io.IOException;
/**
* This TokenFilter limits the number of tokens while indexing. It is
* a replacement for the maximum field length setting inside {@link org.apache.lucene.index.IndexWriter}.
+ * <p>
+ * By default, this filter ignores any tokens in the wrapped {@code TokenStream}
+ * once the limit has been reached, which can result in {@code reset()} being
+ * called prior to {@code incrementToken()} returning {@code false}. For most
+ * {@code TokenStream} implementations this should be acceptable, and faster
+ * then consuming the full stream. If you are wrapping a {@code TokenStream}
+ * which requires that the full stream of tokens be exhausted in order to
+ * function properly, use the
+ * {@link #LimitTokenCountFilter(TokenStream,int,boolean) consumeAllTokens}
+ * option.
*/
public final class LimitTokenCountFilter extends TokenFilter {
private final int maxTokenCount;
+ private final boolean consumeAllTokens;
private int tokenCount = 0;
+ private boolean exhausted = false;
/**
* Build a filter that only accepts tokens up to a maximum number.
+ * This filter will not consume any tokens beyond the maxTokenCount limit
+ *
+ * @see #LimitTokenCountFilter(TokenStream,int,boolean)
*/
public LimitTokenCountFilter(TokenStream in, int maxTokenCount) {
+ this(in, maxTokenCount, false);
+ }
+
+ /**
+ * Build an filter that limits the maximum number of tokens per field.
+ * @param in the stream to wrap
+ * @param maxTokenCount max number of tokens to produce
+ * @param consumeAllTokens whether all tokens from the input must be consumed even if maxTokenCount is reached.
+ */
+ public LimitTokenCountFilter(TokenStream in, int maxTokenCount, boolean consumeAllTokens) {
super(in);
this.maxTokenCount = maxTokenCount;
+ this.consumeAllTokens = consumeAllTokens;
}
@Override
public boolean incrementToken() throws IOException {
- if (tokenCount < maxTokenCount && input.incrementToken()) {
- tokenCount++;
- return true;
+ if (exhausted) {
+ return false;
+ } else if (tokenCount < maxTokenCount) {
+ if (input.incrementToken()) {
+ tokenCount++;
+ return true;
+ } else {
+ exhausted = true;
+ return false;
+ }
+ } else {
+ while (consumeAllTokens && input.incrementToken()) { /* NOOP */ }
+ return false;
}
- return false;
}
@Override
public void reset() throws IOException {
super.reset();
tokenCount = 0;
+ exhausted = false;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java?rev=1411997&r1=1411996&r2=1411997&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountFilterFactory.java Wed Nov 21 07:22:16 2012
@@ -29,25 +29,29 @@ import org.apache.lucene.analysis.util.T
* <fieldType name="text_lngthcnt" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10"/>
+ * <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10" consumeAllTokens="false" />
* </analyzer>
* </fieldType></pre>
- *
+ * <p>
+ * The {@code consumeAllTokens} property is optional and defaults to {@code false}. See {@link LimitTokenCountFilter} for an explanation of it's use.
*/
public class LimitTokenCountFilterFactory extends TokenFilterFactory {
public static final String MAX_TOKEN_COUNT_KEY = "maxTokenCount";
+ public static final String CONSUME_ALL_TOKENS_KEY = "consumeAllTokens";
int maxTokenCount;
+ boolean consumeAllTokens;
@Override
public void init(Map<String, String> args) {
super.init( args );
maxTokenCount = getInt(MAX_TOKEN_COUNT_KEY);
+ consumeAllTokens = getBoolean(CONSUME_ALL_TOKENS_KEY, false);
}
@Override
public TokenStream create(TokenStream input) {
- return new LimitTokenCountFilter(input, maxTokenCount);
+ return new LimitTokenCountFilter(input, maxTokenCount, consumeAllTokens);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1411997&r1=1411996&r2=1411997&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Wed Nov 21 07:22:16 2012
@@ -33,40 +33,64 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util._TestUtil;
public class TestLimitTokenCountAnalyzer extends BaseTokenStreamTestCase {
public void testLimitTokenCountAnalyzer() throws IOException {
- Analyzer a = new LimitTokenCountAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, 4);
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
+ for (boolean consumeAll : new boolean[] { true, false }) {
+ MockAnalyzer mock = new MockAnalyzer(random());
+
+ // if we are consuming all tokens, we can use the checks,
+ // otherwise we can't
+ mock.setEnableChecks(consumeAll);
+ Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
+
+ // dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
+
+ // less than the limit, ensure we behave correctly
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
- a = new LimitTokenCountAnalyzer(new StandardAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
+ // equal to limit
+ assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
+ }
}
public void testLimitTokenCountIndexWriter() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new LimitTokenCountAnalyzer(new MockAnalyzer(random()), 100000)));
-
- Document doc = new Document();
- StringBuilder b = new StringBuilder();
- for(int i=0;i<10000;i++)
- b.append(" a");
- b.append(" x");
- doc.add(newTextField("field", b.toString(), Field.Store.NO));
- writer.addDocument(doc);
- writer.close();
-
- IndexReader reader = DirectoryReader.open(dir);
- Term t = new Term("field", "x");
- assertEquals(1, reader.docFreq(t));
- reader.close();
- dir.close();
+
+ for (boolean consumeAll : new boolean[] { true, false }) {
+ Directory dir = newDirectory();
+ int limit = _TestUtil.nextInt(random(), 50, 101000);
+ MockAnalyzer mock = new MockAnalyzer(random());
+
+ // if we are consuming all tokens, we can use the checks,
+ // otherwise we can't
+ mock.setEnableChecks(consumeAll);
+ Analyzer a = new LimitTokenCountAnalyzer(mock, limit, consumeAll);
+
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig
+ (TEST_VERSION_CURRENT, a));
+
+ Document doc = new Document();
+ StringBuilder b = new StringBuilder();
+ for(int i=1;i<limit;i++)
+ b.append(" a");
+ b.append(" x");
+ b.append(" z");
+ doc.add(newTextField("field", b.toString(), Field.Store.NO));
+ writer.addDocument(doc);
+ writer.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+ Term t = new Term("field", "x");
+ assertEquals(1, reader.docFreq(t));
+ t = new Term("field", "z");
+ assertEquals(0, reader.docFreq(t));
+ reader.close();
+ dir.close();
+ }
}
-}
\ No newline at end of file
+}