You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/05/16 17:11:08 UTC
svn commit: r1483403 - in /lucene/dev/trunk/lucene: ./
analysis/common/src/java/org/apache/lucene/analysis/position/
analysis/common/src/test/org/apache/lucene/analysis/core/
analysis/common/src/test/org/apache/lucene/analysis/ngram/
Author: jpountz
Date: Thu May 16 15:11:08 2013
New Revision: 1483403
URL: http://svn.apache.org/r1483403
Log:
LUCENE-4981: Deprecate PositionFilter.
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilterFactory.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1483403&r1=1483402&r2=1483403&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu May 16 15:11:08 2013
@@ -76,6 +76,12 @@ Changes in backwards compatibility polic
* LUCENE-3907: EdgeNGramTokenFilter does not support backward grams and does
not update offsets anymore. (Adrien Grand)
+* LUCENE-4981: PositionFilter is now deprecated as it can corrupt token stream
+ graphs. Since it main use-case was to make query parsers generate boolean
+ queries instead of phrase queries, it is now advised to use
+ QueryParser.setAutoGeneratePhraseQueries(false) (for simple cases) or to
+ override QueryParser.newFieldQuery. (Adrien Grand, Steve Rowe)
+
Bug Fixes
* LUCENE-4997: Internal test framework's tests are sensitive to previous
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=1483403&r1=1483402&r2=1483403&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Thu May 16 15:11:08 2013
@@ -22,11 +22,19 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.queryparser.classic.QueryParser;
/** Set the positionIncrement of all tokens to the "positionIncrement",
* except the first return token which retains its original positionIncrement value.
* The default positionIncrement value is zero.
+ * @deprecated (4.4) PositionFilter makes {@link TokenStream} graphs inconsistent
+ * which can cause highlighting bugs. Its main use-case being to make
+ * {@link QueryParser} generate boolean queries instead of phrase
+ * queries, it is now advised to use
+ * {@link QueryParser#setAutoGeneratePhraseQueries(boolean) QueryParser.setAutoGeneratePhraseQueries(false)}
+ * (for simple cases) or to override {@link QueryParser#newFieldQuery}.
*/
+@Deprecated
public final class PositionFilter extends TokenFilter {
/** Position increment to assign to all but the first token - default = 0 */
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilterFactory.java?rev=1483403&r1=1483402&r2=1483403&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/position/PositionFilterFactory.java Thu May 16 15:11:08 2013
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.posit
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.position.PositionFilter;
import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.util.Version;
import java.util.Map;
@@ -37,7 +38,9 @@ import java.util.Map;
*
* @see org.apache.lucene.analysis.position.PositionFilter
* @since solr 1.4
+ * @deprecated (4.4)
*/
+@Deprecated
public class PositionFilterFactory extends TokenFilterFactory {
private final int positionIncrement;
@@ -48,6 +51,9 @@ public class PositionFilterFactory exten
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
+ if (luceneMatchVersion != null && luceneMatchVersion.onOrAfter(Version.LUCENE_44)) {
+ throw new IllegalArgumentException("PositionFilter is deprecated as of Lucene 4.4. You should either fix your code to not use it or use Lucene 4.3 version compatibility");
+ }
}
@Override
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1483403&r1=1483402&r2=1483403&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Thu May 16 15:11:08 2013
@@ -174,8 +174,6 @@ public class TestRandomChains extends Ba
PathHierarchyTokenizer.class,
HyphenationCompoundWordTokenFilter.class,
DictionaryCompoundWordTokenFilter.class,
- // TODO: corrumpts graphs (offset consistency check):
- PositionFilter.class,
// TODO: it seems to mess up offsets!?
WikipediaTokenizer.class,
// TODO: doesn't handle graph inputs
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1483403&r1=1483402&r2=1483403&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Thu May 16 15:11:08 2013
@@ -19,13 +19,16 @@ package org.apache.lucene.analysis.ngram
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.position.PositionFilter;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Random;
@@ -101,9 +104,39 @@ public class EdgeNGramTokenFilterTest ex
false);
}
+ private static class PositionFilter extends TokenFilter {
+
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private boolean started;
+
+ PositionFilter(final TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ if (started) {
+ posIncrAtt.setPositionIncrement(0);
+ } else {
+ started = true;
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ started = false;
+ }
+ }
+
public void testFirstTokenPositionIncrement() throws Exception {
TokenStream ts = new MockTokenizer(new StringReader("a abc"), MockTokenizer.WHITESPACE, false);
- ts = new PositionFilter(ts, 0); // All but first token will get 0 position increment
+ ts = new PositionFilter(ts); // All but first token will get 0 position increment
EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, ts, 2, 3);
// The first token "a" will not be output, since it's smaller than the mingram size of 2.
// The second token on input to EdgeNGramTokenFilter will have position increment of 0,