You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2014/02/12 11:47:49 UTC
svn commit: r1567597 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/
Author: simonw
Date: Wed Feb 12 10:47:49 2014
New Revision: 1567597
URL: http://svn.apache.org/r1567597
Log:
LUCENE-5437: ASCIIFoldingFilter now has an option to preserve the original token
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1567597&r1=1567596&r2=1567597&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Wed Feb 12 10:47:49 2014
@@ -91,6 +91,10 @@ New Features
* LUCENE-5440: Add LongBitSet for managing more than 2.1B bits (otherwise use
FixedBitSet). (Shai Erera)
+* LUCENE-5437: ASCIIFoldingFilter now has an option to preserve the original token
+ and emit it on the same position as the folded token only if the actual token was
+ folded. (Simon Willnauer, Nik Everett)
+
Build
* LUCENE-5217,LUCENE-5420: Maven config: get dependencies from Ant+Ivy config;
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java?rev=1567597&r1=1567596&r2=1567597&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilter.java Wed Feb 12 10:47:49 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -57,17 +58,49 @@ import org.apache.lucene.util.RamUsageEs
* For example, 'à' will be replaced by 'a'.
*/
public final class ASCIIFoldingFilter extends TokenFilter {
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncAttr = addAttribute(PositionIncrementAttribute.class);
+ private final boolean preserveOriginal;
+ private char[] output = new char[512];
+ private int outputPos;
+ private State state;
+
public ASCIIFoldingFilter(TokenStream input)
{
+ this(input, false);
+ }
+
+ /**
+ * Create a new {@link ASCIIFoldingFilter}.
+ *
+ * @param input
+ * TokenStream to filter
+ * @param preserveOriginal
+ * should the original tokens be kept on the input stream with a 0 position increment
+ * from the folded tokens?
+ **/
+ public ASCIIFoldingFilter(TokenStream input, boolean preserveOriginal)
+ {
super(input);
+ this.preserveOriginal = preserveOriginal;
}
- private char[] output = new char[512];
- private int outputPos;
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ /**
+ * Does the filter preserve the original tokens?
+ */
+ public boolean isPreserveOriginal() {
+ return preserveOriginal;
+ }
@Override
public boolean incrementToken() throws IOException {
+ if (state != null) {
+ assert preserveOriginal : "state should only be captured if preserveOriginal is true";
+ restoreState(state);
+ posIncAttr.setPositionIncrement(0);
+ state = null;
+ return true;
+ }
if (input.incrementToken()) {
final char[] buffer = termAtt.buffer();
final int length = termAtt.length();
@@ -89,6 +122,12 @@ public final class ASCIIFoldingFilter ex
}
}
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ state = null;
+ }
+
/**
* Converts characters above ASCII to their ASCII equivalents. For example,
* accents are removed from accented characters.
@@ -97,6 +136,9 @@ public final class ASCIIFoldingFilter ex
*/
public void foldToASCII(char[] input, int length)
{
+ if (preserveOriginal) {
+ state = captureState();
+ }
// Worst-case length required:
final int maxSizeNeeded = 4 * length;
if (output.length < maxSizeNeeded) {
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java?rev=1567597&r1=1567596&r2=1567597&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java Wed Feb 12 10:47:49 2014
@@ -31,15 +31,17 @@ import org.apache.lucene.analysis.TokenS
* <fieldType name="text_ascii" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.ASCIIFoldingFilterFactory"/>
+ * <filter class="solr.ASCIIFoldingFilterFactory" preserveOriginal="false"/>
* </analyzer>
* </fieldType></pre>
*/
public class ASCIIFoldingFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+ private final boolean preserveOriginal;
/** Creates a new ASCIIFoldingFilterFactory */
public ASCIIFoldingFilterFactory(Map<String,String> args) {
super(args);
+ preserveOriginal = getBoolean(args, "preserveOriginal", false);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -47,7 +49,7 @@ public class ASCIIFoldingFilterFactory e
@Override
public ASCIIFoldingFilter create(TokenStream input) {
- return new ASCIIFoldingFilter(input);
+ return new ASCIIFoldingFilter(input, preserveOriginal);
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java?rev=1567597&r1=1567596&r2=1567597&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java Wed Feb 12 10:47:49 2014
@@ -33,6 +33,18 @@ import java.util.ArrayList;
import java.util.Iterator;
public class TestASCIIFoldingFilter extends BaseTokenStreamTestCase {
+ /**
+ * Pop one input token's worth of tokens off the filter and verify that they are as expected.
+ */
+ void assertNextTerms(String expectedUnfolded, String expectedFolded, ASCIIFoldingFilter filter,
+ CharTermAttribute termAtt) throws Exception {
+ assertTrue(filter.incrementToken());
+ assertEquals(expectedFolded, termAtt.toString());
+ if (filter.isPreserveOriginal() && !expectedUnfolded.equals(expectedFolded)) {
+ assertTrue(filter.incrementToken());
+ assertEquals(expectedUnfolded, termAtt.toString());
+ }
+ }
// testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
public void testLatin1Accents() throws Exception {
@@ -40,85 +52,85 @@ public class TestASCIIFoldingFilter exte
("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à Ã"
+" à à à à à à Šà à à à à à Ÿ à á â ã ä å æ ç è é ê ë ì à î ï ij"
+" ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"), MockTokenizer.WHITESPACE, false);
- ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
+ ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream, random().nextBoolean());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
filter.reset();
- assertTermEquals("Des", filter, termAtt);
- assertTermEquals("mot", filter, termAtt);
- assertTermEquals("cles", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("LA", filter, termAtt);
- assertTermEquals("CHAINE", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("AE", filter, termAtt);
- assertTermEquals("C", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("IJ", filter, termAtt);
- assertTermEquals("D", filter, termAtt);
- assertTermEquals("N", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("OE", filter, termAtt);
- assertTermEquals("TH", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("Y", filter, termAtt);
- assertTermEquals("Y", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("ae", filter, termAtt);
- assertTermEquals("c", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("ij", filter, termAtt);
- assertTermEquals("d", filter, termAtt);
- assertTermEquals("n", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("oe", filter, termAtt);
- assertTermEquals("ss", filter, termAtt);
- assertTermEquals("th", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("y", filter, termAtt);
- assertTermEquals("y", filter, termAtt);
- assertTermEquals("fi", filter, termAtt);
- assertTermEquals("fl", filter, termAtt);
+ assertNextTerms("Des", "Des", filter, termAtt);
+ assertNextTerms("mot", "mot", filter, termAtt);
+ assertNextTerms("clés", "cles", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("LA", "LA", filter, termAtt);
+ assertNextTerms("CHAÃNE", "CHAINE", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("Ã", "A", filter, termAtt);
+ assertNextTerms("Ã
", "A", filter, termAtt);
+ assertNextTerms("Ã", "AE", filter, termAtt);
+ assertNextTerms("Ã", "C", filter, termAtt);
+ assertNextTerms("Ã", "E", filter, termAtt);
+ assertNextTerms("Ã", "E", filter, termAtt);
+ assertNextTerms("Ã", "E", filter, termAtt);
+ assertNextTerms("Ã", "E", filter, termAtt);
+ assertNextTerms("Ã", "I", filter, termAtt);
+ assertNextTerms("Ã", "I", filter, termAtt);
+ assertNextTerms("Ã", "I", filter, termAtt);
+ assertNextTerms("Ã", "I", filter, termAtt);
+ assertNextTerms("IJ", "IJ", filter, termAtt);
+ assertNextTerms("Ã", "D", filter, termAtt);
+ assertNextTerms("Ã", "N", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Ã", "O", filter, termAtt);
+ assertNextTerms("Å", "OE", filter, termAtt);
+ assertNextTerms("Ã", "TH", filter, termAtt);
+ assertNextTerms("Ã", "U", filter, termAtt);
+ assertNextTerms("Ã", "U", filter, termAtt);
+ assertNextTerms("Ã", "U", filter, termAtt);
+ assertNextTerms("Ã", "U", filter, termAtt);
+ assertNextTerms("Ã", "Y", filter, termAtt);
+ assertNextTerms("Ÿ", "Y", filter, termAtt);
+ assertNextTerms("Ã ", "a", filter, termAtt);
+ assertNextTerms("á", "a", filter, termAtt);
+ assertNextTerms("â", "a", filter, termAtt);
+ assertNextTerms("ã", "a", filter, termAtt);
+ assertNextTerms("ä", "a", filter, termAtt);
+ assertNextTerms("Ã¥", "a", filter, termAtt);
+ assertNextTerms("æ", "ae", filter, termAtt);
+ assertNextTerms("ç", "c", filter, termAtt);
+ assertNextTerms("è", "e", filter, termAtt);
+ assertNextTerms("é", "e", filter, termAtt);
+ assertNextTerms("ê", "e", filter, termAtt);
+ assertNextTerms("ë", "e", filter, termAtt);
+ assertNextTerms("ì", "i", filter, termAtt);
+ assertNextTerms("Ã", "i", filter, termAtt);
+ assertNextTerms("î", "i", filter, termAtt);
+ assertNextTerms("ï", "i", filter, termAtt);
+ assertNextTerms("ij", "ij", filter, termAtt);
+ assertNextTerms("ð", "d", filter, termAtt);
+ assertNextTerms("ñ", "n", filter, termAtt);
+ assertNextTerms("ò", "o", filter, termAtt);
+ assertNextTerms("ó", "o", filter, termAtt);
+ assertNextTerms("ô", "o", filter, termAtt);
+ assertNextTerms("õ", "o", filter, termAtt);
+ assertNextTerms("ö", "o", filter, termAtt);
+ assertNextTerms("ø", "o", filter, termAtt);
+ assertNextTerms("Å", "oe", filter, termAtt);
+ assertNextTerms("Ã", "ss", filter, termAtt);
+ assertNextTerms("þ", "th", filter, termAtt);
+ assertNextTerms("ù", "u", filter, termAtt);
+ assertNextTerms("ú", "u", filter, termAtt);
+ assertNextTerms("û", "u", filter, termAtt);
+ assertNextTerms("ü", "u", filter, termAtt);
+ assertNextTerms("ý", "y", filter, termAtt);
+ assertNextTerms("ÿ", "y", filter, termAtt);
+ assertNextTerms("ï¬", "fi", filter, termAtt);
+ assertNextTerms("ï¬", "fl", filter, termAtt);
assertFalse(filter.incrementToken());
}
@@ -1879,7 +1891,8 @@ public class TestASCIIFoldingFilter exte
};
// Construct input text and expected output tokens
- List<String> expectedOutputTokens = new ArrayList<String>();
+ List<String> expectedUnfoldedTokens = new ArrayList<String>();
+ List<String> expectedFoldedTokens = new ArrayList<String>();
StringBuilder inputText = new StringBuilder();
for (int n = 0 ; n < foldings.length ; n += 2) {
if (n > 0) {
@@ -1887,32 +1900,30 @@ public class TestASCIIFoldingFilter exte
}
inputText.append(foldings[n]);
- // Construct the expected output token: the ASCII string to fold to,
- // duplicated as many times as the number of characters in the input text.
+ // Construct the expected output tokens: both the unfolded and folded string,
+ // with the folded duplicated as many times as the number of characters in
+ // the input text.
StringBuilder expected = new StringBuilder();
int numChars = foldings[n].length();
for (int m = 0 ; m < numChars; ++m) {
expected.append(foldings[n + 1]);
}
- expectedOutputTokens.add(expected.toString());
+ expectedUnfoldedTokens.add(foldings[n]);
+ expectedFoldedTokens.add(expected.toString());
}
TokenStream stream = new MockTokenizer(new StringReader(inputText.toString()), MockTokenizer.WHITESPACE, false);
- ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
+ ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream, random().nextBoolean());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
- Iterator<String> expectedIter = expectedOutputTokens.iterator();
+ Iterator<String> unfoldedIter = expectedUnfoldedTokens.iterator();
+ Iterator<String> foldedIter = expectedFoldedTokens.iterator();
filter.reset();
- while (expectedIter.hasNext()) {
- assertTermEquals(expectedIter.next(), filter, termAtt);
+ while (foldedIter.hasNext()) {
+ assertNextTerms(unfoldedIter.next(), foldedIter.next(), filter, termAtt);
}
assertFalse(filter.incrementToken());
}
- void assertTermEquals(String expected, TokenStream stream, CharTermAttribute termAtt) throws Exception {
- assertTrue(stream.incrementToken());
- assertEquals(expected, termAtt.toString());
- }
-
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@@ -1920,7 +1931,8 @@ public class TestASCIIFoldingFilter exte
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
+ return new TokenStreamComponents(tokenizer,
+ new ASCIIFoldingFilter(tokenizer, random().nextBoolean()));
}
};
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
@@ -1931,7 +1943,8 @@ public class TestASCIIFoldingFilter exte
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
- return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
+ return new TokenStreamComponents(tokenizer,
+ new ASCIIFoldingFilter(tokenizer, random().nextBoolean()));
}
};
checkOneTerm(a, "", "");