You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/12/12 19:59:11 UTC
svn commit: r1213366 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/
lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/
lucene/contrib/analyzers/smartcn/...
Author: rmuir
Date: Mon Dec 12 18:59:10 2011
New Revision: 1213366
URL: http://svn.apache.org/viewvc?rev=1213366&view=rev
Log:
LUCENE-3642: fix invalid offsets from CharTokenizer, [Edge]NGramFilters, SmartChinese, add sanity check to BaseTokenStreamTestCase
Added:
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestDuelingAnalyzers.java
- copied, changed from r1213329, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java (contents, props changed)
lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharTokenizers.java (contents, props changed)
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Mon Dec 12 18:59:10 2011
@@ -73,6 +73,10 @@ Bug fixes
* LUCENE-3641: Fixed MultiReader to correctly propagate readerFinishedListeners
to clones/reopened readers. (Uwe Schindler)
+* LUCENE-3642: Fixed bugs in CharTokenizer, n-gram filters, and smart chinese
+ where they would create invalid offsets in some situations, leading to problems
+ in highlighting. (Max Beutel via Robert Muir)
+
Documentation
* LUCENE-3597: Fixed incorrect grouping documentation. (Martijn van Groningen, Robert Muir)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Mon Dec 12 18:59:10 2011
@@ -71,6 +71,8 @@ public final class EdgeNGramTokenFilter
private int curTermLength;
private int curGramSize;
private int tokStart;
+ private int tokEnd; // only used if the length changed before this filter
+ private boolean hasIllegalOffsets; // only if the length changed before this filter
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -126,6 +128,10 @@ public final class EdgeNGramTokenFilter
curTermLength = termAtt.length();
curGramSize = minGram;
tokStart = offsetAtt.startOffset();
+ tokEnd = offsetAtt.endOffset();
+ // if length by start + end offsets doesn't match the term text then assume
+ // this is a synonym and don't adjust the offsets.
+ hasIllegalOffsets = (tokStart + curTermLength) != tokEnd;
}
}
if (curGramSize <= maxGram) {
@@ -135,7 +141,11 @@ public final class EdgeNGramTokenFilter
int start = side == Side.FRONT ? 0 : curTermLength - curGramSize;
int end = start + curGramSize;
clearAttributes();
- offsetAtt.setOffset(tokStart + start, tokStart + end);
+ if (hasIllegalOffsets) {
+ offsetAtt.setOffset(tokStart, tokEnd);
+ } else {
+ offsetAtt.setOffset(tokStart + start, tokStart + end);
+ }
termAtt.copyBuffer(curTermBuffer, start, curGramSize);
curGramSize++;
return true;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Mon Dec 12 18:59:10 2011
@@ -38,6 +38,8 @@ public final class NGramTokenFilter exte
private int curGramSize;
private int curPos;
private int tokStart;
+ private int tokEnd; // only used if the length changed before this filter
+ private boolean hasIllegalOffsets; // only if the length changed before this filter
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
@@ -81,13 +83,21 @@ public final class NGramTokenFilter exte
curGramSize = minGram;
curPos = 0;
tokStart = offsetAtt.startOffset();
+ tokEnd = offsetAtt.endOffset();
+ // if length by start + end offsets doesn't match the term text then assume
+ // this is a synonym and don't adjust the offsets.
+ hasIllegalOffsets = (tokStart + curTermLength) != tokEnd;
}
}
while (curGramSize <= maxGram) {
while (curPos+curGramSize <= curTermLength) { // while there is input
clearAttributes();
termAtt.copyBuffer(curTermBuffer, curPos, curGramSize);
- offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
+ if (hasIllegalOffsets) {
+ offsetAtt.setOffset(tokStart, tokEnd);
+ } else {
+ offsetAtt.setOffset(tokStart + curPos, tokStart + curPos + curGramSize);
+ }
curPos++;
return true;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Mon Dec 12 18:59:10 2011
@@ -17,11 +17,17 @@ package org.apache.lucene.analysis.ngram
* limitations under the License.
*/
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
+import java.io.Reader;
import java.io.StringReader;
/**
@@ -104,4 +110,24 @@ public class EdgeNGramTokenFilterTest ex
tokenizer.reset(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
}
+
+ // LUCENE-3642
+ // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
+ // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
+ // so in this case we behave like WDF, and preserve any modified offsets
+ public void testInvalidOffsets() throws Exception {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new EdgeNGramTokenFilter(filters, EdgeNGramTokenFilter.Side.FRONT, 2, 15);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ };
+ assertAnalyzesTo(analyzer, "mosfellsbær",
+ new String[] { "mo", "mos", "mosf", "mosfe", "mosfel", "mosfell", "mosfells", "mosfellsb", "mosfellsba", "mosfellsbae", "mosfellsbaer" },
+ new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 });
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Mon Dec 12 18:59:10 2011
@@ -17,11 +17,17 @@ package org.apache.lucene.analysis.ngram
* limitations under the License.
*/
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
+import java.io.Reader;
import java.io.StringReader;
/**
@@ -93,4 +99,24 @@ public class NGramTokenFilterTest extend
tokenizer.reset(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
}
+
+ // LUCENE-3642
+ // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
+ // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
+ // so in this case we behave like WDF, and preserve any modified offsets
+ public void testInvalidOffsets() throws Exception {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new NGramTokenFilter(filters, 2, 2);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ };
+ assertAnalyzesTo(analyzer, "mosfellsbær",
+ new String[] { "mo", "os", "sf", "fe", "el", "ll", "ls", "sb", "ba", "ae", "er" },
+ new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
+ new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 });
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java Mon Dec 12 18:59:10 2011
@@ -43,6 +43,10 @@ public final class WordTokenFilter exten
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+
+ private int tokStart; // only used if the length changed before this filter
+ private int tokEnd; // only used if the length changed before this filter
+ private boolean hasIllegalOffsets; // only if the length changed before this filter
/**
* Construct a new WordTokenizer.
@@ -59,6 +63,11 @@ public final class WordTokenFilter exten
if (tokenIter == null || !tokenIter.hasNext()) {
// there are no remaining tokens from the current sentence... are there more sentences?
if (input.incrementToken()) {
+ tokStart = offsetAtt.startOffset();
+ tokEnd = offsetAtt.endOffset();
+ // if length by start + end offsets doesn't match the term text then assume
+ // this is a synonym and don't adjust the offsets.
+ hasIllegalOffsets = (tokStart + termAtt.length()) != tokEnd;
// a new sentence is available: process it.
tokenBuffer = wordSegmenter.segmentSentence(termAtt.toString(), offsetAtt.startOffset());
tokenIter = tokenBuffer.iterator();
@@ -77,7 +86,11 @@ public final class WordTokenFilter exten
// There are remaining tokens from the current sentence, return the next one.
SegToken nextWord = tokenIter.next();
termAtt.copyBuffer(nextWord.charArray, 0, nextWord.charArray.length);
- offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
+ if (hasIllegalOffsets) {
+ offsetAtt.setOffset(tokStart, tokEnd);
+ } else {
+ offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
+ }
typeAtt.setType("word");
return true;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java Mon Dec 12 18:59:10 2011
@@ -17,11 +17,17 @@
package org.apache.lucene.analysis.cn.smart;
+import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ASCIIFoldingFilter;
import org.apache.lucene.util.Version;
public class TestSmartChineseAnalyzer extends BaseTokenStreamTestCase {
@@ -196,6 +202,24 @@ public class TestSmartChineseAnalyzer ex
}
}
+ // LUCENE-3642
+ public void testInvalidOffset() throws Exception {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new WordTokenFilter(filters);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ };
+
+ assertAnalyzesTo(analyzer, "mosfellsbær",
+ new String[] { "mosfellsbaer" },
+ new int[] { 0 },
+ new int[] { 11 });
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random, new SmartChineseAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java Mon Dec 12 18:59:10 2011
@@ -266,6 +266,7 @@ public abstract class CharTokenizer exte
return incrementTokenOld();
int length = 0;
int start = -1; // this variable is always initialized
+ int end = -1;
char[] buffer = termAtt.buffer();
while (true) {
if (bufferIndex >= dataLen) {
@@ -284,15 +285,18 @@ public abstract class CharTokenizer exte
}
// use CharacterUtils here to support < 3.1 UTF-16 code unit behavior if the char based methods are gone
final int c = charUtils.codePointAt(ioBuffer.getBuffer(), bufferIndex);
- bufferIndex += Character.charCount(c);
+ final int charCount = Character.charCount(c);
+ bufferIndex += charCount;
if (isTokenChar(c)) { // if it's a token char
if (length == 0) { // start of token
assert start == -1;
- start = offset + bufferIndex - 1;
+ start = offset + bufferIndex - charCount;
+ end = start;
} else if (length >= buffer.length-1) { // check if a supplementary could run out of bounds
buffer = termAtt.resizeBuffer(2+length); // make sure a supplementary fits in the buffer
}
+ end += charCount;
length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
if (length >= MAX_WORD_LEN) // buffer overflow! make sure to check for >= surrogate pair could break == test
break;
@@ -302,7 +306,7 @@ public abstract class CharTokenizer exte
termAtt.setLength(length);
assert start != -1;
- offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(start+length));
+ offsetAtt.setOffset(correctOffset(start), finalOffset = correctOffset(end));
return true;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Mon Dec 12 18:59:10 2011
@@ -135,6 +135,10 @@ public abstract class BaseTokenStreamTes
assertTrue("startOffset must be >= 0", offsetAtt.startOffset() >= 0);
assertTrue("endOffset must be >= 0", offsetAtt.endOffset() >= 0);
assertTrue("endOffset must be >= startOffset", offsetAtt.endOffset() >= offsetAtt.startOffset());
+ if (finalOffset != null) {
+ assertTrue("startOffset must be <= finalOffset", offsetAtt.startOffset() <= finalOffset.intValue());
+ assertTrue("endOffset must be <= finalOffset", offsetAtt.endOffset() <= finalOffset.intValue());
+ }
}
if (posIncrAtt != null) {
assertTrue("posIncrement must be >= 0", posIncrAtt.getPositionIncrement() >= 0);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharTokenizers.java?rev=1213366&r1=1213365&r2=1213366&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharTokenizers.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestCharTokenizers.java Mon Dec 12 18:59:10 2011
@@ -22,6 +22,12 @@ import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.util.Version;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util._TestUtil;
/**
* Testcase for {@link CharTokenizer} subclasses
@@ -219,4 +225,80 @@ public class TestCharTokenizers extends
return Character.isLetter(c);
}
}
+
+ // LUCENE-3642: normalize SMP->BMP and check that offsets are correct
+ public void testCrossPlaneNormalization() throws IOException {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ @Override
+ protected int normalize(int c) {
+ if (c > 0xffff) {
+ return 'δ';
+ } else {
+ return c;
+ }
+ }
+ };
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ int num = 10000 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) {
+ String s = _TestUtil.randomUnicodeString(random);
+ TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ ts.reset();
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+ while (ts.incrementToken()) {
+ String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
+ for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
+ cp = highlightedText.codePointAt(j);
+ assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ }
+ }
+ ts.end();
+ ts.close();
+ }
+ // just for fun
+ checkRandomData(random, analyzer, num);
+ }
+
+ // LUCENE-3642: normalize BMP->SMP and check that offsets are correct
+ public void testCrossPlaneNormalization2() throws IOException {
+ Analyzer analyzer = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ @Override
+ protected int normalize(int c) {
+ if (c <= 0xffff) {
+ return 0x1043C;
+ } else {
+ return c;
+ }
+ }
+ };
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ int num = 10000 * RANDOM_MULTIPLIER;
+ for (int i = 0; i < num; i++) {
+ String s = _TestUtil.randomUnicodeString(random);
+ TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ ts.reset();
+ OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
+ while (ts.incrementToken()) {
+ String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
+ for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
+ cp = highlightedText.codePointAt(j);
+ assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
+ }
+ }
+ ts.end();
+ ts.close();
+ }
+ // just for fun
+ checkRandomData(random, analyzer, num);
+ }
}
Copied: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestDuelingAnalyzers.java (from r1213329, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestDuelingAnalyzers.java?p2=lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestDuelingAnalyzers.java&p1=lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java&r1=1213329&r2=1213366&rev=1213366&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestDuelingAnalyzers.java Mon Dec 12 18:59:10 2011
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.core;
+package org.apache.lucene.analysis;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -29,11 +29,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.automaton.Automaton;
-import org.apache.lucene.util.automaton.BasicOperations;
-import org.apache.lucene.util.automaton.CharacterRunAutomaton;
-import org.apache.lucene.util.automaton.State;
-import org.apache.lucene.util.automaton.Transition;
/**
* Compares MockTokenizer (which is simple with no optimizations) with equivalent
@@ -43,28 +38,10 @@ import org.apache.lucene.util.automaton.
* cause false fails).
*/
public class TestDuelingAnalyzers extends LuceneTestCase {
- private CharacterRunAutomaton jvmLetter;
-
- public void setUp() throws Exception {
- super.setUp();
- // build an automaton matching this jvm's letter definition
- State initial = new State();
- State accept = new State();
- accept.setAccept(true);
- for (int i = 0; i <= 0x10FFFF; i++) {
- if (Character.isLetter(i)) {
- initial.addTransition(new Transition(i, i, accept));
- }
- }
- Automaton single = new Automaton(initial);
- single.reduce();
- Automaton repeat = BasicOperations.repeat(single);
- jvmLetter = new CharacterRunAutomaton(repeat);
- }
public void testLetterAscii() throws Exception {
- Analyzer left = new MockAnalyzer(random, jvmLetter, false);
- Analyzer right = new Analyzer() {
+ Analyzer left = new MockAnalyzer(random, MockTokenizer.SIMPLE, false);
+ Analyzer right = new ReusableAnalyzerBase() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
@@ -79,8 +56,8 @@ public class TestDuelingAnalyzers extend
}
public void testLetterUnicode() throws Exception {
- Analyzer left = new MockAnalyzer(random, jvmLetter, false);
- Analyzer right = new Analyzer() {
+ Analyzer left = new MockAnalyzer(random, MockTokenizer.SIMPLE, false);
+ Analyzer right = new ReusableAnalyzerBase() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);