You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/04/24 18:14:20 UTC
svn commit: r1096341 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/
lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/
solr/
Author: rmuir
Date: Sun Apr 24 16:14:19 2011
New Revision: 1096341
URL: http://svn.apache.org/viewvc?rev=1096341&view=rev
Log:
LUCENE-2560: stress tests for icu integration
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java?rev=1096341&r1=1096340&r2=1096341&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java Sun Apr 24 16:14:19 2011
@@ -29,15 +29,14 @@ import org.apache.lucene.analysis.Whites
* Tests ICUFoldingFilter
*/
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
+ Analyzer a = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new ICUFoldingFilter(
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
+ }
+ };
public void testDefaults() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new ICUFoldingFilter(
- new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
- }
- };
-
// case folding
assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" });
@@ -76,4 +75,9 @@ public class TestICUFoldingFilter extend
// handling of decomposed combining-dot-above
assertAnalyzesTo(a, "eli\u0307f", new String[] { "elif" });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java?rev=1096341&r1=1096340&r2=1096341&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java Sun Apr 24 16:14:19 2011
@@ -31,16 +31,15 @@ import com.ibm.icu.text.Normalizer2;
* Tests the ICUNormalizer2Filter
*/
public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
+ Analyzer a = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new ICUNormalizer2Filter(
+ new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
+ }
+ };
public void testDefaults() throws IOException {
- Analyzer a = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new ICUNormalizer2Filter(
- new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
- }
- };
-
// case folding
assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" });
@@ -75,4 +74,9 @@ public class TestICUNormalizer2Filter ex
// decompose EAcute into E + combining Acute
assertAnalyzesTo(a, "\u00E9", new String[] { "\u0065\u0301" });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java?rev=1096341&r1=1096340&r2=1096341&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java Sun Apr 24 16:14:19 2011
@@ -18,10 +18,15 @@ package org.apache.lucene.analysis.icu;
*/
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.KeywordTokenizer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import com.ibm.icu.text.Transliterator;
@@ -83,4 +88,17 @@ public class TestICUTransformFilter exte
TokenStream ts = new ICUTransformFilter(new KeywordTokenizer((new StringReader(input))), transform);
assertTokenStreamContents(ts, new String[] { expected });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ final Transliterator transform = Transliterator.getInstance("Any-Latin");
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
+ return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, transform));
+ }
+ };
+ checkRandomData(random, a, 1000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1096341&r1=1096340&r2=1096341&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Sun Apr 24 16:14:19 2011
@@ -232,4 +232,9 @@ public class TestICUTokenizer extends Ba
new String[] { "ä»®", "å", "é£", "ã", "ã«ã¿ã«ã" },
new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}