You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/24 11:50:13 UTC
svn commit: r1235201 - in /lucene/dev/branches/branch_3x:
lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/
lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/
lucene/contrib/analyzers/common/src/test/o...
Author: rmuir
Date: Tue Jan 24 10:50:13 2012
New Revision: 1235201
URL: http://svn.apache.org/viewvc?rev=1235201&view=rev
Log:
LUCENE-3717: add tests
Modified:
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilter.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilter.java
lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Tue Jan 24 10:50:13 2012
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.tokena
public final class PositionFilter extends TokenFilter {
/** Position increment to assign to all but the first token - default = 0 */
- private int positionIncrement = 0;
+ private final int positionIncrement;
/** The first token must have non-zero positionIncrement **/
private boolean firstTokenPositioned = false;
@@ -44,7 +44,7 @@ public final class PositionFilter extend
* @param input the input stream
*/
public PositionFilter(final TokenStream input) {
- super(input);
+ this(input, 0);
}
/**
@@ -56,7 +56,7 @@ public final class PositionFilter extend
* token from the input stream
*/
public PositionFilter(final TokenStream input, final int positionIncrement) {
- this(input);
+ super(input);
this.positionIncrement = positionIncrement;
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java Tue Jan 24 10:50:13 2012
@@ -17,12 +17,16 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.MappingCharFilter;
import org.apache.lucene.analysis.NormalizeCharMap;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.Tokenizer;
public class TestPathHierarchyTokenizer extends BaseTokenStreamTestCase {
@@ -193,4 +197,16 @@ public class TestPathHierarchyTokenizer
new int[]{1},
path.length());
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java Tue Jan 24 10:50:13 2012
@@ -17,9 +17,13 @@ package org.apache.lucene.analysis.path;
* limitations under the License.
*/
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.Tokenizer;
public class TestReversePathHierarchyTokenizer extends BaseTokenStreamTestCase {
@@ -154,4 +158,16 @@ public class TestReversePathHierarchyTok
new int[]{1, 0},
path.length());
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java Tue Jan 24 10:50:13 2012
@@ -17,11 +17,16 @@
package org.apache.lucene.analysis.reverse;
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.Version;
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
@@ -93,4 +98,16 @@ public class TestReverseStringFilter ext
ReverseStringFilter.reverse(TEST_VERSION_CURRENT, buffer, 3, 7);
assertEquals("abcfedð©¬
æ¯ç", new String(buffer));
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Tue Jan 24 10:50:13 2012
@@ -18,9 +18,13 @@ package org.apache.lucene.analysis.shing
*/
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -1129,4 +1133,16 @@ public class ShingleFilterTest extends B
token.setPositionIncrement(positionIncrement);
return token;
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilter.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceCharFilter.java Tue Jan 24 10:50:13 2012
@@ -18,15 +18,20 @@
package org.apache.solr.analysis;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
*
@@ -189,4 +194,21 @@ public class TestPatternReplaceCharFilte
private Pattern pattern( String p ){
return Pattern.compile( p );
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(Reader reader) {
+ return new PatternReplaceCharFilter(Pattern.compile("a"), "b", CharReader.get(reader));
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilter.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilter.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternReplaceFilter.java Tue Jan 24 10:50:13 2012
@@ -17,9 +17,14 @@
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import java.io.Reader;
import java.io.StringReader;
import java.util.regex.Pattern;
@@ -77,5 +82,28 @@ public class TestPatternReplaceFilter ex
assertTokenStreamContents(ts,
new String[] { "aa$fooaa$fooa$foo$", "a$", "caaaaaaaaa$" });
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
+ return new TokenStreamComponents(tokenizer, filter);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
+ return new TokenStreamComponents(tokenizer, filter);
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}
Modified: lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java?rev=1235201&r1=1235200&r2=1235201&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_3x/solr/core/src/test/org/apache/solr/analysis/TestPatternTokenizerFactory.java Tue Jan 24 10:50:13 2012
@@ -18,17 +18,22 @@
package org.apache.solr.analysis;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Pattern;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.MappingCharFilter;
import org.apache.lucene.analysis.NormalizeCharMap;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class TestPatternTokenizerFactory extends BaseTokenTestCase
@@ -136,4 +141,35 @@ public class TestPatternTokenizerFactory
in.close();
return out.toString();
}
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ Analyzer a = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = null;
+ try {
+ tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), -1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, a, 10000*RANDOM_MULTIPLIER);
+
+ Analyzer b = new ReusableAnalyzerBase() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = null;
+ try {
+ tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), 0);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ checkRandomData(random, b, 10000*RANDOM_MULTIPLIER);
+ }
}