You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2015/12/10 19:38:59 UTC
[10/27] lucenenet git commit: adding converted analysis common tests
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenizerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenizerTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenizerTest.cs
new file mode 100644
index 0000000..d633096
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenizerTest.cs
@@ -0,0 +1,329 @@
+namespace org.apache.lucene.analysis.ngram
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+
+ using Version = org.apache.lucene.util.Version;
+ using TestUtil = org.apache.lucene.util.TestUtil;
+
+ using RandomStrings = com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+ /// <summary>
+ /// Tests <seealso cref="EdgeNGramTokenizer"/> for correctness.
+ /// </summary>
+ public class EdgeNGramTokenizerTest : BaseTokenStreamTestCase
+ {
+ private StringReader input;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ input = new StringReader("abcde");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput() throws Exception
+ public virtual void testInvalidInput()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 0, 0);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput2() throws Exception
+ public virtual void testInvalidInput2()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 2, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput3() throws Exception
+ public virtual void testInvalidInput3()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, -1, 2);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFrontUnigram() throws Exception
+ public virtual void testFrontUnigram()
+ {
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"a"}, new int[]{0}, new int[]{1}, 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBackUnigram() throws Exception
+ public virtual void testBackUnigram()
+ {
+ Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.BACK, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"e"}, new int[]{4}, new int[]{5}, 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOversizedNgrams() throws Exception
+ public virtual void testOversizedNgrams()
+ {
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 6, 6);
+ assertTokenStreamContents(tokenizer, new string[0], new int[0], new int[0], 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFrontRangeOfNgrams() throws Exception
+ public virtual void testFrontRangeOfNgrams()
+ {
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBackRangeOfNgrams() throws Exception
+ public virtual void testBackRangeOfNgrams()
+ {
+ Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.BACK, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, null, null, null, 5, false); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReset() throws Exception
+ public virtual void testReset()
+ {
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5); // abcde
+ tokenizer.Reader = new StringReader("abcde");
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5); // abcde
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ for (int i = 0; i < 10; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int min = org.apache.lucene.util.TestUtil.nextInt(random(), 2, 10);
+ int min = TestUtil.Next(random(), 2, 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int max = org.apache.lucene.util.TestUtil.nextInt(random(), min, 20);
+ int max = TestUtil.Next(random(), min, 20);
+
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper(this, min, max);
+ checkRandomData(random(), a, 100 * RANDOM_MULTIPLIER, 20);
+ checkRandomData(random(), a, 10 * RANDOM_MULTIPLIER, 8192);
+ }
+
+ Analyzer b = new AnalyzerAnonymousInnerClassHelper2(this);
+ checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER, 20, false, false);
+ checkRandomData(random(), b, 100 * RANDOM_MULTIPLIER, 8192, false, false);
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly EdgeNGramTokenizerTest outerInstance;
+
+ private int min;
+ private int max;
+
+ public AnalyzerAnonymousInnerClassHelper(EdgeNGramTokenizerTest outerInstance, int min, int max)
+ {
+ this.outerInstance = outerInstance;
+ this.min = min;
+ this.max = max;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly EdgeNGramTokenizerTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper2(EdgeNGramTokenizerTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, reader, Lucene43EdgeNGramTokenizer.Side.BACK, 2, 4);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTokenizerPositions() throws Exception
+ public virtual void testTokenizerPositions()
+ {
+ Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.FRONT, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, null, new int[] {1,0,0}, null, null, false);
+
+ tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"), 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, null, new int[]{1,1,1}, null, null, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private static void testNGrams(int minGram, int maxGram, int length, string nonTokenChars)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength(random(), length);
+ string s = RandomStrings.randomAsciiOfLength(random(), length);
+ testNGrams(minGram, maxGram, s, nonTokenChars);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private static void testNGrams(int minGram, int maxGram, String s, String nonTokenChars) throws java.io.IOException
+ private static void testNGrams(int minGram, int maxGram, string s, string nonTokenChars)
+ {
+ NGramTokenizerTest.testNGrams(minGram, maxGram, s, nonTokenChars, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLargeInput() throws java.io.IOException
+ public virtual void testLargeInput()
+ {
+ // test sliding
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLargeMaxGram() throws java.io.IOException
+ public virtual void testLargeMaxGram()
+ {
+ // test sliding with maxGram > 1024
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1290, 1300);
+ int minGram = TestUtil.Next(random(), 1290, 1300);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 1300);
+ int maxGram = TestUtil.Next(random(), minGram, 1300);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPreTokenization() throws java.io.IOException
+ public virtual void testPreTokenization()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 0, 4 * 1024), "a");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testHeavyPreTokenization() throws java.io.IOException
+ public virtual void testHeavyPreTokenization()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 0, 4 * 1024), "abcdef");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFewTokenChars() throws java.io.IOException
+ public virtual void testFewTokenChars()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final char[] chrs = new char[org.apache.lucene.util.TestUtil.nextInt(random(), 4000, 5000)];
+ char[] chrs = new char[TestUtil.Next(random(), 4000, 5000)];
+ Arrays.fill(chrs, ' ');
+ for (int i = 0; i < chrs.Length; ++i)
+ {
+ if (random().nextFloat() < 0.1)
+ {
+ chrs[i] = 'a';
+ }
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 2);
+ int minGram = TestUtil.Next(random(), 1, 2);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 2);
+ int maxGram = TestUtil.Next(random(), minGram, 2);
+ testNGrams(minGram, maxGram, new string(chrs), " ");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFullUTF8Range() throws java.io.IOException
+ public virtual void testFullUTF8Range()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = org.apache.lucene.util.TestUtil.randomUnicodeString(random(), 4 * 1024);
+ string s = TestUtil.randomUnicodeString(random(), 4 * 1024);
+ testNGrams(minGram, maxGram, s, "");
+ testNGrams(minGram, maxGram, s, "abcdef");
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenFilterTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenFilterTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenFilterTest.cs
new file mode 100644
index 0000000..0c9fa9a
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenFilterTest.cs
@@ -0,0 +1,288 @@
+using System;
+
+namespace org.apache.lucene.analysis.ngram
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using WhitespaceTokenizer = org.apache.lucene.analysis.core.WhitespaceTokenizer;
+ using ASCIIFoldingFilter = org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ using OffsetAttribute = org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+ using TestUtil = org.apache.lucene.util.TestUtil;
+ using Version = org.apache.lucene.util.Version;
+
+
+ /// <summary>
+ /// Tests <seealso cref="NGramTokenFilter"/> for correctness.
+ /// </summary>
+ public class NGramTokenFilterTest : BaseTokenStreamTestCase
+ {
+ private TokenStream input;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput() throws Exception
+ public virtual void testInvalidInput()
+ {
+ bool gotException = false;
+ try
+ {
+ new NGramTokenFilter(TEST_VERSION_CURRENT, input, 2, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput2() throws Exception
+ public virtual void testInvalidInput2()
+ {
+ bool gotException = false;
+ try
+ {
+ new NGramTokenFilter(TEST_VERSION_CURRENT, input, 0, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testUnigrams() throws Exception
+ public virtual void testUnigrams()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 1, 1);
+ assertTokenStreamContents(filter, new string[]{"a","b","c","d","e"}, new int[]{0,0,0,0,0}, new int[]{5,5,5,5,5}, new int[]{1,0,0,0,0});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBigrams() throws Exception
+ public virtual void testBigrams()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 2, 2);
+ assertTokenStreamContents(filter, new string[]{"ab","bc","cd","de"}, new int[]{0,0,0,0}, new int[]{5,5,5,5}, new int[]{1,0,0,0});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNgrams() throws Exception
+ public virtual void testNgrams()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 1, 3);
+ assertTokenStreamContents(filter, new string[]{"a","ab","abc","b","bc","bcd","c","cd","cde","d","de","e"}, new int[]{0,0,0,0,0,0,0,0,0,0,0,0}, new int[]{5,5,5,5,5,5,5,5,5,5,5,5}, null, new int[]{1,0,0,0,0,0,0,0,0,0,0,0}, null, null, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNgramsNoIncrement() throws Exception
+ public virtual void testNgramsNoIncrement()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 1, 3);
+ assertTokenStreamContents(filter, new string[]{"a","ab","abc","b","bc","bcd","c","cd","cde","d","de","e"}, new int[]{0,0,0,0,0,0,0,0,0,0,0,0}, new int[]{5,5,5,5,5,5,5,5,5,5,5,5}, null, new int[]{1,0,0,0,0,0,0,0,0,0,0,0}, null, null, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOversizedNgrams() throws Exception
+ public virtual void testOversizedNgrams()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 6, 7);
+ assertTokenStreamContents(filter, new string[0], new int[0], new int[0]);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testSmallTokenInStream() throws Exception
+ public virtual void testSmallTokenInStream()
+ {
+ input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 3, 3);
+ assertTokenStreamContents(filter, new string[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10}, new int[] {1, 2});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReset() throws Exception
+ public virtual void testReset()
+ {
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
+ NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 1, 1);
+ assertTokenStreamContents(filter, new string[]{"a","b","c","d","e"}, new int[]{0,0,0,0,0}, new int[]{5,5,5,5,5}, new int[]{1,0,0,0,0});
+ tokenizer.Reader = new StringReader("abcde");
+ assertTokenStreamContents(filter, new string[]{"a","b","c","d","e"}, new int[]{0,0,0,0,0}, new int[]{5,5,5,5,5}, new int[]{1,0,0,0,0});
+ }
+
+ // LUCENE-3642
+ // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
+ // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
+ // so in this case we behave like WDF, and preserve any modified offsets
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidOffsets() throws Exception
+ public virtual void testInvalidOffsets()
+ {
+ Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper(this);
+ assertAnalyzesTo(analyzer, "mosfellsbær", new string[] {"mo", "os", "sf", "fe", "el", "ll", "ls", "sb", "ba", "ae", "er"}, new int[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, new int[] {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11}, new int[] {1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0});
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly NGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper(NGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new NGramTokenFilter(TEST_VERSION_CURRENT, filters, 2, 2);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ for (int i = 0; i < 10; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int min = org.apache.lucene.util.TestUtil.nextInt(random(), 2, 10);
+ int min = TestUtil.Next(random(), 2, 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int max = org.apache.lucene.util.TestUtil.nextInt(random(), min, 20);
+ int max = TestUtil.Next(random(), min, 20);
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper2(this, min, max);
+ checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER, 20);
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly NGramTokenFilterTest outerInstance;
+
+ private int min;
+ private int max;
+
+ public AnalyzerAnonymousInnerClassHelper2(NGramTokenFilterTest outerInstance, int min, int max)
+ {
+ this.outerInstance = outerInstance;
+ this.min = min;
+ this.max = max;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, min, max));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws Exception
+ public virtual void testEmptyTerm()
+ {
+ Random random = random();
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper3(this);
+ checkAnalysisConsistency(random, a, random.nextBoolean(), "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly NGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper3(NGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 2, 15));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLucene43() throws java.io.IOException
+ public virtual void testLucene43()
+ {
+ NGramTokenFilter filter = new NGramTokenFilter(Version.LUCENE_43, input, 2, 3);
+ assertTokenStreamContents(filter, new string[]{"ab","bc","cd","de","abc","bcd","cde"}, new int[]{0,1,2,3,0,1,2}, new int[]{2,3,4,5,3,4,5}, null, new int[]{1,1,1,1,1,1,1}, null, null, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testSupplementaryCharacters() throws java.io.IOException
+ public virtual void testSupplementaryCharacters()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = org.apache.lucene.util.TestUtil.randomUnicodeString(random(), 10);
+ string s = TestUtil.randomUnicodeString(random(), 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int codePointCount = s.codePointCount(0, s.length());
+ int codePointCount = s.codePointCount(0, s.Length);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 3);
+ int minGram = TestUtil.Next(random(), 1, 3);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 10);
+ int maxGram = TestUtil.Next(random(), minGram, 10);
+ TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ tk = new NGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt = tk.addAttribute(org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class);
+ CharTermAttribute termAtt = tk.addAttribute(typeof(CharTermAttribute));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.OffsetAttribute offsetAtt = tk.addAttribute(org.apache.lucene.analysis.tokenattributes.OffsetAttribute.class);
+ OffsetAttribute offsetAtt = tk.addAttribute(typeof(OffsetAttribute));
+ tk.reset();
+ for (int start = 0; start < codePointCount; ++start)
+ {
+ for (int end = start + minGram; end <= Math.Min(codePointCount, start + maxGram); ++end)
+ {
+ assertTrue(tk.incrementToken());
+ assertEquals(0, offsetAtt.startOffset());
+ assertEquals(s.Length, offsetAtt.endOffset());
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int startIndex = Character.offsetByCodePoints(s, 0, start);
+ int startIndex = char.offsetByCodePoints(s, 0, start);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int endIndex = Character.offsetByCodePoints(s, 0, end);
+ int endIndex = char.offsetByCodePoints(s, 0, end);
+ assertEquals(s.Substring(startIndex, endIndex - startIndex), termAtt.ToString());
+ }
+ }
+ assertFalse(tk.incrementToken());
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenizerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenizerTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenizerTest.cs
new file mode 100644
index 0000000..15167bb
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/NGramTokenizerTest.cs
@@ -0,0 +1,386 @@
+namespace org.apache.lucene.analysis.ngram
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.ngram.NGramTokenizerTest.isTokenChar;
+
+
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ using OffsetAttribute = org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+ using PositionIncrementAttribute = org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+ using PositionLengthAttribute = org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+ using TestUtil = org.apache.lucene.util.TestUtil;
+
+ using RandomStrings = com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
+ /// <summary>
+ /// Tests <seealso cref="NGramTokenizer"/> for correctness.
+ /// </summary>
+ public class NGramTokenizerTest : BaseTokenStreamTestCase
+ {
+ private StringReader input;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ input = new StringReader("abcde");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput() throws Exception
+ public virtual void testInvalidInput()
+ {
+ bool gotException = false;
+ try
+ {
+ new NGramTokenizer(TEST_VERSION_CURRENT, input, 2, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput2() throws Exception
+ public virtual void testInvalidInput2()
+ {
+ bool gotException = false;
+ try
+ {
+ new NGramTokenizer(TEST_VERSION_CURRENT, input, 0, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testUnigrams() throws Exception
+ public virtual void testUnigrams()
+ {
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBigrams() throws Exception
+ public virtual void testBigrams()
+ {
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 2, 2);
+ assertTokenStreamContents(tokenizer, new string[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}, 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNgrams() throws Exception
+ public virtual void testNgrams()
+ {
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab", "abc", "b", "bc", "bcd", "c", "cd", "cde", "d", "de", "e"}, new int[]{0,0,0,1,1,1,2,2,2,3,3,4}, new int[]{1,2,3,2,3,4,3,4,5,4,5,5}, null, null, null, 5, false); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOversizedNgrams() throws Exception
+ public virtual void testOversizedNgrams()
+ {
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 6, 7);
+ assertTokenStreamContents(tokenizer, new string[0], new int[0], new int[0], 5); // abcde
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReset() throws Exception
+ public virtual void testReset()
+ {
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5); // abcde
+ tokenizer.Reader = new StringReader("abcde");
+ assertTokenStreamContents(tokenizer, new string[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5); // abcde
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ for (int i = 0; i < 10; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int min = org.apache.lucene.util.TestUtil.nextInt(random(), 2, 10);
+ int min = TestUtil.Next(random(), 2, 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int max = org.apache.lucene.util.TestUtil.nextInt(random(), min, 20);
+ int max = TestUtil.Next(random(), min, 20);
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper(this, min, max);
+ checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER, 20);
+ checkRandomData(random(), a, 10 * RANDOM_MULTIPLIER, 1027);
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly NGramTokenizerTest outerInstance;
+
+ private int min;
+ private int max;
+
+ public AnalyzerAnonymousInnerClassHelper(NGramTokenizerTest outerInstance, int min, int max)
+ {
+ this.outerInstance = outerInstance;
+ this.min = min;
+ this.max = max;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private static void testNGrams(int minGram, int maxGram, int length, string nonTokenChars)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiOfLength(random(), length);
+ string s = RandomStrings.randomAsciiOfLength(random(), length);
+ testNGrams(minGram, maxGram, s, nonTokenChars);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private static void testNGrams(int minGram, int maxGram, String s, String nonTokenChars) throws java.io.IOException
+ private static void testNGrams(int minGram, int maxGram, string s, string nonTokenChars)
+ {
+ testNGrams(minGram, maxGram, s, nonTokenChars, false);
+ }
+
+ internal static int[] toCodePoints(CharSequence s)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] codePoints = new int[Character.codePointCount(s, 0, s.length())];
+ int[] codePoints = new int[char.codePointCount(s, 0, s.length())];
+ for (int i = 0, j = 0; i < s.length(); ++j)
+ {
+ codePoints[j] = char.codePointAt(s, i);
+ i += char.charCount(codePoints[j]);
+ }
+ return codePoints;
+ }
+
+ internal static bool isTokenChar(string nonTokenChars, int codePoint)
+ {
+ for (int i = 0; i < nonTokenChars.Length;)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int cp = nonTokenChars.codePointAt(i);
+ int cp = char.ConvertToUtf32(nonTokenChars, i);
+ if (cp == codePoint)
+ {
+ return false;
+ }
+ i += char.charCount(cp);
+ }
+ return true;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: static void testNGrams(int minGram, int maxGram, String s, final String nonTokenChars, boolean edgesOnly) throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ internal static void testNGrams(int minGram, int maxGram, string s, string nonTokenChars, bool edgesOnly)
+ {
+ // convert the string to code points
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] codePoints = toCodePoints(s);
+ int[] codePoints = toCodePoints(s);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int[] offsets = new int[codePoints.length + 1];
+ int[] offsets = new int[codePoints.Length + 1];
+ for (int i = 0; i < codePoints.Length; ++i)
+ {
+ offsets[i + 1] = offsets[i] + char.charCount(codePoints[i]);
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new java.io.StringReader(s), minGram, maxGram, edgesOnly)
+ TokenStream grams = new NGramTokenizerAnonymousInnerClassHelper(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram, edgesOnly, nonTokenChars);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt = grams.addAttribute(org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class);
+ CharTermAttribute termAtt = grams.addAttribute(typeof(CharTermAttribute));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute posIncAtt = grams.addAttribute(org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute.class);
+ PositionIncrementAttribute posIncAtt = grams.addAttribute(typeof(PositionIncrementAttribute));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute posLenAtt = grams.addAttribute(org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute.class);
+ PositionLengthAttribute posLenAtt = grams.addAttribute(typeof(PositionLengthAttribute));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.OffsetAttribute offsetAtt = grams.addAttribute(org.apache.lucene.analysis.tokenattributes.OffsetAttribute.class);
+ OffsetAttribute offsetAtt = grams.addAttribute(typeof(OffsetAttribute));
+ grams.reset();
+ for (int start = 0; start < codePoints.Length; ++start)
+ {
+ for (int end = start + minGram; end <= start + maxGram && end <= codePoints.Length; ++end)
+ {
+ if (edgesOnly && start > 0 && isTokenChar(nonTokenChars, codePoints[start - 1]))
+ {
+ // not on an edge
+ goto nextGramContinue;
+ }
+ for (int j = start; j < end; ++j)
+ {
+ if (!isTokenChar(nonTokenChars, codePoints[j]))
+ {
+ goto nextGramContinue;
+ }
+ }
+ assertTrue(grams.incrementToken());
+ assertArrayEquals(Arrays.copyOfRange(codePoints, start, end), toCodePoints(termAtt));
+ assertEquals(1, posIncAtt.PositionIncrement);
+ assertEquals(1, posLenAtt.PositionLength);
+ assertEquals(offsets[start], offsetAtt.startOffset());
+ assertEquals(offsets[end], offsetAtt.endOffset());
+ nextGramContinue:;
+ }
+ nextGramBreak:;
+ }
+ assertFalse(grams.incrementToken());
+ grams.end();
+ assertEquals(s.Length, offsetAtt.startOffset());
+ assertEquals(s.Length, offsetAtt.endOffset());
+ }
+
+ private class NGramTokenizerAnonymousInnerClassHelper : NGramTokenizer
+ {
+ private string nonTokenChars;
+
+ public NGramTokenizerAnonymousInnerClassHelper(UnknownType TEST_VERSION_CURRENT, StringReader java, int minGram, int maxGram, bool edgesOnly, string nonTokenChars) : base(TEST_VERSION_CURRENT, StringReader, minGram, maxGram, edgesOnly)
+ {
+ this.nonTokenChars = nonTokenChars;
+ }
+
+ protected internal override bool isTokenChar(int chr)
+ {
+ return nonTokenChars.IndexOf(chr) < 0;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLargeInput() throws java.io.IOException
+ public virtual void testLargeInput()
+ {
+ // test sliding
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLargeMaxGram() throws java.io.IOException
+ public virtual void testLargeMaxGram()
+ {
+ // test sliding with maxGram > 1024
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1290, 1300);
+ int minGram = TestUtil.Next(random(), 1290, 1300);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 1300);
+ int maxGram = TestUtil.Next(random(), minGram, 1300);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPreTokenization() throws java.io.IOException
+ public virtual void testPreTokenization()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 0, 4 * 1024), "a");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testHeavyPreTokenization() throws java.io.IOException
+ public virtual void testHeavyPreTokenization()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+ testNGrams(minGram, maxGram, TestUtil.Next(random(), 0, 4 * 1024), "abcdef");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFewTokenChars() throws java.io.IOException
+ public virtual void testFewTokenChars()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final char[] chrs = new char[org.apache.lucene.util.TestUtil.nextInt(random(), 4000, 5000)];
+ char[] chrs = new char[TestUtil.Next(random(), 4000, 5000)];
+ Arrays.fill(chrs, ' ');
+ for (int i = 0; i < chrs.Length; ++i)
+ {
+ if (random().nextFloat() < 0.1)
+ {
+ chrs[i] = 'a';
+ }
+ }
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 2);
+ int minGram = TestUtil.Next(random(), 1, 2);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 2);
+ int maxGram = TestUtil.Next(random(), minGram, 2);
+ testNGrams(minGram, maxGram, new string(chrs), " ");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFullUTF8Range() throws java.io.IOException
+ public virtual void testFullUTF8Range()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 100);
+ int minGram = TestUtil.Next(random(), 1, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 100);
+ int maxGram = TestUtil.Next(random(), minGram, 100);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = org.apache.lucene.util.TestUtil.randomUnicodeString(random(), 4 * 1024);
+ string s = TestUtil.randomUnicodeString(random(), 4 * 1024);
+ testNGrams(minGram, maxGram, s, "");
+ testNGrams(minGram, maxGram, s, "abcdef");
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/TestNGramFilters.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/TestNGramFilters.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/TestNGramFilters.cs
new file mode 100644
index 0000000..5674622
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/TestNGramFilters.cs
@@ -0,0 +1,203 @@
+namespace org.apache.lucene.analysis.ngram
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+ using Version = org.apache.lucene.util.Version;
+
+ /// <summary>
+ /// Simple tests to ensure the NGram filter factories are working.
+ /// </summary>
+ public class TestNGramFilters : BaseTokenStreamFactoryTestCase
+ {
+ /// <summary>
+ /// Test NGramTokenizerFactory
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNGramTokenizer() throws Exception
+ public virtual void testNGramTokenizer()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = tokenizerFactory("NGram").create(reader);
+ assertTokenStreamContents(stream, new string[] {"t", "te", "e", "es", "s", "st", "t"});
+ }
+
+ /// <summary>
+ /// Test NGramTokenizerFactory with min and max gram options
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNGramTokenizer2() throws Exception
+ public virtual void testNGramTokenizer2()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = tokenizerFactory("NGram", "minGramSize", "2", "maxGramSize", "3").create(reader);
+ assertTokenStreamContents(stream, new string[] {"te", "tes", "es", "est", "st"});
+ }
+
+ /// <summary>
+ /// Test the NGramFilterFactory
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNGramFilter() throws Exception
+ public virtual void testNGramFilter()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NGram").create(stream);
+ assertTokenStreamContents(stream, new string[] {"t", "te", "e", "es", "s", "st", "t"});
+ }
+
+ /// <summary>
+ /// Test the NGramFilterFactory with min and max gram options
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNGramFilter2() throws Exception
+ public virtual void testNGramFilter2()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NGram", "minGramSize", "2", "maxGramSize", "3").create(stream);
+ assertTokenStreamContents(stream, new string[] {"te", "tes", "es", "est", "st"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramTokenizerFactory
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramTokenizer() throws Exception
+ public virtual void testEdgeNGramTokenizer()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = tokenizerFactory("EdgeNGram").create(reader);
+ assertTokenStreamContents(stream, new string[] {"t"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramTokenizerFactory with min and max gram size
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramTokenizer2() throws Exception
+ public virtual void testEdgeNGramTokenizer2()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = tokenizerFactory("EdgeNGram", "minGramSize", "1", "maxGramSize", "2").create(reader);
+ assertTokenStreamContents(stream, new string[] {"t", "te"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramTokenizerFactory with side option
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramTokenizer3() throws Exception
+ public virtual void testEdgeNGramTokenizer3()
+ {
+ Reader reader = new StringReader("ready");
+ TokenStream stream = tokenizerFactory("EdgeNGram", Version.LUCENE_43, "side", "back").create(reader);
+ assertTokenStreamContents(stream, new string[] {"y"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramFilterFactory
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramFilter() throws Exception
+ public virtual void testEdgeNGramFilter()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("EdgeNGram").create(stream);
+ assertTokenStreamContents(stream, new string[] {"t"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramFilterFactory with min and max gram size
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramFilter2() throws Exception
+ public virtual void testEdgeNGramFilter2()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("EdgeNGram", "minGramSize", "1", "maxGramSize", "2").create(stream);
+ assertTokenStreamContents(stream, new string[] {"t", "te"});
+ }
+
+ /// <summary>
+ /// Test EdgeNGramFilterFactory with side option
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEdgeNGramFilter3() throws Exception
+ public virtual void testEdgeNGramFilter3()
+ {
+ Reader reader = new StringReader("ready");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("EdgeNGram", Version.LUCENE_43, "side", "back").create(stream);
+ assertTokenStreamContents(stream, new string[] {"y"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenizerFactory("NGram", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+
+ try
+ {
+ tokenizerFactory("EdgeNGram", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+
+ try
+ {
+ tokenFilterFactory("NGram", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+
+ try
+ {
+ tokenFilterFactory("EdgeNGram", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Nl/TestDutchStemmer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Nl/TestDutchStemmer.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Nl/TestDutchStemmer.cs
new file mode 100644
index 0000000..351b21f
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Nl/TestDutchStemmer.cs
@@ -0,0 +1,255 @@
+using System;
+
+namespace org.apache.lucene.analysis.nl
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using CharArrayMap = org.apache.lucene.analysis.util.CharArrayMap;
+ using CharArraySet = org.apache.lucene.analysis.util.CharArraySet;
+ using Version = org.apache.lucene.util.Version;
+
+ /// <summary>
+ /// Test the Dutch Stem Filter, which only modifies the term text.
+ ///
+ /// The code states that it uses the snowball algorithm, but tests reveal some differences.
+ ///
+ /// </summary>
+ public class TestDutchStemmer : BaseTokenStreamTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testWithSnowballExamples() throws Exception
+ public virtual void testWithSnowballExamples()
+ {
+ check("lichaamsziek", "lichaamsziek");
+ check("lichamelijk", "licham");
+ check("lichamelijke", "licham");
+ check("lichamelijkheden", "licham");
+ check("lichamen", "licham");
+ check("lichere", "licher");
+ check("licht", "licht");
+ check("lichtbeeld", "lichtbeeld");
+ check("lichtbruin", "lichtbruin");
+ check("lichtdoorlatende", "lichtdoorlat");
+ check("lichte", "licht");
+ check("lichten", "licht");
+ check("lichtende", "lichtend");
+ check("lichtenvoorde", "lichtenvoord");
+ check("lichter", "lichter");
+ check("lichtere", "lichter");
+ check("lichters", "lichter");
+ check("lichtgevoeligheid", "lichtgevoel");
+ check("lichtgewicht", "lichtgewicht");
+ check("lichtgrijs", "lichtgrijs");
+ check("lichthoeveelheid", "lichthoevel");
+ check("lichtintensiteit", "lichtintensiteit");
+ check("lichtje", "lichtj");
+ check("lichtjes", "lichtjes");
+ check("lichtkranten", "lichtkrant");
+ check("lichtkring", "lichtkring");
+ check("lichtkringen", "lichtkring");
+ check("lichtregelsystemen", "lichtregelsystem");
+ check("lichtste", "lichtst");
+ check("lichtstromende", "lichtstrom");
+ check("lichtte", "licht");
+ check("lichtten", "licht");
+ check("lichttoetreding", "lichttoetred");
+ check("lichtverontreinigde", "lichtverontreinigd");
+ check("lichtzinnige", "lichtzinn");
+ check("lid", "lid");
+ check("lidia", "lidia");
+ check("lidmaatschap", "lidmaatschap");
+ check("lidstaten", "lidstat");
+ check("lidvereniging", "lidveren");
+ check("opgingen", "opging");
+ check("opglanzing", "opglanz");
+ check("opglanzingen", "opglanz");
+ check("opglimlachten", "opglimlacht");
+ check("opglimpen", "opglimp");
+ check("opglimpende", "opglimp");
+ check("opglimping", "opglimp");
+ check("opglimpingen", "opglimp");
+ check("opgraven", "opgrav");
+ check("opgrijnzen", "opgrijnz");
+ check("opgrijzende", "opgrijz");
+ check("opgroeien", "opgroei");
+ check("opgroeiende", "opgroei");
+ check("opgroeiplaats", "opgroeiplat");
+ check("ophaal", "ophal");
+ check("ophaaldienst", "ophaaldienst");
+ check("ophaalkosten", "ophaalkost");
+ check("ophaalsystemen", "ophaalsystem");
+ check("ophaalt", "ophaalt");
+ check("ophaaltruck", "ophaaltruck");
+ check("ophalen", "ophal");
+ check("ophalend", "ophal");
+ check("ophalers", "ophaler");
+ check("ophef", "ophef");
+ check("opheldering", "ophelder");
+ check("ophemelde", "ophemeld");
+ check("ophemelen", "ophemel");
+ check("opheusden", "opheusd");
+ check("ophief", "ophief");
+ check("ophield", "ophield");
+ check("ophieven", "ophiev");
+ check("ophoepelt", "ophoepelt");
+ check("ophoog", "ophog");
+ check("ophoogzand", "ophoogzand");
+ check("ophopen", "ophop");
+ check("ophoping", "ophop");
+ check("ophouden", "ophoud");
+ }
+
+ /// @deprecated (3.1) remove this test in Lucene 5.0
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Deprecated("(3.1) remove this test in Lucene 5.0") public void testOldBuggyStemmer() throws Exception
+ [Obsolete("(3.1) remove this test in Lucene 5.0")]
+ public virtual void testOldBuggyStemmer()
+ {
+ Analyzer a = new DutchAnalyzer(Version.LUCENE_30);
+ checkOneTerm(a, "opheffen", "ophef"); // versus snowball 'opheff'
+ checkOneTerm(a, "opheffende", "ophef"); // versus snowball 'opheff'
+ checkOneTerm(a, "opheffing", "ophef"); // versus snowball 'opheff'
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testSnowballCorrectness() throws Exception
+ public virtual void testSnowballCorrectness()
+ {
+ Analyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTerm(a, "opheffen", "opheff");
+ checkOneTerm(a, "opheffende", "opheff");
+ checkOneTerm(a, "opheffing", "opheff");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReusableTokenStream() throws Exception
+ public virtual void testReusableTokenStream()
+ {
+ Analyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTerm(a, "lichaamsziek", "lichaamsziek");
+ checkOneTerm(a, "lichamelijk", "licham");
+ checkOneTerm(a, "lichamelijke", "licham");
+ checkOneTerm(a, "lichamelijkheden", "licham");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testExclusionTableViaCtor() throws java.io.IOException
+ public virtual void testExclusionTableViaCtor()
+ {
+ CharArraySet set = new CharArraySet(Version.LUCENE_30, 1, true);
+ set.add("lichamelijk");
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET, set);
+ assertAnalyzesTo(a, "lichamelijk lichamelijke", new string[] {"lichamelijk", "licham"});
+
+ a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET, set);
+ assertAnalyzesTo(a, "lichamelijk lichamelijke", new string[] {"lichamelijk", "licham"});
+
+ }
+
+ /// <summary>
+ /// check that the default stem overrides are used
+ /// even if you use a non-default ctor.
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testStemOverrides() throws java.io.IOException
+ public virtual void testStemOverrides()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET);
+ checkOneTerm(a, "fiets", "fiets");
+ }
+ /// <summary>
+ /// 3.0 still uses the chararraymap internally check if that works as well </summary>
+ /// @deprecated (4.3) remove this test in Lucene 5.0
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Deprecated("(4.3) remove this test in Lucene 5.0") public void test30StemOverrides() throws java.io.IOException
+ [Obsolete("(4.3) remove this test in Lucene 5.0")]
+ public virtual void test30StemOverrides()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_30);
+ checkOneTerm(a, "fiets", "fiets");
+ a = new DutchAnalyzer(Version.LUCENE_30, CharArraySet.EMPTY_SET);
+ checkOneTerm(a, "fiets", "fiet"); // only the default ctor populates the dict
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyStemDictionary() throws java.io.IOException
+ public virtual void testEmptyStemDictionary()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET, CharArraySet.EMPTY_SET, CharArrayMap.emptyMap<string>());
+ checkOneTerm(a, "fiets", "fiet");
+ }
+
+ /// <summary>
+ /// prior to 3.6, this confusingly did not happen if
+ /// you specified your own stoplist!!!! </summary>
+ /// @deprecated (3.6) Remove this test in Lucene 5.0
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Deprecated("(3.6) Remove this test in Lucene 5.0") public void testBuggyStemOverrides() throws java.io.IOException
+ [Obsolete("(3.6) Remove this test in Lucene 5.0")]
+ public virtual void testBuggyStemOverrides()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_35, CharArraySet.EMPTY_SET);
+ checkOneTerm(a, "fiets", "fiet");
+ }
+
+ /// <summary>
+ /// Prior to 3.1, this analyzer had no lowercase filter.
+ /// stopwords were case sensitive. Preserve this for back compat. </summary>
+ /// @deprecated (3.1) Remove this test in Lucene 5.0
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Deprecated("(3.1) Remove this test in Lucene 5.0") public void testBuggyStopwordsCasing() throws java.io.IOException
+ [Obsolete("(3.1) Remove this test in Lucene 5.0")]
+ public virtual void testBuggyStopwordsCasing()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_30);
+ assertAnalyzesTo(a, "Zelf", new string[] {"zelf"});
+ }
+
+ /// <summary>
+ /// Test that stopwords are not case sensitive
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testStopwordsCasing() throws java.io.IOException
+ public virtual void testStopwordsCasing()
+ {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_31);
+ assertAnalyzesTo(a, "Zelf", new string[] { });
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: private void check(final String input, final String expected) throws Exception
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ private void check(string input, string expected)
+ {
+ checkOneTerm(new DutchAnalyzer(TEST_VERSION_CURRENT), input, expected);
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ checkRandomData(random(), new DutchAnalyzer(TEST_VERSION_CURRENT), 1000 * RANDOM_MULTIPLIER);
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianAnalyzer.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianAnalyzer.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianAnalyzer.cs
new file mode 100644
index 0000000..136ec1f
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianAnalyzer.cs
@@ -0,0 +1,70 @@
+namespace org.apache.lucene.analysis.no
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using CharArraySet = org.apache.lucene.analysis.util.CharArraySet;
+
+ public class TestNorwegianAnalyzer : BaseTokenStreamTestCase
+ {
+ /// <summary>
+ /// This test fails with NPE when the
+ /// stopwords file is missing in classpath
+ /// </summary>
+ public virtual void testResourcesAvailable()
+ {
+ new NorwegianAnalyzer(TEST_VERSION_CURRENT);
+ }
+
+ /// <summary>
+ /// test stopwords and stemming </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBasics() throws java.io.IOException
+ public virtual void testBasics()
+ {
+ Analyzer a = new NorwegianAnalyzer(TEST_VERSION_CURRENT);
+ // stemming
+ checkOneTerm(a, "havnedistriktene", "havnedistrikt");
+ checkOneTerm(a, "havnedistrikter", "havnedistrikt");
+ // stopword
+ assertAnalyzesTo(a, "det", new string[] {});
+ }
+
+ /// <summary>
+ /// test use of exclusion set </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testExclude() throws java.io.IOException
+ public virtual void testExclude()
+ {
+ CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("havnedistriktene"), false);
+ Analyzer a = new NorwegianAnalyzer(TEST_VERSION_CURRENT, NorwegianAnalyzer.DefaultStopSet, exclusionSet);
+ checkOneTerm(a, "havnedistriktene", "havnedistriktene");
+ checkOneTerm(a, "havnedistrikter", "havnedistrikt");
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ checkRandomData(random(), new NorwegianAnalyzer(TEST_VERSION_CURRENT), 1000 * RANDOM_MULTIPLIER);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilter.cs
new file mode 100644
index 0000000..edbbf53
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilter.cs
@@ -0,0 +1,157 @@
+using System;
+
+namespace org.apache.lucene.analysis.no
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using SetKeywordMarkerFilter = org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+ using CharArraySet = org.apache.lucene.analysis.util.CharArraySet;
+
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.VocabularyAssert.*;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK;
+
+
+ /// <summary>
+ /// Simple tests for <seealso cref="NorwegianLightStemFilter"/>
+ /// </summary>
+ public class TestNorwegianLightStemFilter : BaseTokenStreamTestCase
+ {
+ private Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper();
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ public AnalyzerAnonymousInnerClassHelper()
+ {
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL));
+ }
+ }
+
+ /// <summary>
+ /// Test against a vocabulary file </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testVocabulary() throws java.io.IOException
+ public virtual void testVocabulary()
+ {
+ assertVocabulary(analyzer, new System.IO.FileStream(getDataFile("nb_light.txt"), System.IO.FileMode.Open, System.IO.FileAccess.Read));
+ }
+
+ /// <summary>
+ /// Test against a Nynorsk vocabulary file </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNynorskVocabulary() throws java.io.IOException
+ public virtual void testNynorskVocabulary()
+ {
+ Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper2(this);
+ assertVocabulary(analyzer, new System.IO.FileStream(getDataFile("nn_light.txt"), System.IO.FileMode.Open, System.IO.FileAccess.Read));
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly TestNorwegianLightStemFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper2(TestNorwegianLightStemFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NYNORSK));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testKeyword() throws java.io.IOException
+ public virtual void testKeyword()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.util.CharArraySet exclusionSet = new org.apache.lucene.analysis.util.CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
+ CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper3(this, exclusionSet);
+ checkOneTerm(a, "sekretæren", "sekretæren");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly TestNorwegianLightStemFilter outerInstance;
+
+ private CharArraySet exclusionSet;
+
+ public AnalyzerAnonymousInnerClassHelper3(TestNorwegianLightStemFilter outerInstance, CharArraySet exclusionSet)
+ {
+ this.outerInstance = outerInstance;
+ this.exclusionSet = exclusionSet;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(sink));
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ Random random = random();
+ checkRandomData(random, analyzer, 1000 * RANDOM_MULTIPLIER);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws java.io.IOException
+ public virtual void testEmptyTerm()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper4(this);
+ checkOneTerm(a, "", "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper4 : Analyzer
+ {
+ private readonly TestNorwegianLightStemFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper4(TestNorwegianLightStemFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new NorwegianLightStemFilter(tokenizer));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilterFactory.cs
new file mode 100644
index 0000000..a4fa114
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/No/TestNorwegianLightStemFilterFactory.cs
@@ -0,0 +1,81 @@
+namespace org.apache.lucene.analysis.no
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+ /// <summary>
+ /// Simple tests to ensure the Norwegian Light stem factory is working.
+ /// </summary>
+ public class TestNorwegianLightStemFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testStemming() throws Exception
+ public virtual void testStemming()
+ {
+ Reader reader = new StringReader("epler eple");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem").create(stream);
+ assertTokenStreamContents(stream, new string[] {"epl", "epl"});
+ }
+
+ /// <summary>
+ /// Test stemming with variant set explicitly to Bokmål </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBokmaalStemming() throws Exception
+ public virtual void testBokmaalStemming()
+ {
+ Reader reader = new StringReader("epler eple");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nb").create(stream);
+ assertTokenStreamContents(stream, new string[] {"epl", "epl"});
+ }
+
+ /// <summary>
+ /// Test stemming with variant set explicitly to Nynorsk </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNynorskStemming() throws Exception
+ public virtual void testNynorskStemming()
+ {
+ Reader reader = new StringReader("gutar gutane");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nn").create(stream);
+ assertTokenStreamContents(stream, new string[] {"gut", "gut"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("NorwegianLightStem", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+
+}
\ No newline at end of file