You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2015/12/10 19:39:00 UTC
[11/27] lucenenet git commit: adding converted analysis common tests
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilter.cs
new file mode 100644
index 0000000..57a264b
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilter.cs
@@ -0,0 +1,154 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+
+
+ public class TestScandinavianNormalizationFilter : BaseTokenStreamTestCase
+ {
+
+
+ private Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper();
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ public AnalyzerAnonymousInnerClassHelper()
+ {
+ }
+
+ protected internal override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.Tokenizer tokenizer = new org.apache.lucene.analysis.MockTokenizer(reader, org.apache.lucene.analysis.MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.TokenStream stream = new ScandinavianNormalizationFilter(tokenizer);
+ TokenStream stream = new ScandinavianNormalizationFilter(tokenizer);
+ return new TokenStreamComponents(tokenizer, stream);
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void test() throws Exception
+ public virtual void test()
+ {
+
+ checkOneTerm(analyzer, "aeäaeeea", "æææeea"); // should not cause ArrayIndexOutOfBoundsException
+
+ checkOneTerm(analyzer, "aeäaeeeae", "æææeeæ");
+ checkOneTerm(analyzer, "aeaeeeae", "ææeeæ");
+
+ checkOneTerm(analyzer, "bøen", "bøen");
+ checkOneTerm(analyzer, "bOEen", "bØen");
+ checkOneTerm(analyzer, "åene", "åene");
+
+
+ checkOneTerm(analyzer, "blåbærsyltetøj", "blåbærsyltetøj");
+ checkOneTerm(analyzer, "blaabaersyltetöj", "blåbærsyltetøj");
+ checkOneTerm(analyzer, "räksmörgås", "ræksmørgås");
+ checkOneTerm(analyzer, "raeksmörgaos", "ræksmørgås");
+ checkOneTerm(analyzer, "raeksmörgaas", "ræksmørgås");
+ checkOneTerm(analyzer, "raeksmoergås", "ræksmørgås");
+
+
+ checkOneTerm(analyzer, "ab", "ab");
+ checkOneTerm(analyzer, "ob", "ob");
+ checkOneTerm(analyzer, "Ab", "Ab");
+ checkOneTerm(analyzer, "Ob", "Ob");
+
+ checkOneTerm(analyzer, "å", "å");
+
+ checkOneTerm(analyzer, "aa", "å");
+ checkOneTerm(analyzer, "aA", "å");
+ checkOneTerm(analyzer, "ao", "å");
+ checkOneTerm(analyzer, "aO", "å");
+
+ checkOneTerm(analyzer, "AA", "Å");
+ checkOneTerm(analyzer, "Aa", "Å");
+ checkOneTerm(analyzer, "Ao", "Å");
+ checkOneTerm(analyzer, "AO", "Å");
+
+ checkOneTerm(analyzer, "æ", "æ");
+ checkOneTerm(analyzer, "ä", "æ");
+
+ checkOneTerm(analyzer, "Æ", "Æ");
+ checkOneTerm(analyzer, "Ä", "Æ");
+
+ checkOneTerm(analyzer, "ae", "æ");
+ checkOneTerm(analyzer, "aE", "æ");
+
+ checkOneTerm(analyzer, "Ae", "Æ");
+ checkOneTerm(analyzer, "AE", "Æ");
+
+
+ checkOneTerm(analyzer, "ö", "ø");
+ checkOneTerm(analyzer, "ø", "ø");
+ checkOneTerm(analyzer, "Ö", "Ø");
+ checkOneTerm(analyzer, "Ø", "Ø");
+
+
+ checkOneTerm(analyzer, "oo", "ø");
+ checkOneTerm(analyzer, "oe", "ø");
+ checkOneTerm(analyzer, "oO", "ø");
+ checkOneTerm(analyzer, "oE", "ø");
+
+ checkOneTerm(analyzer, "Oo", "Ø");
+ checkOneTerm(analyzer, "Oe", "Ø");
+ checkOneTerm(analyzer, "OO", "Ø");
+ checkOneTerm(analyzer, "OE", "Ø");
+ }
+
+ /// <summary>
+ /// check that the empty string doesn't cause issues </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws Exception
+ public virtual void testEmptyTerm()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper2(this);
+ checkOneTerm(a, "", "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly TestScandinavianNormalizationFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper2(TestScandinavianNormalizationFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new ScandinavianNormalizationFilter(tokenizer));
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomData() throws Exception
+ public virtual void testRandomData()
+ {
+ checkRandomData(random(), analyzer, 1000 * RANDOM_MULTIPLIER);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilterFactory.cs
new file mode 100644
index 0000000..e4a1b09
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestScandinavianNormalizationFilterFactory.cs
@@ -0,0 +1,53 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /// <summary>
+ /// Copyright 2004 The Apache Software Foundation
+ ///
+ /// Licensed under the Apache License, Version 2.0 (the "License");
+ /// you may not use this file except in compliance with the License.
+ /// You may obtain a copy of the License at
+ ///
+ /// http://www.apache.org/licenses/LICENSE-2.0
+ ///
+ /// Unless required by applicable law or agreed to in writing, software
+ /// distributed under the License is distributed on an "AS IS" BASIS,
+ /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ /// See the License for the specific language governing permissions and
+ /// limitations under the License.
+ /// </summary>
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+
+ public class TestScandinavianNormalizationFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testStemming() throws Exception
+ public virtual void testStemming()
+ {
+ Reader reader = new StringReader("räksmörgås");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("ScandinavianNormalization").create(stream);
+ assertTokenStreamContents(stream, new string[] {"ræksmørgås"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("ScandinavianNormalization", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestSingleTokenTokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestSingleTokenTokenFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestSingleTokenTokenFilter.cs
new file mode 100644
index 0000000..7781dc4
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestSingleTokenTokenFilter.cs
@@ -0,0 +1,52 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using LuceneTestCase = org.apache.lucene.util.LuceneTestCase;
+ using AttributeImpl = org.apache.lucene.util.AttributeImpl;
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+ public class TestSingleTokenTokenFilter : LuceneTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void test() throws java.io.IOException
+ public virtual void test()
+ {
+ Token token = new Token();
+ SingleTokenTokenStream ts = new SingleTokenTokenStream(token);
+ AttributeImpl tokenAtt = (AttributeImpl) ts.addAttribute(typeof(CharTermAttribute));
+ assertTrue(tokenAtt is Token);
+ ts.reset();
+
+ assertTrue(ts.incrementToken());
+ assertEquals(token, tokenAtt);
+ assertFalse(ts.incrementToken());
+
+ token = new Token("hallo", 10, 20, "someType");
+ ts.Token = token;
+ ts.reset();
+
+ assertTrue(ts.incrementToken());
+ assertEquals(token, tokenAtt);
+ assertFalse(ts.incrementToken());
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilter.cs
new file mode 100644
index 0000000..d28bfae
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilter.cs
@@ -0,0 +1,157 @@
+using System.Collections.Generic;
+using System.Text;
+
+namespace org.apache.lucene.analysis.miscellaneous
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using WhitespaceTokenizer = org.apache.lucene.analysis.core.WhitespaceTokenizer;
+ using PorterStemFilter = org.apache.lucene.analysis.en.PorterStemFilter;
+ using StemmerOverrideMap = org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter.StemmerOverrideMap;
+ using TestUtil = org.apache.lucene.util.TestUtil;
+
+ ///
+ public class TestStemmerOverrideFilter : BaseTokenStreamTestCase
+ {
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOverride() throws java.io.IOException
+ public virtual void testOverride()
+ {
+ // lets make booked stem to books
+ // the override filter will convert "booked" to "books",
+ // but also mark it with KeywordAttribute so Porter will not change it.
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder();
+ builder.add("booked", "books");
+ Tokenizer tokenizer = new KeywordTokenizer(new StringReader("booked"));
+ TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(tokenizer, builder.build()));
+ assertTokenStreamContents(stream, new string[] {"books"});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testIgnoreCase() throws java.io.IOException
+ public virtual void testIgnoreCase()
+ {
+ // lets make booked stem to books
+ // the override filter will convert "booked" to "books",
+ // but also mark it with KeywordAttribute so Porter will not change it.
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(true);
+ builder.add("boOkEd", "books");
+ Tokenizer tokenizer = new KeywordTokenizer(new StringReader("BooKeD"));
+ TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(tokenizer, builder.build()));
+ assertTokenStreamContents(stream, new string[] {"books"});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNoOverrides() throws java.io.IOException
+ public virtual void testNoOverrides()
+ {
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(true);
+ Tokenizer tokenizer = new KeywordTokenizer(new StringReader("book"));
+ TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(tokenizer, builder.build()));
+ assertTokenStreamContents(stream, new string[] {"book"});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomRealisticWhiteSpace() throws java.io.IOException
+ public virtual void testRandomRealisticWhiteSpace()
+ {
+ IDictionary<string, string> map = new Dictionary<string, string>();
+ int numTerms = atLeast(50);
+ for (int i = 0; i < numTerms; i++)
+ {
+ string randomRealisticUnicodeString = TestUtil.randomRealisticUnicodeString(random());
+ char[] charArray = randomRealisticUnicodeString.ToCharArray();
+ StringBuilder builder = new StringBuilder();
+ for (int j = 0; j < charArray.Length;)
+ {
+ int cp = char.codePointAt(charArray, j, charArray.Length);
+ if (!char.IsWhiteSpace(cp))
+ {
+ builder.appendCodePoint(cp);
+ }
+ j += char.charCount(cp);
+ }
+ if (builder.Length > 0)
+ {
+ string value = TestUtil.randomSimpleString(random());
+ map[builder.ToString()] = value.Length == 0 ? "a" : value;
+
+ }
+ }
+ if (map.Count == 0)
+ {
+ map["booked"] = "books";
+ }
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(random().nextBoolean());
+ ISet<KeyValuePair<string, string>> entrySet = map.SetOfKeyValuePairs();
+ StringBuilder input = new StringBuilder();
+ IList<string> output = new List<string>();
+ foreach (KeyValuePair<string, string> entry in entrySet)
+ {
+ builder.add(entry.Key, entry.Value);
+ if (random().nextBoolean() || output.Count == 0)
+ {
+ input.Append(entry.Key).Append(" ");
+ output.Add(entry.Value);
+ }
+ }
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input.ToString()));
+ TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(tokenizer, builder.build()));
+ assertTokenStreamContents(stream, output.ToArray());
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomRealisticKeyword() throws java.io.IOException
+ public virtual void testRandomRealisticKeyword()
+ {
+ IDictionary<string, string> map = new Dictionary<string, string>();
+ int numTerms = atLeast(50);
+ for (int i = 0; i < numTerms; i++)
+ {
+ string randomRealisticUnicodeString = TestUtil.randomRealisticUnicodeString(random());
+ if (randomRealisticUnicodeString.Length > 0)
+ {
+ string value = TestUtil.randomSimpleString(random());
+ map[randomRealisticUnicodeString] = value.Length == 0 ? "a" : value;
+ }
+ }
+ if (map.Count == 0)
+ {
+ map["booked"] = "books";
+ }
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(random().nextBoolean());
+ ISet<KeyValuePair<string, string>> entrySet = map.SetOfKeyValuePairs();
+ foreach (KeyValuePair<string, string> entry in entrySet)
+ {
+ builder.add(entry.Key, entry.Value);
+ }
+ StemmerOverrideMap build = builder.build();
+ foreach (KeyValuePair<string, string> entry in entrySet)
+ {
+ if (random().nextBoolean())
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(new StringReader(entry.Key));
+ TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(tokenizer, build));
+ assertTokenStreamContents(stream, new string[] {entry.Value});
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilterFactory.cs
new file mode 100644
index 0000000..68abb57
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestStemmerOverrideFilterFactory.cs
@@ -0,0 +1,73 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+ using StringMockResourceLoader = org.apache.lucene.analysis.util.StringMockResourceLoader;
+
+ /// <summary>
+ /// Simple tests to ensure the stemmer override filter factory is working.
+ /// </summary>
+ public class TestStemmerOverrideFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testKeywords() throws Exception
+ public virtual void testKeywords()
+ {
+ // our stemdict stems dogs to 'cat'
+ Reader reader = new StringReader("testing dogs");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("StemmerOverride", TEST_VERSION_CURRENT, new StringMockResourceLoader("dogs\tcat"), "dictionary", "stemdict.txt").create(stream);
+ stream = tokenFilterFactory("PorterStem").create(stream);
+
+ assertTokenStreamContents(stream, new string[] {"test", "cat"});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testKeywordsCaseInsensitive() throws Exception
+ public virtual void testKeywordsCaseInsensitive()
+ {
+ Reader reader = new StringReader("testing DoGs");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("StemmerOverride", TEST_VERSION_CURRENT, new StringMockResourceLoader("dogs\tcat"), "dictionary", "stemdict.txt", "ignoreCase", "true").create(stream);
+ stream = tokenFilterFactory("PorterStem").create(stream);
+
+ assertTokenStreamContents(stream, new string[] {"test", "cat"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("StemmerOverride", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilter.cs
new file mode 100644
index 0000000..3d394c5
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilter.cs
@@ -0,0 +1,180 @@
+using System;
+using System.Collections.Generic;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using org.apache.lucene.analysis.tokenattributes;
+ using Version = org.apache.lucene.util.Version;
+
+ public class TestTrimFilter : BaseTokenStreamTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTrim() throws Exception
+ public virtual void testTrim()
+ {
+ char[] a = " a ".ToCharArray();
+ char[] b = "b ".ToCharArray();
+ char[] ccc = "cCc".ToCharArray();
+ char[] whitespace = " ".ToCharArray();
+ char[] empty = "".ToCharArray();
+
+ TokenStream ts = new IterTokenStream(new Token(a, 0, a.Length, 1, 5), new Token(b, 0, b.Length, 6, 10), new Token(ccc, 0, ccc.Length, 11, 15), new Token(whitespace, 0, whitespace.Length, 16, 20), new Token(empty, 0, empty.Length, 21, 21));
+ ts = new TrimFilter(TEST_VERSION_CURRENT, ts, false);
+
+ assertTokenStreamContents(ts, new string[] {"a", "b", "cCc", "", ""});
+
+ a = " a".ToCharArray();
+ b = "b ".ToCharArray();
+ ccc = " c ".ToCharArray();
+ whitespace = " ".ToCharArray();
+ ts = new IterTokenStream(new Token(a, 0, a.Length, 0, 2), new Token(b, 0, b.Length, 0, 2), new Token(ccc, 0, ccc.Length, 0, 3), new Token(whitespace, 0, whitespace.Length, 0, 3));
+ ts = new TrimFilter(Version.LUCENE_43, ts, true);
+
+ assertTokenStreamContents(ts, new string[] {"a", "b", "c", ""}, new int[] {1, 0, 1, 3}, new int[] {2, 1, 2, 3}, null, new int[] {1, 1, 1, 1}, null, null, false);
+ }
+
+ /// @deprecated (3.0) does not support custom attributes
+ [Obsolete("(3.0) does not support custom attributes")]
+ private class IterTokenStream : TokenStream
+ {
+ internal readonly Token[] tokens;
+ internal int index = 0;
+ internal CharTermAttribute termAtt = addAttribute(typeof(CharTermAttribute));
+ internal OffsetAttribute offsetAtt = addAttribute(typeof(OffsetAttribute));
+ internal PositionIncrementAttribute posIncAtt = addAttribute(typeof(PositionIncrementAttribute));
+ internal FlagsAttribute flagsAtt = addAttribute(typeof(FlagsAttribute));
+ internal TypeAttribute typeAtt = addAttribute(typeof(TypeAttribute));
+ internal PayloadAttribute payloadAtt = addAttribute(typeof(PayloadAttribute));
+
+ public IterTokenStream(params Token[] tokens) : base()
+ {
+ this.tokens = tokens;
+ }
+
+ public IterTokenStream(ICollection<Token> tokens) : this(tokens.toArray(new Token[tokens.Count]))
+ {
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
+ public override bool incrementToken()
+ {
+ if (index >= tokens.Length)
+ {
+ return false;
+ }
+ else
+ {
+ clearAttributes();
+ Token token = tokens[index++];
+ termAtt.setEmpty().append(token);
+ offsetAtt.setOffset(token.startOffset(), token.endOffset());
+ posIncAtt.PositionIncrement = token.PositionIncrement;
+ flagsAtt.Flags = token.Flags;
+ typeAtt.Type = token.type();
+ payloadAtt.Payload = token.Payload;
+ return true;
+ }
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper(this);
+ checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
+
+ Analyzer b = new AnalyzerAnonymousInnerClassHelper2(this);
+ checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER);
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly TestTrimFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper(TestTrimFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ return new TokenStreamComponents(tokenizer, new TrimFilter(Version.LUCENE_43, tokenizer, true));
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly TestTrimFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper2(TestTrimFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ return new TokenStreamComponents(tokenizer, new TrimFilter(TEST_VERSION_CURRENT, tokenizer, false));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws java.io.IOException
+ public virtual void testEmptyTerm()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper3(this);
+ checkOneTerm(a, "", "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly TestTrimFilter outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper3(TestTrimFilter outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final boolean updateOffsets = random().nextBoolean();
+ bool updateOffsets = random().nextBoolean();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.util.Version version = updateOffsets ? org.apache.lucene.util.Version.LUCENE_43 : TEST_VERSION_CURRENT;
+ Version version = updateOffsets ? Version.LUCENE_43 : TEST_VERSION_CURRENT;
+ return new TokenStreamComponents(tokenizer, new TrimFilter(version, tokenizer, updateOffsets));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilterFactory.cs
new file mode 100644
index 0000000..8d703bf
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTrimFilterFactory.cs
@@ -0,0 +1,57 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+ /// <summary>
+ /// Simple tests to ensure this factory is working
+ /// </summary>
+ public class TestTrimFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTrimming() throws Exception
+ public virtual void testTrimming()
+ {
+ Reader reader = new StringReader("trim me ");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ stream = tokenFilterFactory("Trim").create(stream);
+ assertTokenStreamContents(stream, new string[] {"trim me"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("Trim", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilter.cs
new file mode 100644
index 0000000..eb84659
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilter.cs
@@ -0,0 +1,47 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Test = org.junit.Test;
+
+ /// <summary>
+ /// Test the truncate token filter.
+ /// </summary>
+ public class TestTruncateTokenFilter : BaseTokenStreamTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTruncating() throws Exception
+ public virtual void testTruncating()
+ {
+ TokenStream stream = new MockTokenizer(new StringReader("abcdefg 1234567 ABCDEFG abcde abc 12345 123"), MockTokenizer.WHITESPACE, false);
+ stream = new TruncateTokenFilter(stream, 5);
+ assertTokenStreamContents(stream, new string[]{"abcde", "12345", "ABCDE", "abcde", "abc", "12345", "123"});
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test(expected = IllegalArgumentException.class) public void testNonPositiveLength() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testNonPositiveLength()
+ {
+ new TruncateTokenFilter(new MockTokenizer(new StringReader("length must be a positive number")), -48);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilterFactory.cs
new file mode 100644
index 0000000..9b647f0
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestTruncateTokenFilterFactory.cs
@@ -0,0 +1,81 @@
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+
+ /// <summary>
+ /// Simple tests to ensure the simple truncation filter factory is working.
+ /// </summary>
+ public class TestTruncateTokenFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+ /// <summary>
+ /// Ensure the filter actually truncates text.
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTruncating() throws Exception
+ public virtual void testTruncating()
+ {
+ Reader reader = new StringReader("abcdefg 1234567 ABCDEFG abcde abc 12345 123");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "5").create(stream);
+ assertTokenStreamContents(stream, new string[]{"abcde", "12345", "ABCDE", "abcde", "abc", "12345", "123"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "5", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameter(s):"));
+ }
+ }
+
+ /// <summary>
+ /// Test that negative prefix length result in exception
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNonPositivePrefixLengthArgument() throws Exception
+ public virtual void testNonPositivePrefixLengthArgument()
+ {
+ try
+ {
+ tokenFilterFactory("Truncate", TruncateTokenFilterFactory.PREFIX_LENGTH_KEY, "-5");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains(TruncateTokenFilterFactory.PREFIX_LENGTH_KEY + " parameter must be a positive number: -5"));
+ }
+ }
+ }
+
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestWordDelimiterFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestWordDelimiterFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestWordDelimiterFilter.cs
new file mode 100644
index 0000000..f646d02
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestWordDelimiterFilter.cs
@@ -0,0 +1,568 @@
+using System;
+using System.Collections.Generic;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace org.apache.lucene.analysis.miscellaneous
+{
+
+ using org.apache.lucene.analysis;
+ using TokenStreamComponents = org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using StopFilter = org.apache.lucene.analysis.core.StopFilter;
+ using CzechStemFilter = org.apache.lucene.analysis.cz.CzechStemFilter;
+ using StandardAnalyzer = org.apache.lucene.analysis.standard.StandardAnalyzer;
+ using PositionIncrementAttribute = org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ using CharArraySet = org.apache.lucene.analysis.util.CharArraySet;
+ using Test = org.junit.Test;
+
+
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.*;
+//JAVA TO C# CONVERTER TODO TASK: This Java 'import static' statement cannot be converted to C#:
+// import static org.apache.lucene.analysis.miscellaneous.WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE;
+
+ /// <summary>
+ /// New WordDelimiterFilter tests... most of the tests are in ConvertedLegacyTest
+ /// TODO: should explicitly test things like protWords and not rely on
+ /// the factory tests in Solr.
+ /// </summary>
+ public class TestWordDelimiterFilter : BaseTokenStreamTestCase
+ {
+
+ /// <summary>
+ ///*
+ /// public void testPerformance() throws IOException {
+ /// String s = "now is the time-for all good men to come to-the aid of their country.";
+ /// Token tok = new Token();
+ /// long start = System.currentTimeMillis();
+ /// int ret=0;
+ /// for (int i=0; i<1000000; i++) {
+ /// StringReader r = new StringReader(s);
+ /// TokenStream ts = new WhitespaceTokenizer(r);
+ /// ts = new WordDelimiterFilter(ts, 1,1,1,1,0);
+ ///
+ /// while (ts.next(tok) != null) ret++;
+ /// }
+ ///
+ /// System.out.println("ret="+ret+" time="+(System.currentTimeMillis()-start));
+ /// }
+ /// **
+ /// </summary>
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testOffsets() throws java.io.IOException
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testOffsets()
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ // test that subwords and catenated subwords have
+ // the correct offsets.
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("foo-bar", 5, 12)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"foo", "foobar", "bar"}, new int[] {5, 5, 9}, new int[] {8, 12, 12});
+
+ wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("foo-bar", 5, 6)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"foo", "bar", "foobar"}, new int[] {5, 5, 5}, new int[] {6, 6, 6});
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testOffsetChange() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testOffsetChange()
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("übelkeit)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"übelkeit"}, new int[] {7}, new int[] {15});
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testOffsetChange2() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testOffsetChange2()
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("(übelkeit", 7, 17)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"übelkeit"}, new int[] {8}, new int[] {17});
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testOffsetChange3() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testOffsetChange3()
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("(übelkeit", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"übelkeit"}, new int[] {8}, new int[] {16});
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testOffsetChange4() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testOffsetChange4()
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new SingleTokenTokenStream(new Token("(foo,bar)", 7, 16)), DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, new string[] {"foo", "foobar", "bar"}, new int[] {8, 8, 12}, new int[] {11, 15, 15});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void doSplit(final String input, String... output) throws Exception
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public virtual void doSplit(string input, params string[] output)
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new MockTokenizer(new StringReader(input), MockTokenizer.KEYWORD, false), WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, flags, null);
+
+ assertTokenStreamContents(wdf, output);
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testSplits() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testSplits()
+ {
+ doSplit("basic-split","basic","split");
+ doSplit("camelCase","camel","Case");
+
+ // non-space marking symbol shouldn't cause split
+ // this is an example in Thai
+ doSplit("\u0e1a\u0e49\u0e32\u0e19","\u0e1a\u0e49\u0e32\u0e19");
+ // possessive followed by delimiter
+ doSplit("test's'", "test");
+
+ // some russian upper and lowercase
+ doSplit("Роберт", "Роберт");
+ // now cause a split (russian camelCase)
+ doSplit("РобЕрт", "Роб", "Ерт");
+
+ // a composed titlecase character, don't split
+ doSplit("aDžungla", "aDžungla");
+
+ // a modifier letter, don't split
+ doSplit("ســـــــــــــــــلام", "ســـــــــــــــــلام");
+
+ // enclosing mark, don't split
+ doSplit("test⃝", "test⃝");
+
+ // combining spacing mark (the virama), don't split
+ doSplit("हिन्दी", "हिन्दी");
+
+ // don't split non-ascii digits
+ doSplit("١٢٣٤", "١٢٣٤");
+
+ // don't split supplementaries into unpaired surrogates
+ doSplit("𠀀𠀀", "𠀀𠀀");
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void doSplitPossessive(int stemPossessive, final String input, final String... output) throws Exception
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+ public virtual void doSplitPossessive(int stemPossessive, string input, params string[] output)
+ {
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
+ flags |= (stemPossessive == 1) ? STEM_ENGLISH_POSSESSIVE : 0;
+ WordDelimiterFilter wdf = new WordDelimiterFilter(TEST_VERSION_CURRENT, new MockTokenizer(new StringReader(input), MockTokenizer.KEYWORD, false), flags, null);
+
+ assertTokenStreamContents(wdf, output);
+ }
+
+ /*
+ * Test option that allows disabling the special "'s" stemming, instead treating the single quote like other delimiters.
+ */
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testPossessives() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testPossessives()
+ {
+ doSplitPossessive(1, "ra's", "ra");
+ doSplitPossessive(0, "ra's", "ra", "s");
+ }
+
+ /*
+ * Set a large position increment gap of 10 if the token is "largegap" or "/"
+ */
+ private sealed class LargePosIncTokenFilter : TokenFilter
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ internal CharTermAttribute termAtt = addAttribute(typeof(CharTermAttribute));
+ internal PositionIncrementAttribute posIncAtt = addAttribute(typeof(PositionIncrementAttribute));
+
+ protected internal LargePosIncTokenFilter(TestWordDelimiterFilter outerInstance, TokenStream input) : base(input)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
+ public override bool incrementToken()
+ {
+ if (input.incrementToken())
+ {
+ if (termAtt.ToString().Equals("largegap") || termAtt.ToString().Equals("/"))
+ {
+ posIncAtt.PositionIncrement = 10;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test public void testPositionIncrements() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testPositionIncrements()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.util.CharArraySet protWords = new org.apache.lucene.analysis.util.CharArraySet(TEST_VERSION_CURRENT, new HashSet<>(Arrays.asList("NUTCH")), false);
+ CharArraySet protWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<>("NUTCH"), false);
+
+ /* analyzer that uses whitespace + wdf */
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper(this, flags, protWords);
+
+ /* in this case, works as expected. */
+ assertAnalyzesTo(a, "LUCENE / SOLR", new string[] {"LUCENE", "SOLR"}, new int[] {0, 9}, new int[] {6, 13}, new int[] {1, 1});
+
+ /* only in this case, posInc of 2 ?! */
+ assertAnalyzesTo(a, "LUCENE / solR", new string[] {"LUCENE", "sol", "solR", "R"}, new int[] {0, 9, 9, 12}, new int[] {6, 12, 13, 13}, new int[] {1, 1, 0, 1});
+
+ assertAnalyzesTo(a, "LUCENE / NUTCH SOLR", new string[] {"LUCENE", "NUTCH", "SOLR"}, new int[] {0, 9, 15}, new int[] {6, 14, 19}, new int[] {1, 1, 1});
+
+ /* analyzer that will consume tokens with large position increments */
+ Analyzer a2 = new AnalyzerAnonymousInnerClassHelper2(this, flags, protWords);
+
+ /* increment of "largegap" is preserved */
+ assertAnalyzesTo(a2, "LUCENE largegap SOLR", new string[] {"LUCENE", "largegap", "SOLR"}, new int[] {0, 7, 16}, new int[] {6, 15, 20}, new int[] {1, 10, 1});
+
+ /* the "/" had a position increment of 10, where did it go?!?!! */
+ assertAnalyzesTo(a2, "LUCENE / SOLR", new string[] {"LUCENE", "SOLR"}, new int[] {0, 9}, new int[] {6, 13}, new int[] {1, 11});
+
+ /* in this case, the increment of 10 from the "/" is carried over */
+ assertAnalyzesTo(a2, "LUCENE / solR", new string[] {"LUCENE", "sol", "solR", "R"}, new int[] {0, 9, 9, 12}, new int[] {6, 12, 13, 13}, new int[] {1, 11, 0, 1});
+
+ assertAnalyzesTo(a2, "LUCENE / NUTCH SOLR", new string[] {"LUCENE", "NUTCH", "SOLR"}, new int[] {0, 9, 15}, new int[] {6, 14, 19}, new int[] {1, 11, 1});
+
+ Analyzer a3 = new AnalyzerAnonymousInnerClassHelper3(this, flags, protWords);
+
+ assertAnalyzesTo(a3, "lucene.solr", new string[] {"lucene", "lucenesolr", "solr"}, new int[] {0, 0, 7}, new int[] {6, 11, 11}, new int[] {1, 0, 1});
+
+ /* the stopword should add a gap here */
+ assertAnalyzesTo(a3, "the lucene.solr", new string[] {"lucene", "lucenesolr", "solr"}, new int[] {4, 4, 11}, new int[] {10, 15, 15}, new int[] {2, 0, 1});
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protWords;
+
+ public AnalyzerAnonymousInnerClassHelper(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protWords = protWords;
+ }
+
+ public override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protWords));
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protWords;
+
+ public AnalyzerAnonymousInnerClassHelper2(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protWords = protWords;
+ }
+
+ public override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, new LargePosIncTokenFilter(outerInstance, tokenizer), flags, protWords));
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protWords;
+
+ public AnalyzerAnonymousInnerClassHelper3(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protWords = protWords;
+ }
+
+ public override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ StopFilter filter = new StopFilter(TEST_VERSION_CURRENT, tokenizer, StandardAnalyzer.STOP_WORDS_SET);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, filter, flags, protWords));
+ }
+ }
+
+ /// <summary>
+ /// concat numbers + words + all </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLotsOfConcatenating() throws Exception
+ public virtual void testLotsOfConcatenating()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+
+ /* analyzer that uses whitespace + wdf */
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper4(this, flags);
+
+ assertAnalyzesTo(a, "abc-def-123-456", new string[] {"abc", "abcdef", "abcdef123456", "def", "123", "123456", "456"}, new int[] {0, 0, 0, 4, 8, 8, 12}, new int[] {3, 7, 15, 7, 11, 15, 15}, new int[] {1, 0, 0, 1, 1, 0, 1});
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper4 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+
+ public AnalyzerAnonymousInnerClassHelper4(TestWordDelimiterFilter outerInstance, int flags)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ }
+
+ public override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, null));
+ }
+ }
+
+ /// <summary>
+ /// concat numbers + words + all + preserve original </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testLotsOfConcatenating2() throws Exception
+ public virtual void testLotsOfConcatenating2()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = PRESERVE_ORIGINAL | GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+ int flags = PRESERVE_ORIGINAL | GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | CATENATE_WORDS | CATENATE_NUMBERS | CATENATE_ALL | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
+
+ /* analyzer that uses whitespace + wdf */
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper5(this, flags);
+
+ assertAnalyzesTo(a, "abc-def-123-456", new string[] {"abc-def-123-456", "abc", "abcdef", "abcdef123456", "def", "123", "123456", "456"}, new int[] {0, 0, 0, 0, 4, 8, 8, 12}, new int[] {15, 3, 7, 15, 7, 11, 15, 15}, new int[] {1, 0, 0, 0, 1, 1, 0, 1});
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper5 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+
+ public AnalyzerAnonymousInnerClassHelper5(TestWordDelimiterFilter outerInstance, int flags)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ }
+
+ public override TokenStreamComponents createComponents(string field, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, null));
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ int numIterations = atLeast(5);
+ for (int i = 0; i < numIterations; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = random().nextInt(512);
+ int flags = random().Next(512);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.util.CharArraySet protectedWords;
+ CharArraySet protectedWords;
+ if (random().nextBoolean())
+ {
+ protectedWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<>("a", "b", "cd"), false);
+ }
+ else
+ {
+ protectedWords = null;
+ }
+
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper6(this, flags, protectedWords);
+ checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper6 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protectedWords;
+
+ public AnalyzerAnonymousInnerClassHelper6(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protectedWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protectedWords = protectedWords;
+ }
+
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
+ }
+ }
+
+ /// <summary>
+ /// blast some enormous random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomHugeStrings() throws Exception
+ public virtual void testRandomHugeStrings()
+ {
+ int numIterations = atLeast(5);
+ for (int i = 0; i < numIterations; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = random().nextInt(512);
+ int flags = random().Next(512);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.util.CharArraySet protectedWords;
+ CharArraySet protectedWords;
+ if (random().nextBoolean())
+ {
+ protectedWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<>("a", "b", "cd"), false);
+ }
+ else
+ {
+ protectedWords = null;
+ }
+
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper7(this, flags, protectedWords);
+ checkRandomData(random(), a, 100 * RANDOM_MULTIPLIER, 8192);
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper7 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protectedWords;
+
+ public AnalyzerAnonymousInnerClassHelper7(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protectedWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protectedWords = protectedWords;
+ }
+
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws java.io.IOException
+ public virtual void testEmptyTerm()
+ {
+ Random random = random();
+ for (int i = 0; i < 512; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int flags = i;
+ int flags = i;
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.util.CharArraySet protectedWords;
+ CharArraySet protectedWords;
+ if (random.nextBoolean())
+ {
+ protectedWords = new CharArraySet(TEST_VERSION_CURRENT, new HashSet<>("a", "b", "cd"), false);
+ }
+ else
+ {
+ protectedWords = null;
+ }
+
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper8(this, flags, protectedWords);
+ // depending upon options, this thing may or may not preserve the empty term
+ checkAnalysisConsistency(random, a, random.nextBoolean(), "");
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper8 : Analyzer
+ {
+ private readonly TestWordDelimiterFilter outerInstance;
+
+ private int flags;
+ private CharArraySet protectedWords;
+
+ public AnalyzerAnonymousInnerClassHelper8(TestWordDelimiterFilter outerInstance, int flags, CharArraySet protectedWords)
+ {
+ this.outerInstance = outerInstance;
+ this.flags = flags;
+ this.protectedWords = protectedWords;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(TEST_VERSION_CURRENT, tokenizer, flags, protectedWords));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenFilterTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenFilterTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenFilterTest.cs
new file mode 100644
index 0000000..10ab5ec
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Ngram/EdgeNGramTokenFilterTest.cs
@@ -0,0 +1,407 @@
+using System;
+
+namespace org.apache.lucene.analysis.ngram
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using LetterTokenizer = org.apache.lucene.analysis.core.LetterTokenizer;
+ using WhitespaceTokenizer = org.apache.lucene.analysis.core.WhitespaceTokenizer;
+ using ASCIIFoldingFilter = org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+ using ShingleFilter = org.apache.lucene.analysis.shingle.ShingleFilter;
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ using OffsetAttribute = org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+ using PositionIncrementAttribute = org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+ using Version = org.apache.lucene.util.Version;
+ using TestUtil = org.apache.lucene.util.TestUtil;
+
+ /// <summary>
+ /// Tests <seealso cref="EdgeNGramTokenFilter"/> for correctness.
+ /// </summary>
+ public class EdgeNGramTokenFilterTest : BaseTokenStreamTestCase
+ {
+ private TokenStream input;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput() throws Exception
+ public virtual void testInvalidInput()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 0, 0);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput2() throws Exception
+ public virtual void testInvalidInput2()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 2, 1);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidInput3() throws Exception
+ public virtual void testInvalidInput3()
+ {
+ bool gotException = false;
+ try
+ {
+ new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, -1, 2);
+ }
+ catch (System.ArgumentException)
+ {
+ gotException = true;
+ }
+ assertTrue(gotException);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFrontUnigram() throws Exception
+ public virtual void testFrontUnigram()
+ {
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"a"}, new int[]{0}, new int[]{5});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBackUnigram() throws Exception
+ public virtual void testBackUnigram()
+ {
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(Version.LUCENE_43, input, EdgeNGramTokenFilter.Side.BACK, 1, 1);
+ assertTokenStreamContents(tokenizer, new string[]{"e"}, new int[]{4}, new int[]{5});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOversizedNgrams() throws Exception
+ public virtual void testOversizedNgrams()
+ {
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 6, 6);
+ assertTokenStreamContents(tokenizer, new string[0], new int[0], new int[0]);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFrontRangeOfNgrams() throws Exception
+ public virtual void testFrontRangeOfNgrams()
+ {
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{5,5,5});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBackRangeOfNgrams() throws Exception
+ public virtual void testBackRangeOfNgrams()
+ {
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(Version.LUCENE_43, input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"e","de","cde"}, new int[]{4,3,2}, new int[]{5,5,5}, null, null, null, null, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFilterPositions() throws Exception
+ public virtual void testFilterPositions()
+ {
+ TokenStream ts = new MockTokenizer(new StringReader("abcde vwxyz"), MockTokenizer.WHITESPACE, false);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, ts, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"a","ab","abc","v","vw","vwx"}, new int[]{0,0,0,6,6,6}, new int[]{5,5,5,11,11,11}, null, new int[]{1,0,0,1,0,0}, null, null, false);
+ }
+
+ private class PositionFilter : TokenFilter
+ {
+
+ internal readonly PositionIncrementAttribute posIncrAtt = addAttribute(typeof(PositionIncrementAttribute));
+ internal bool started;
+
+//JAVA TO C# CONVERTER WARNING: 'final' parameters are not available in .NET:
+//ORIGINAL LINE: PositionFilter(final org.apache.lucene.analysis.TokenStream input)
+ internal PositionFilter(TokenStream input) : base(input)
+ {
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public final boolean incrementToken() throws java.io.IOException
+ public override bool incrementToken()
+ {
+ if (outerInstance.input.incrementToken())
+ {
+ if (started)
+ {
+ posIncrAtt.PositionIncrement = 0;
+ }
+ else
+ {
+ started = true;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void reset() throws java.io.IOException
+ public override void reset()
+ {
+ base.reset();
+ started = false;
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFirstTokenPositionIncrement() throws Exception
+ public virtual void testFirstTokenPositionIncrement()
+ {
+ TokenStream ts = new MockTokenizer(new StringReader("a abc"), MockTokenizer.WHITESPACE, false);
+ ts = new PositionFilter(ts); // All but first token will get 0 position increment
+ EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, ts, EdgeNGramTokenFilter.Side.FRONT, 2, 3);
+ // The first token "a" will not be output, since it's smaller than the mingram size of 2.
+ // The second token on input to EdgeNGramTokenFilter will have position increment of 0,
+ // which should be increased to 1, since this is the first output token in the stream.
+ assertTokenStreamContents(filter, new string[] {"ab", "abc"}, new int[] {2, 2}, new int[] {5, 5}, new int[] {1, 0});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testSmallTokenInStream() throws Exception
+ public virtual void testSmallTokenInStream()
+ {
+ input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
+ assertTokenStreamContents(tokenizer, new string[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReset() throws Exception
+ public virtual void testReset()
+ {
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
+ EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
+ assertTokenStreamContents(filter, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{5,5,5});
+ tokenizer.Reader = new StringReader("abcde");
+ assertTokenStreamContents(filter, new string[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{5,5,5});
+ }
+
+ // LUCENE-3642
+ // EdgeNgram blindly adds term length to offset, but this can take things out of bounds
+ // wrt original text if a previous filter increases the length of the word (in this case æ -> ae)
+ // so in this case we behave like WDF, and preserve any modified offsets
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testInvalidOffsets() throws Exception
+ public virtual void testInvalidOffsets()
+ {
+ Analyzer analyzer = new AnalyzerAnonymousInnerClassHelper(this);
+ assertAnalyzesTo(analyzer, "mosfellsbær", new string[] {"mo", "mos", "mosf", "mosfe", "mosfel", "mosfell", "mosfells", "mosfellsb", "mosfellsba", "mosfellsbae", "mosfellsbaer"}, new int[] {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, new int[] {11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11});
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly EdgeNGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper(EdgeNGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
+ filters = new EdgeNGramTokenFilter(Version.LUCENE_43, filters, EdgeNGramTokenFilter.Side.FRONT, 2, 15);
+ return new TokenStreamComponents(tokenizer, filters);
+ }
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ for (int i = 0; i < 10; i++)
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int min = org.apache.lucene.util.TestUtil.nextInt(random(), 2, 10);
+ int min = TestUtil.Next(random(), 2, 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int max = org.apache.lucene.util.TestUtil.nextInt(random(), min, 20);
+ int max = TestUtil.Next(random(), min, 20);
+
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper2(this, min, max);
+ checkRandomData(random(), a, 100 * RANDOM_MULTIPLIER);
+ }
+
+ Analyzer b = new AnalyzerAnonymousInnerClassHelper3(this);
+ checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER, 20, false, false);
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly EdgeNGramTokenFilterTest outerInstance;
+
+ private int min;
+ private int max;
+
+ public AnalyzerAnonymousInnerClassHelper2(EdgeNGramTokenFilterTest outerInstance, int min, int max)
+ {
+ this.outerInstance = outerInstance;
+ this.min = min;
+ this.max = max;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, min, max));
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly EdgeNGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper3(EdgeNGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new EdgeNGramTokenFilter(Version.LUCENE_43, tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 4));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws Exception
+ public virtual void testEmptyTerm()
+ {
+ Random random = random();
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper4(this);
+ checkAnalysisConsistency(random, a, random.nextBoolean(), "");
+
+ Analyzer b = new AnalyzerAnonymousInnerClassHelper5(this);
+ checkAnalysisConsistency(random, b, random.nextBoolean(), "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper4 : Analyzer
+ {
+ private readonly EdgeNGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper4(EdgeNGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, EdgeNGramTokenFilter.Side.FRONT, 2, 15));
+ }
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper5 : Analyzer
+ {
+ private readonly EdgeNGramTokenFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper5(EdgeNGramTokenFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new EdgeNGramTokenFilter(Version.LUCENE_43, tokenizer, EdgeNGramTokenFilter.Side.BACK, 2, 15));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testGraphs() throws java.io.IOException
+ public virtual void testGraphs()
+ {
+ TokenStream tk = new LetterTokenizer(TEST_VERSION_CURRENT, new StringReader("abc d efgh ij klmno p q"));
+ tk = new ShingleFilter(tk);
+ tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, 7, 10);
+ assertTokenStreamContents(tk, new string[] {"efgh ij", "ij klmn", "ij klmno", "klmno p"}, new int[] {6,11,11,14}, new int[] {13,19,19,21}, new int[] {3,1,0,1}, new int[] {2,2,2,2}, 23);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testSupplementaryCharacters() throws java.io.IOException
+ public virtual void testSupplementaryCharacters()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final String s = org.apache.lucene.util.TestUtil.randomUnicodeString(random(), 10);
+ string s = TestUtil.randomUnicodeString(random(), 10);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int codePointCount = s.codePointCount(0, s.length());
+ int codePointCount = s.codePointCount(0, s.Length);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int minGram = org.apache.lucene.util.TestUtil.nextInt(random(), 1, 3);
+ int minGram = TestUtil.Next(random(), 1, 3);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int maxGram = org.apache.lucene.util.TestUtil.nextInt(random(), minGram, 10);
+ int maxGram = TestUtil.Next(random(), minGram, 10);
+ TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt = tk.addAttribute(org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class);
+ CharTermAttribute termAtt = tk.addAttribute(typeof(CharTermAttribute));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final org.apache.lucene.analysis.tokenattributes.OffsetAttribute offsetAtt = tk.addAttribute(org.apache.lucene.analysis.tokenattributes.OffsetAttribute.class);
+ OffsetAttribute offsetAtt = tk.addAttribute(typeof(OffsetAttribute));
+ tk.reset();
+ for (int i = minGram; i <= Math.Min(codePointCount, maxGram); ++i)
+ {
+ assertTrue(tk.incrementToken());
+ assertEquals(0, offsetAtt.startOffset());
+ assertEquals(s.Length, offsetAtt.endOffset());
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final int end = Character.offsetByCodePoints(s, 0, i);
+ int end = char.offsetByCodePoints(s, 0, i);
+ assertEquals(s.Substring(0, end), termAtt.ToString());
+ }
+ assertFalse(tk.incrementToken());
+ }
+
+ }
+
+}
\ No newline at end of file