You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2015/12/10 19:38:55 UTC
[06/27] lucenenet git commit: adding converted analysis common tests
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/ShingleFilterTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/ShingleFilterTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/ShingleFilterTest.cs
new file mode 100644
index 0000000..cf8311f
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/ShingleFilterTest.cs
@@ -0,0 +1,716 @@
+using System;
+
+namespace org.apache.lucene.analysis.shingle
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using KeywordTokenizer = org.apache.lucene.analysis.core.KeywordTokenizer;
+ using WhitespaceTokenizer = org.apache.lucene.analysis.core.WhitespaceTokenizer;
+ using org.apache.lucene.analysis.tokenattributes;
+
+ public class ShingleFilterTest : BaseTokenStreamTestCase
+ {
+
+ public static readonly Token[] TEST_TOKEN = new Token[] {createToken("please", 0, 6), createToken("divide", 7, 13), createToken("this", 14, 18), createToken("sentence", 19, 27), createToken("into", 28, 32), createToken("shingles", 33, 39)};
+
+ public static readonly int[] UNIGRAM_ONLY_POSITION_INCREMENTS = new int[] {1, 1, 1, 1, 1, 1};
+
+ public static readonly string[] UNIGRAM_ONLY_TYPES = new string[] {"word", "word", "word", "word", "word", "word"};
+
+ public static Token[] testTokenWithHoles;
+
+ public static readonly Token[] BI_GRAM_TOKENS = new Token[] {createToken("please", 0, 6), createToken("please divide", 0, 13), createToken("divide", 7, 13), createToken("divide this", 7, 18), createToken("this", 14, 18), createToken("this sentence", 14, 27), createToken("sentence", 19, 27), createToken("sentence into", 19, 32), createToken("into", 28, 32), createToken("into shingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] BI_GRAM_TYPES = new string[] {"word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_WITH_HOLES = new Token[] {createToken("please", 0, 6), createToken("please divide", 0, 13), createToken("divide", 7, 13), createToken("divide _", 7, 19), createToken("_ sentence", 19, 27), createToken("sentence", 19, 27), createToken("sentence _", 19, 33), createToken("_ shingles", 33, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES = new int[] {1, 0, 1, 0, 1, 1, 0, 1, 1};
+
+ private static readonly string[] BI_GRAM_TYPES_WITH_HOLES = new string[] {"word", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {createToken("please divide", 0, 13), createToken("divide this", 7, 18), createToken("this sentence", 14, 27), createToken("sentence into", 19, 32), createToken("into shingles", 28, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {1, 1, 1, 1, 1};
+
+ public static readonly string[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS = new Token[] {createToken("please divide", 0, 13), createToken("divide _", 7, 19), createToken("_ sentence", 19, 27), createToken("sentence _", 19, 33), createToken("_ shingles", 33, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS = new int[] {1, 1, 1, 1, 1, 1};
+
+
+ public static readonly Token[] TEST_SINGLE_TOKEN = new Token[] {createToken("please", 0, 6)};
+
+ public static readonly Token[] SINGLE_TOKEN = new Token[] {createToken("please", 0, 6)};
+
+ public static readonly int[] SINGLE_TOKEN_INCREMENTS = new int[] {1};
+
+ public static readonly string[] SINGLE_TOKEN_TYPES = new string[] {"word"};
+
+ public static readonly Token[] EMPTY_TOKEN_ARRAY = new Token[] { };
+
+ public static readonly int[] EMPTY_TOKEN_INCREMENTS_ARRAY = new int[] { };
+
+ public static readonly string[] EMPTY_TOKEN_TYPES_ARRAY = new string[] { };
+
+ public static readonly Token[] TRI_GRAM_TOKENS = new Token[] {createToken("please", 0, 6), createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("divide", 7, 13), createToken("divide this", 7, 18), createToken("divide this sentence", 7, 27), createToken("this", 14, 18), createToken("this sentence", 14, 27), createToken("this sentence into", 14, 32), createToken("sentence", 19, 27), createToken("sentence into", 19, 32), createToken("sentence into shingles", 19, 39), createToken("into", 28, 32), createToken("into shingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("divide this", 7, 18), createToken("divide this sentence", 7, 27), createToken("this sentence", 14, 27), createToken("this sentence into", 14, 32), createToken("sentence into", 19, 32), createToken("sentence into shingles", 19, 39), createToken("into shingles", 28, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS = new Token[] {createToken("please", 0, 6), createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("please divide this sentence", 0, 27), createToken("divide", 7, 13), createToken("divide this", 7, 18), createToken("divide this sentence", 7, 27), createToken("divide this sentence into", 7, 32), createToken("this", 14, 18), createToken("this sentence", 14, 27), createToken("this sentence into", 14, 32), createToken("this sentence into shingles", 14, 39), createToken("sentence", 19, 27), createToken("sentence into", 19, 32), createToken("sentence into shingles", 19, 39), createToken("into", 28, 32), createToken("into shingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS = new int[] {1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES = new string[] {"word", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("please divide this sentence", 0, 27), createToken("divide this", 7, 18), createToken("divide this sentence", 7, 27), createToken("divide this sentence into", 7, 32), createToken("this sentence", 14, 27), createToken("this sentence into", 14, 32), createToken("this sentence into shingles", 14, 39), createToken("sentence into", 19, 32), createToken("sentence into shingles", 19, 39), createToken("into shingles", 28, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_MIN_TRI_GRAM = new Token[] {createToken("please", 0, 6), createToken("please divide this", 0, 18), createToken("divide", 7, 13), createToken("divide this sentence", 7, 27), createToken("this", 14, 18), createToken("this sentence into", 14, 32), createToken("sentence", 19, 27), createToken("sentence into shingles", 19, 39), createToken("into", 28, 32), createToken("shingles", 33, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_MIN_TRI_GRAM = new string[] {"word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new Token[] {createToken("please divide this", 0, 18), createToken("divide this sentence", 7, 27), createToken("this sentence into", 14, 32), createToken("sentence into shingles", 19, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new int[] {1, 1, 1, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new string[] {"shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS_MIN_TRI_GRAM = new Token[] {createToken("please", 0, 6), createToken("please divide this", 0, 18), createToken("please divide this sentence", 0, 27), createToken("divide", 7, 13), createToken("divide this sentence", 7, 27), createToken("divide this sentence into", 7, 32), createToken("this", 14, 18), createToken("this sentence into", 14, 32), createToken("this sentence into shingles", 14, 39), createToken("sentence", 19, 27), createToken("sentence into shingles", 19, 39), createToken("into", 28, 32), createToken("shingles", 33, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES_MIN_TRI_GRAM = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word", "word"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new Token[] {createToken("please divide this", 0, 18), createToken("please divide this sentence", 0, 27), createToken("divide this sentence", 7, 27), createToken("divide this sentence into", 7, 32), createToken("this sentence into", 14, 32), createToken("this sentence into shingles", 14, 39), createToken("sentence into shingles", 19, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new int[] {1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS_MIN_FOUR_GRAM = new Token[] {createToken("please", 0, 6), createToken("please divide this sentence", 0, 27), createToken("divide", 7, 13), createToken("divide this sentence into", 7, 32), createToken("this", 14, 18), createToken("this sentence into shingles", 14, 39), createToken("sentence", 19, 27), createToken("into", 28, 32), createToken("shingles", 33, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM = new int[] {1, 0, 1, 0, 1, 0, 1, 1, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES_MIN_FOUR_GRAM = new string[] {"word", "shingle", "word", "shingle", "word", "shingle", "word", "word", "word"};
+
+ public static readonly Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new Token[] {createToken("please divide this sentence", 0, 27), createToken("divide this sentence into", 7, 32), createToken("this sentence into shingles", 14, 39)};
+
+ public static readonly int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new int[] {1, 1, 1};
+
+ public static readonly string[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new string[] {"shingle", "shingle", "shingle"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_NO_SEPARATOR = new Token[] {createToken("please", 0, 6), createToken("pleasedivide", 0, 13), createToken("divide", 7, 13), createToken("dividethis", 7, 18), createToken("this", 14, 18), createToken("thissentence", 14, 27), createToken("sentence", 19, 27), createToken("sentenceinto", 19, 32), createToken("into", 28, 32), createToken("intoshingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] BI_GRAM_TYPES_NO_SEPARATOR = new string[] {"word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new Token[] {createToken("pleasedivide", 0, 13), createToken("dividethis", 7, 18), createToken("thissentence", 14, 27), createToken("sentenceinto", 19, 32), createToken("intoshingles", 28, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new int[] {1, 1, 1, 1, 1};
+
+ public static readonly string[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_NO_SEPARATOR = new Token[] {createToken("please", 0, 6), createToken("pleasedivide", 0, 13), createToken("pleasedividethis", 0, 18), createToken("divide", 7, 13), createToken("dividethis", 7, 18), createToken("dividethissentence", 7, 27), createToken("this", 14, 18), createToken("thissentence", 14, 27), createToken("thissentenceinto", 14, 32), createToken("sentence", 19, 27), createToken("sentenceinto", 19, 32), createToken("sentenceintoshingles", 19, 39), createToken("into", 28, 32), createToken("intoshingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_NO_SEPARATOR = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new Token[] {createToken("pleasedivide", 0, 13), createToken("pleasedividethis", 0, 18), createToken("dividethis", 7, 18), createToken("dividethissentence", 7, 27), createToken("thissentence", 14, 27), createToken("thissentenceinto", 14, 32), createToken("sentenceinto", 19, 32), createToken("sentenceintoshingles", 19, 39), createToken("intoshingles", 28, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_ALT_SEPARATOR = new Token[] {createToken("please", 0, 6), createToken("please<SEP>divide", 0, 13), createToken("divide", 7, 13), createToken("divide<SEP>this", 7, 18), createToken("this", 14, 18), createToken("this<SEP>sentence", 14, 27), createToken("sentence", 19, 27), createToken("sentence<SEP>into", 19, 32), createToken("into", 28, 32), createToken("into<SEP>shingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] BI_GRAM_TYPES_ALT_SEPARATOR = new string[] {"word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new Token[] {createToken("please<SEP>divide", 0, 13), createToken("divide<SEP>this", 7, 18), createToken("this<SEP>sentence", 14, 27), createToken("sentence<SEP>into", 19, 32), createToken("into<SEP>shingles", 28, 39)};
+
+ public static readonly int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new int[] {1, 1, 1, 1, 1};
+
+ public static readonly string[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_ALT_SEPARATOR = new Token[] {createToken("please", 0, 6), createToken("please<SEP>divide", 0, 13), createToken("please<SEP>divide<SEP>this", 0, 18), createToken("divide", 7, 13), createToken("divide<SEP>this", 7, 18), createToken("divide<SEP>this<SEP>sentence", 7, 27), createToken("this", 14, 18), createToken("this<SEP>sentence", 14, 27), createToken("this<SEP>sentence<SEP>into", 14, 32), createToken("sentence", 19, 27), createToken("sentence<SEP>into", 19, 32), createToken("sentence<SEP>into<SEP>shingles", 19, 39), createToken("into", 28, 32), createToken("into<SEP>shingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_ALT_SEPARATOR = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new Token[] {createToken("please<SEP>divide", 0, 13), createToken("please<SEP>divide<SEP>this", 0, 18), createToken("divide<SEP>this", 7, 18), createToken("divide<SEP>this<SEP>sentence", 7, 27), createToken("this<SEP>sentence", 14, 27), createToken("this<SEP>sentence<SEP>into", 14, 32), createToken("sentence<SEP>into", 19, 32), createToken("sentence<SEP>into<SEP>shingles", 19, 39), createToken("into<SEP>shingles", 28, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_NULL_SEPARATOR = new Token[] {createToken("please", 0, 6), createToken("pleasedivide", 0, 13), createToken("pleasedividethis", 0, 18), createToken("divide", 7, 13), createToken("dividethis", 7, 18), createToken("dividethissentence", 7, 27), createToken("this", 14, 18), createToken("thissentence", 14, 27), createToken("thissentenceinto", 14, 32), createToken("sentence", 19, 27), createToken("sentenceinto", 19, 32), createToken("sentenceintoshingles", 19, 39), createToken("into", 28, 32), createToken("intoshingles", 28, 39), createToken("shingles", 33, 39)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_NULL_SEPARATOR = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TEST_TOKEN_POS_INCR_EQUAL_TO_N = new Token[] {createToken("please", 0, 6), createToken("divide", 7, 13), createToken("this", 14, 18), createToken("sentence", 29, 37, 3), createToken("into", 38, 42), createToken("shingles", 43, 49)};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N = new Token[] {createToken("please", 0, 6), createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("divide", 7, 13), createToken("divide this", 7, 18), createToken("divide this _", 7, 29), createToken("this", 14, 18), createToken("this _", 14, 29), createToken("this _ _", 14, 29), createToken("_ _ sentence", 29, 37), createToken("_ sentence", 29, 37), createToken("_ sentence into", 29, 42), createToken("sentence", 29, 37), createToken("sentence into", 29, 42), createToken("sentence into shingles", 29, 49), createToken("into", 38, 42), createToken("into shingles", 38, 49), createToken("shingles", 43, 49)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N = new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N = new string[] {"word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new Token[] {createToken("please divide", 0, 13), createToken("please divide this", 0, 18), createToken("divide this", 7, 18), createToken("divide this _", 7, 29), createToken("this _", 14, 29), createToken("this _ _", 14, 29), createToken("_ _ sentence", 29, 37), createToken("_ sentence", 29, 37), createToken("_ sentence into", 29, 42), createToken("sentence into", 29, 42), createToken("sentence into shingles", 29, 49), createToken("into shingles", 38, 49)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new int[] {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+ public static readonly Token[] TEST_TOKEN_POS_INCR_GREATER_THAN_N = new Token[] {createToken("please", 0, 6), createToken("divide", 57, 63, 8), createToken("this", 64, 68), createToken("sentence", 69, 77), createToken("into", 78, 82), createToken("shingles", 83, 89)};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N = new Token[] {createToken("please", 0, 6), createToken("please _", 0, 57), createToken("please _ _", 0, 57), createToken("_ _ divide", 57, 63), createToken("_ divide", 57, 63), createToken("_ divide this", 57, 68), createToken("divide", 57, 63), createToken("divide this", 57, 68), createToken("divide this sentence", 57, 77), createToken("this", 64, 68), createToken("this sentence", 64, 77), createToken("this sentence into", 64, 82), createToken("sentence", 69, 77), createToken("sentence into", 69, 82), createToken("sentence into shingles", 69, 89), createToken("into", 78, 82), createToken("into shingles", 78, 89), createToken("shingles", 83, 89)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N = new int[] {1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1};
+ public static readonly string[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N = new string[] {"word", "shingle", "shingle", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word"};
+
+ public static readonly Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new Token[] {createToken("please _", 0, 57), createToken("please _ _", 0, 57), createToken("_ _ divide", 57, 63), createToken("_ divide", 57, 63), createToken("_ divide this", 57, 68), createToken("divide this", 57, 68), createToken("divide this sentence", 57, 77), createToken("this sentence", 64, 77), createToken("this sentence into", 64, 82), createToken("sentence into", 69, 82), createToken("sentence into shingles", 69, 89), createToken("into shingles", 78, 89)};
+
+ public static readonly int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new int[] {1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+
+ public static readonly string[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new string[] {"shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle"};
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ testTokenWithHoles = new Token[] {createToken("please", 0, 6), createToken("divide", 7, 13), createToken("sentence", 19, 27, 2), createToken("shingles", 33, 39, 2)};
+ }
+
+ /*
+ * Class under test for void ShingleFilter(TokenStream, int)
+ */
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilter() throws java.io.IOException
+ public virtual void testBiGramFilter()
+ {
+ this.shingleFilterTest(2, TEST_TOKEN, BI_GRAM_TOKENS, BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithHoles() throws java.io.IOException
+ public virtual void testBiGramFilterWithHoles()
+ {
+ this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES, BI_GRAM_POSITION_INCREMENTS_WITH_HOLES, BI_GRAM_TYPES_WITH_HOLES, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithoutUnigrams() throws java.io.IOException
+ public virtual void testBiGramFilterWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS, BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithHolesWithoutUnigrams() throws java.io.IOException
+ public virtual void testBiGramFilterWithHolesWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS, BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithSingleToken() throws java.io.IOException
+ public virtual void testBiGramFilterWithSingleToken()
+ {
+ this.shingleFilterTest(2, TEST_SINGLE_TOKEN, SINGLE_TOKEN, SINGLE_TOKEN_INCREMENTS, SINGLE_TOKEN_TYPES, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithSingleTokenWithoutUnigrams() throws java.io.IOException
+ public virtual void testBiGramFilterWithSingleTokenWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, TEST_SINGLE_TOKEN, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithEmptyTokenStream() throws java.io.IOException
+ public virtual void testBiGramFilterWithEmptyTokenStream()
+ {
+ this.shingleFilterTest(2, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithEmptyTokenStreamWithoutUnigrams() throws java.io.IOException
+ public virtual void testBiGramFilterWithEmptyTokenStreamWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilter() throws java.io.IOException
+ public virtual void testTriGramFilter()
+ {
+ this.shingleFilterTest(3, TEST_TOKEN, TRI_GRAM_TOKENS, TRI_GRAM_POSITION_INCREMENTS, TRI_GRAM_TYPES, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterWithoutUnigrams() throws java.io.IOException
+ public virtual void testTriGramFilterWithoutUnigrams()
+ {
+ this.shingleFilterTest(3, TEST_TOKEN, TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS, TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, TRI_GRAM_TYPES_WITHOUT_UNIGRAMS, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilter() throws java.io.IOException
+ public virtual void testFourGramFilter()
+ {
+ this.shingleFilterTest(4, TEST_TOKEN, FOUR_GRAM_TOKENS, FOUR_GRAM_POSITION_INCREMENTS, FOUR_GRAM_TYPES, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilterWithoutUnigrams() throws java.io.IOException
+ public virtual void testFourGramFilterWithoutUnigrams()
+ {
+ this.shingleFilterTest(4, TEST_TOKEN, FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS, FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS, false);
+ }
+
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterMinTriGram() throws java.io.IOException
+ public virtual void testTriGramFilterMinTriGram()
+ {
+ this.shingleFilterTest(3, 3, TEST_TOKEN, TRI_GRAM_TOKENS_MIN_TRI_GRAM, TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM, TRI_GRAM_TYPES_MIN_TRI_GRAM, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterWithoutUnigramsMinTriGram() throws java.io.IOException
+ public virtual void testTriGramFilterWithoutUnigramsMinTriGram()
+ {
+ this.shingleFilterTest(3, 3, TEST_TOKEN, TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilterMinTriGram() throws java.io.IOException
+ public virtual void testFourGramFilterMinTriGram()
+ {
+ this.shingleFilterTest(3, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_MIN_TRI_GRAM, FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM, FOUR_GRAM_TYPES_MIN_TRI_GRAM, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilterWithoutUnigramsMinTriGram() throws java.io.IOException
+ public virtual void testFourGramFilterWithoutUnigramsMinTriGram()
+ {
+ this.shingleFilterTest(3, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilterMinFourGram() throws java.io.IOException
+ public virtual void testFourGramFilterMinFourGram()
+ {
+ this.shingleFilterTest(4, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_MIN_FOUR_GRAM, FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM, FOUR_GRAM_TYPES_MIN_FOUR_GRAM, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testFourGramFilterWithoutUnigramsMinFourGram() throws java.io.IOException
+ public virtual void testFourGramFilterWithoutUnigramsMinFourGram()
+ {
+ this.shingleFilterTest(4, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterNoSeparator() throws java.io.IOException
+ public virtual void testBiGramFilterNoSeparator()
+ {
+ this.shingleFilterTest("", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_NO_SEPARATOR, BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR, BI_GRAM_TYPES_NO_SEPARATOR, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithoutUnigramsNoSeparator() throws java.io.IOException
+ public virtual void testBiGramFilterWithoutUnigramsNoSeparator()
+ {
+ this.shingleFilterTest("", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR, BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR, BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR, false);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterNoSeparator() throws java.io.IOException
+ public virtual void testTriGramFilterNoSeparator()
+ {
+ this.shingleFilterTest("", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_NO_SEPARATOR, TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR, TRI_GRAM_TYPES_NO_SEPARATOR, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterWithoutUnigramsNoSeparator() throws java.io.IOException
+ public virtual void testTriGramFilterWithoutUnigramsNoSeparator()
+ {
+ this.shingleFilterTest("", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR, TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR, TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterAltSeparator() throws java.io.IOException
+ public virtual void testBiGramFilterAltSeparator()
+ {
+ this.shingleFilterTest("<SEP>", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_ALT_SEPARATOR, BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR, BI_GRAM_TYPES_ALT_SEPARATOR, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBiGramFilterWithoutUnigramsAltSeparator() throws java.io.IOException
+ public virtual void testBiGramFilterWithoutUnigramsAltSeparator()
+ {
+ this.shingleFilterTest("<SEP>", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR, false);
+ }
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterAltSeparator() throws java.io.IOException
+ public virtual void testTriGramFilterAltSeparator()
+ {
+ this.shingleFilterTest("<SEP>", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_ALT_SEPARATOR, TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR, TRI_GRAM_TYPES_ALT_SEPARATOR, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterWithoutUnigramsAltSeparator() throws java.io.IOException
+ public virtual void testTriGramFilterWithoutUnigramsAltSeparator()
+ {
+ this.shingleFilterTest("<SEP>", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTriGramFilterNullSeparator() throws java.io.IOException
+ public virtual void testTriGramFilterNullSeparator()
+ {
+ this.shingleFilterTest(null, 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_NULL_SEPARATOR, TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR, TRI_GRAM_TYPES_NULL_SEPARATOR, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPositionIncrementEqualToN() throws java.io.IOException
+ public virtual void testPositionIncrementEqualToN()
+ {
+ this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N, TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N, TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPositionIncrementEqualToNWithoutUnigrams() throws java.io.IOException
+ public virtual void testPositionIncrementEqualToNWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, false);
+ }
+
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPositionIncrementGreaterThanN() throws java.io.IOException
+ public virtual void testPositionIncrementGreaterThanN()
+ {
+ this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N, TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N, TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testPositionIncrementGreaterThanNWithoutUnigrams() throws java.io.IOException
+ public virtual void testPositionIncrementGreaterThanNWithoutUnigrams()
+ {
+ this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, false);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testReset() throws Exception
+ public virtual void testReset()
+ {
+ Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence"));
+ TokenStream filter = new ShingleFilter(wsTokenizer, 2);
+ assertTokenStreamContents(filter, new string[]{"please","please divide","divide","divide this","this","this sentence","sentence"}, new int[]{0,0,7,7,14,14,19}, new int[]{6,13,13,18,18,27,27}, new string[]{TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE}, new int[]{1,0,1,0,1,0,1});
+ wsTokenizer.Reader = new StringReader("please divide this sentence");
+ assertTokenStreamContents(filter, new string[]{"please","please divide","divide","divide this","this","this sentence","sentence"}, new int[]{0,0,7,7,14,14,19}, new int[]{6,13,13,18,18,27,27}, new string[]{TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE,"shingle",TypeAttribute.DEFAULT_TYPE}, new int[]{1,0,1,0,1,0,1});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOutputUnigramsIfNoShinglesSingleTokenCase() throws java.io.IOException
+ public virtual void testOutputUnigramsIfNoShinglesSingleTokenCase()
+ {
+ // Single token input with outputUnigrams==false is the primary case where
+ // enabling this option should alter program behavior.
+ this.shingleFilterTest(2, 2, TEST_SINGLE_TOKEN, SINGLE_TOKEN, SINGLE_TOKEN_INCREMENTS, SINGLE_TOKEN_TYPES, false, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOutputUnigramsIfNoShinglesWithSimpleBigram() throws java.io.IOException
+ public virtual void testOutputUnigramsIfNoShinglesWithSimpleBigram()
+ {
+ // Here we expect the same result as with testBiGramFilter().
+ this.shingleFilterTest(2, 2, TEST_TOKEN, BI_GRAM_TOKENS, BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES, true, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOutputUnigramsIfNoShinglesWithSimpleUnigramlessBigram() throws java.io.IOException
+ public virtual void testOutputUnigramsIfNoShinglesWithSimpleUnigramlessBigram()
+ {
+ // Here we expect the same result as with testBiGramFilterWithoutUnigrams().
+ this.shingleFilterTest(2, 2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS, BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS, false, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOutputUnigramsIfNoShinglesWithMultipleInputTokens() throws java.io.IOException
+ public virtual void testOutputUnigramsIfNoShinglesWithMultipleInputTokens()
+ {
+ // Test when the minimum shingle size is greater than the number of input tokens
+ this.shingleFilterTest(7, 7, TEST_TOKEN, TEST_TOKEN, UNIGRAM_ONLY_POSITION_INCREMENTS, UNIGRAM_ONLY_TYPES, false, true);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: protected void shingleFilterTest(int maxSize, org.apache.lucene.analysis.Token[] tokensToShingle, org.apache.lucene.analysis.Token[] tokensToCompare, int[] positionIncrements, String[] types, boolean outputUnigrams) throws java.io.IOException
+ protected internal virtual void shingleFilterTest(int maxSize, Token[] tokensToShingle, Token[] tokensToCompare, int[] positionIncrements, string[] types, bool outputUnigrams)
+ {
+
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), maxSize);
+ filter.OutputUnigrams = outputUnigrams;
+ shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: protected void shingleFilterTest(int minSize, int maxSize, org.apache.lucene.analysis.Token[] tokensToShingle, org.apache.lucene.analysis.Token[] tokensToCompare, int[] positionIncrements, String[] types, boolean outputUnigrams) throws java.io.IOException
+ protected internal virtual void shingleFilterTest(int minSize, int maxSize, Token[] tokensToShingle, Token[] tokensToCompare, int[] positionIncrements, string[] types, bool outputUnigrams)
+ {
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize);
+ filter.OutputUnigrams = outputUnigrams;
+ shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: protected void shingleFilterTest(int minSize, int maxSize, org.apache.lucene.analysis.Token[] tokensToShingle, org.apache.lucene.analysis.Token[] tokensToCompare, int[] positionIncrements, String[] types, boolean outputUnigrams, boolean outputUnigramsIfNoShingles) throws java.io.IOException
+ protected internal virtual void shingleFilterTest(int minSize, int maxSize, Token[] tokensToShingle, Token[] tokensToCompare, int[] positionIncrements, string[] types, bool outputUnigrams, bool outputUnigramsIfNoShingles)
+ {
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize);
+ filter.OutputUnigrams = outputUnigrams;
+ filter.OutputUnigramsIfNoShingles = outputUnigramsIfNoShingles;
+ shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: protected void shingleFilterTest(String tokenSeparator, int minSize, int maxSize, org.apache.lucene.analysis.Token[] tokensToShingle, org.apache.lucene.analysis.Token[] tokensToCompare, int[] positionIncrements, String[] types, boolean outputUnigrams) throws java.io.IOException
+ protected internal virtual void shingleFilterTest(string tokenSeparator, int minSize, int maxSize, Token[] tokensToShingle, Token[] tokensToCompare, int[] positionIncrements, string[] types, bool outputUnigrams)
+ {
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize);
+ filter.TokenSeparator = tokenSeparator;
+ filter.OutputUnigrams = outputUnigrams;
+ shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: protected void shingleFilterTestCommon(ShingleFilter filter, org.apache.lucene.analysis.Token[] tokensToCompare, int[] positionIncrements, String[] types) throws java.io.IOException
+ protected internal virtual void shingleFilterTestCommon(ShingleFilter filter, Token[] tokensToCompare, int[] positionIncrements, string[] types)
+ {
+ string[] text = new string[tokensToCompare.Length];
+ int[] startOffsets = new int[tokensToCompare.Length];
+ int[] endOffsets = new int[tokensToCompare.Length];
+
+ for (int i = 0; i < tokensToCompare.Length; i++)
+ {
+ text[i] = new string(tokensToCompare[i].buffer(),0, tokensToCompare[i].length());
+ startOffsets[i] = tokensToCompare[i].startOffset();
+ endOffsets[i] = tokensToCompare[i].endOffset();
+ }
+
+ assertTokenStreamContents(filter, text, startOffsets, endOffsets, types, positionIncrements);
+ }
+
+ private static Token createToken(string term, int start, int offset)
+ {
+ return createToken(term, start, offset, 1);
+ }
+
+ private static Token createToken(string term, int start, int offset, int positionIncrement)
+ {
+ Token token = new Token(start, offset);
+ token.copyBuffer(term.ToCharArray(), 0, term.Length);
+ token.PositionIncrement = positionIncrement;
+ return token;
+ }
+
+ /// <summary>
+ /// blast some random strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomStrings() throws Exception
+ public virtual void testRandomStrings()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper(this);
+ checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper : Analyzer
+ {
+ private readonly ShingleFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper(ShingleFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
+ }
+ }
+
+ /// <summary>
+ /// blast some random large strings through the analyzer </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testRandomHugeStrings() throws Exception
+ public virtual void testRandomHugeStrings()
+ {
+ Random random = random();
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper2(this);
+ checkRandomData(random, a, 100 * RANDOM_MULTIPLIER, 8192);
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper2 : Analyzer
+ {
+ private readonly ShingleFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper2(ShingleFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTerm() throws java.io.IOException
+ public virtual void testEmptyTerm()
+ {
+ Analyzer a = new AnalyzerAnonymousInnerClassHelper3(this);
+ checkOneTerm(a, "", "");
+ }
+
+ private class AnalyzerAnonymousInnerClassHelper3 : Analyzer
+ {
+ private readonly ShingleFilterTest outerInstance;
+
+ public AnalyzerAnonymousInnerClassHelper3(ShingleFilterTest outerInstance)
+ {
+ this.outerInstance = outerInstance;
+ }
+
+ protected internal override TokenStreamComponents createComponents(string fieldName, Reader reader)
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(reader);
+ return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
+ }
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTrailingHole1() throws java.io.IOException
+ public virtual void testTrailingHole1()
+ {
+ // Analyzing "wizard of", where of is removed as a
+ // stopword leaving a trailing hole:
+ Token[] inputTokens = new Token[] {createToken("wizard", 0, 6)};
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(1, 9, inputTokens), 2, 2);
+
+ assertTokenStreamContents(filter, new string[] {"wizard", "wizard _"}, new int[] {0, 0}, new int[] {6, 9}, new int[] {1, 0}, 9);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTrailingHole2() throws java.io.IOException
+ public virtual void testTrailingHole2()
+ {
+ // Analyzing "purple wizard of", where of is removed as a
+ // stopword leaving a trailing hole:
+ Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)};
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(1, 16, inputTokens), 2, 2);
+
+ assertTokenStreamContents(filter, new string[] {"purple", "purple wizard", "wizard", "wizard _"}, new int[] {0, 0, 7, 7}, new int[] {6, 13, 13, 16}, new int[] {1, 0, 1, 0}, 16);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTwoTrailingHoles() throws java.io.IOException
+ public virtual void testTwoTrailingHoles()
+ {
+ // Analyzing "purple wizard of the", where of and the are removed as a
+ // stopwords, leaving two trailing holes:
+ Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)};
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 2);
+
+ assertTokenStreamContents(filter, new string[] {"purple", "purple wizard", "wizard", "wizard _"}, new int[] {0, 0, 7, 7}, new int[] {6, 13, 13, 20}, new int[] {1, 0, 1, 0}, 20);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTwoTrailingHolesTriShingle() throws java.io.IOException
+ public virtual void testTwoTrailingHolesTriShingle()
+ {
+ // Analyzing "purple wizard of the", where of and the are removed as a
+ // stopwords, leaving two trailing holes:
+ Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)};
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3);
+
+ assertTokenStreamContents(filter, new string[] {"purple", "purple wizard", "purple wizard _", "wizard", "wizard _", "wizard _ _"}, new int[] {0, 0, 0, 7, 7, 7}, new int[] {6, 13, 20, 13, 20, 20}, new int[] {1, 0, 0, 1, 0, 0}, 20);
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTwoTrailingHolesTriShingleWithTokenFiller() throws java.io.IOException
+ public virtual void testTwoTrailingHolesTriShingleWithTokenFiller()
+ {
+ // Analyzing "purple wizard of the", where of and the are removed as a
+ // stopwords, leaving two trailing holes:
+ Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)};
+ ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3);
+ filter.FillerToken = "--";
+
+ assertTokenStreamContents(filter, new string[]{"purple", "purple wizard", "purple wizard --", "wizard", "wizard --", "wizard -- --"}, new int[]{0, 0, 0, 7, 7, 7}, new int[]{6, 13, 20, 13, 20, 20}, new int[]{1, 0, 0, 1, 0, 0}, 20);
+
+ filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3);
+ filter.FillerToken = "";
+
+ assertTokenStreamContents(filter, new string[]{"purple", "purple wizard", "purple wizard ", "wizard", "wizard ", "wizard "}, new int[]{0, 0, 0, 7, 7, 7}, new int[]{6, 13, 20, 13, 20, 20}, new int[]{1, 0, 0, 1, 0, 0}, 20);
+
+
+ filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3);
+ filter.FillerToken = null;
+
+ assertTokenStreamContents(filter, new string[] {"purple", "purple wizard", "purple wizard ", "wizard", "wizard ", "wizard "}, new int[] {0, 0, 0, 7, 7, 7}, new int[] {6, 13, 20, 13, 20, 20}, new int[] {1, 0, 0, 1, 0, 0}, 20);
+
+
+ filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3);
+ filter.FillerToken = null;
+ filter.TokenSeparator = null;
+
+ assertTokenStreamContents(filter, new string[] {"purple", "purplewizard", "purplewizard", "wizard", "wizard", "wizard"}, new int[] {0, 0, 0, 7, 7, 7}, new int[] {6, 13, 20, 13, 20, 20}, new int[] {1, 0, 0, 1, 0, 0}, 20);
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/TestShingleFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/TestShingleFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/TestShingleFilterFactory.cs
new file mode 100644
index 0000000..36c2928
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Shingle/TestShingleFilterFactory.cs
@@ -0,0 +1,224 @@
+namespace org.apache.lucene.analysis.shingle
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using BaseTokenStreamFactoryTestCase = org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+
+ /// <summary>
+ /// Simple tests to ensure the Shingle filter factory works.
+ /// </summary>
+ public class TestShingleFilterFactory : BaseTokenStreamFactoryTestCase
+ {
+ /// <summary>
+ /// Test the defaults
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testDefaults() throws Exception
+ public virtual void testDefaults()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this is", "is", "is a", "a", "a test", "test"});
+ }
+
+ /// <summary>
+ /// Test with unigrams disabled
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testNoUnigrams() throws Exception
+ public virtual void testNoUnigrams()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "outputUnigrams", "false").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this is", "is a", "a test"});
+ }
+
+ /// <summary>
+ /// Test with a higher max shingle size
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMaxShingleSize() throws Exception
+ public virtual void testMaxShingleSize()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "maxShingleSize", "3").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this is", "this is a", "is", "is a", "is a test", "a", "a test", "test"});
+ }
+
+ /// <summary>
+ /// Test with higher min (and max) shingle size
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMinShingleSize() throws Exception
+ public virtual void testMinShingleSize()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "4").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this is a", "this is a test", "is", "is a test", "a", "test"});
+ }
+
+ /// <summary>
+ /// Test with higher min (and max) shingle size and with unigrams disabled
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMinShingleSizeNoUnigrams() throws Exception
+ public virtual void testMinShingleSizeNoUnigrams()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "4", "outputUnigrams", "false").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this is a", "this is a test", "is a test"});
+ }
+
+ /// <summary>
+ /// Test with higher same min and max shingle size
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEqualMinAndMaxShingleSize() throws Exception
+ public virtual void testEqualMinAndMaxShingleSize()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "3").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this is a", "is", "is a test", "a", "test"});
+ }
+
+ /// <summary>
+ /// Test with higher same min and max shingle size and with unigrams disabled
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEqualMinAndMaxShingleSizeNoUnigrams() throws Exception
+ public virtual void testEqualMinAndMaxShingleSizeNoUnigrams()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "3", "outputUnigrams", "false").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this is a", "is a test"});
+ }
+
+ /// <summary>
+ /// Test with a non-default token separator
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTokenSeparator() throws Exception
+ public virtual void testTokenSeparator()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "tokenSeparator", "=BLAH=").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this=BLAH=is", "is", "is=BLAH=a", "a", "a=BLAH=test", "test"});
+ }
+
+ /// <summary>
+ /// Test with a non-default token separator and with unigrams disabled
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testTokenSeparatorNoUnigrams() throws Exception
+ public virtual void testTokenSeparatorNoUnigrams()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "tokenSeparator", "=BLAH=", "outputUnigrams", "false").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this=BLAH=is", "is=BLAH=a", "a=BLAH=test"});
+ }
+
+ /// <summary>
+ /// Test with an empty token separator
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEmptyTokenSeparator() throws Exception
+ public virtual void testEmptyTokenSeparator()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "tokenSeparator", "").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "thisis", "is", "isa", "a", "atest", "test"});
+ }
+
+ /// <summary>
+ /// Test with higher min (and max) shingle size
+ /// and with a non-default token separator
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMinShingleSizeAndTokenSeparator() throws Exception
+ public virtual void testMinShingleSizeAndTokenSeparator()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "4", "tokenSeparator", "=BLAH=").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this", "this=BLAH=is=BLAH=a", "this=BLAH=is=BLAH=a=BLAH=test", "is", "is=BLAH=a=BLAH=test", "a", "test"});
+ }
+
+ /// <summary>
+ /// Test with higher min (and max) shingle size
+ /// and with a non-default token separator
+ /// and with unigrams disabled
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMinShingleSizeAndTokenSeparatorNoUnigrams() throws Exception
+ public virtual void testMinShingleSizeAndTokenSeparatorNoUnigrams()
+ {
+ Reader reader = new StringReader("this is a test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "minShingleSize", "3", "maxShingleSize", "4", "tokenSeparator", "=BLAH=", "outputUnigrams", "false").create(stream);
+ assertTokenStreamContents(stream, new string[] {"this=BLAH=is=BLAH=a", "this=BLAH=is=BLAH=a=BLAH=test", "is=BLAH=a=BLAH=test"});
+ }
+
+ /// <summary>
+ /// Test with unigrams disabled except when there are no shingles, with
+ /// a single input token. Using default min/max shingle sizes: 2/2. No
+ /// shingles will be created, since there are fewer input tokens than
+ /// min shingle size. However, because outputUnigramsIfNoShingles is
+ /// set to true, even though outputUnigrams is set to false, one
+ /// unigram should be output.
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testOutputUnigramsIfNoShingles() throws Exception
+ public virtual void testOutputUnigramsIfNoShingles()
+ {
+ Reader reader = new StringReader("test");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("Shingle", "outputUnigrams", "false", "outputUnigramsIfNoShingles", "true").create(stream);
+ assertTokenStreamContents(stream, new string[] {"test"});
+ }
+
+ /// <summary>
+ /// Test that bogus arguments result in exception </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testBogusArguments() throws Exception
+ public virtual void testBogusArguments()
+ {
+ try
+ {
+ tokenFilterFactory("Shingle", "bogusArg", "bogusValue");
+ fail();
+ }
+ catch (System.ArgumentException expected)
+ {
+ assertTrue(expected.Message.contains("Unknown parameters"));
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/DateRecognizerSinkTokenizerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/DateRecognizerSinkTokenizerTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/DateRecognizerSinkTokenizerTest.cs
new file mode 100644
index 0000000..a23933c
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/DateRecognizerSinkTokenizerTest.cs
@@ -0,0 +1,52 @@
+namespace org.apache.lucene.analysis.sinks
+{
+
+ /// <summary>
+ /// Copyright 2004 The Apache Software Foundation
+ ///
+ /// Licensed under the Apache License, Version 2.0 (the "License");
+ /// you may not use this file except in compliance with the License.
+ /// You may obtain a copy of the License at
+ ///
+ /// http://www.apache.org/licenses/LICENSE-2.0
+ ///
+ /// Unless required by applicable law or agreed to in writing, software
+ /// distributed under the License is distributed on an "AS IS" BASIS,
+ /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ /// See the License for the specific language governing permissions and
+ /// limitations under the License.
+ /// </summary>
+
+
+
+ public class DateRecognizerSinkTokenizerTest : BaseTokenStreamTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void test() throws java.io.IOException
+ public virtual void test()
+ {
+ DateRecognizerSinkFilter sinkFilter = new DateRecognizerSinkFilter(new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT));
+ string test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006 The dogs finally reacted on 7/12/2006";
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter.SinkTokenStream sink = tee.newSinkTokenStream(sinkFilter);
+ int count = 0;
+
+ tee.reset();
+ while (tee.incrementToken())
+ {
+ count++;
+ }
+ assertTrue(count + " does not equal: " + 18, count == 18);
+
+ int sinkCount = 0;
+ sink.reset();
+ while (sink.incrementToken())
+ {
+ sinkCount++;
+ }
+ assertTrue("sink Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TestTeeSinkTokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TestTeeSinkTokenFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TestTeeSinkTokenFilter.cs
new file mode 100644
index 0000000..eb161cd
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TestTeeSinkTokenFilter.cs
@@ -0,0 +1,357 @@
+using System;
+using System.Text;
+
+namespace org.apache.lucene.analysis.sinks
+{
+
+ /// <summary>
+ /// Copyright 2004 The Apache Software Foundation
+ ///
+ /// Licensed under the Apache License, Version 2.0 (the "License");
+ /// you may not use this file except in compliance with the License.
+ /// You may obtain a copy of the License at
+ ///
+ /// http://www.apache.org/licenses/LICENSE-2.0
+ ///
+ /// Unless required by applicable law or agreed to in writing, software
+ /// distributed under the License is distributed on an "AS IS" BASIS,
+ /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ /// See the License for the specific language governing permissions and
+ /// limitations under the License.
+ /// </summary>
+
+
+ using org.apache.lucene.analysis;
+ using LowerCaseFilter = org.apache.lucene.analysis.core.LowerCaseFilter;
+ using StandardFilter = org.apache.lucene.analysis.standard.StandardFilter;
+ using StandardTokenizer = org.apache.lucene.analysis.standard.StandardTokenizer;
+ using CharTermAttribute = org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+ using PositionIncrementAttribute = org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+ using Document = org.apache.lucene.document.Document;
+ using Field = org.apache.lucene.document.Field;
+ using FieldType = org.apache.lucene.document.FieldType;
+ using TextField = org.apache.lucene.document.TextField;
+ using DirectoryReader = org.apache.lucene.index.DirectoryReader;
+ using DocsAndPositionsEnum = org.apache.lucene.index.DocsAndPositionsEnum;
+ using IndexReader = org.apache.lucene.index.IndexReader;
+ using IndexWriter = org.apache.lucene.index.IndexWriter;
+ using Terms = org.apache.lucene.index.Terms;
+ using TermsEnum = org.apache.lucene.index.TermsEnum;
+ using DocIdSetIterator = org.apache.lucene.search.DocIdSetIterator;
+ using Directory = org.apache.lucene.store.Directory;
+ using AttributeSource = org.apache.lucene.util.AttributeSource;
+ using English = org.apache.lucene.util.English;
+
+
+ /// <summary>
+ /// tests for the TestTeeSinkTokenFilter
+ /// </summary>
+ public class TestTeeSinkTokenFilter : BaseTokenStreamTestCase
+ {
+ protected internal StringBuilder buffer1;
+ protected internal StringBuilder buffer2;
+ protected internal string[] tokens1;
+ protected internal string[] tokens2;
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public void setUp() throws Exception
+ public override void setUp()
+ {
+ base.setUp();
+ tokens1 = new string[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
+ tokens2 = new string[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
+ buffer1 = new StringBuilder();
+
+ for (int i = 0; i < tokens1.Length; i++)
+ {
+ buffer1.Append(tokens1[i]).Append(' ');
+ }
+ buffer2 = new StringBuilder();
+ for (int i = 0; i < tokens2.Length; i++)
+ {
+ buffer2.Append(tokens2[i]).Append(' ');
+ }
+ }
+
+ internal static readonly TeeSinkTokenFilter.SinkFilter theFilter = new SinkFilterAnonymousInnerClassHelper();
+
+ private class SinkFilterAnonymousInnerClassHelper : TeeSinkTokenFilter.SinkFilter
+ {
+ public SinkFilterAnonymousInnerClassHelper()
+ {
+ }
+
+ public override bool accept(AttributeSource a)
+ {
+ CharTermAttribute termAtt = a.getAttribute(typeof(CharTermAttribute));
+ return termAtt.ToString().Equals("The", StringComparison.CurrentCultureIgnoreCase);
+ }
+ }
+
+ internal static readonly TeeSinkTokenFilter.SinkFilter dogFilter = new SinkFilterAnonymousInnerClassHelper2();
+
+ private class SinkFilterAnonymousInnerClassHelper2 : TeeSinkTokenFilter.SinkFilter
+ {
+ public SinkFilterAnonymousInnerClassHelper2()
+ {
+ }
+
+ public override bool accept(AttributeSource a)
+ {
+ CharTermAttribute termAtt = a.getAttribute(typeof(CharTermAttribute));
+ return termAtt.ToString().Equals("Dogs", StringComparison.CurrentCultureIgnoreCase);
+ }
+ }
+
+ // LUCENE-1448
+ // TODO: instead of testing it this way, we can test
+ // with BaseTokenStreamTestCase now...
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testEndOffsetPositionWithTeeSinkTokenFilter() throws Exception
+ public virtual void testEndOffsetPositionWithTeeSinkTokenFilter()
+ {
+ Directory dir = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+ Document doc = new Document();
+ TokenStream tokenStream = analyzer.tokenStream("field", "abcd ");
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(tokenStream);
+ TokenStream sink = tee.newSinkTokenStream();
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.StoreTermVectors = true;
+ ft.StoreTermVectorOffsets = true;
+ ft.StoreTermVectorPositions = true;
+ Field f1 = new Field("field", tee, ft);
+ Field f2 = new Field("field", sink, ft);
+ doc.add(f1);
+ doc.add(f2);
+ w.addDocument(doc);
+ w.close();
+
+ IndexReader r = DirectoryReader.open(dir);
+ Terms vector = r.getTermVectors(0).terms("field");
+ assertEquals(1, vector.size());
+ TermsEnum termsEnum = vector.iterator(null);
+ termsEnum.next();
+ assertEquals(2, termsEnum.totalTermFreq());
+ DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null);
+ assertTrue(positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertEquals(2, positions.freq());
+ positions.nextPosition();
+ assertEquals(0, positions.startOffset());
+ assertEquals(4, positions.endOffset());
+ positions.nextPosition();
+ assertEquals(8, positions.startOffset());
+ assertEquals(12, positions.endOffset());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, positions.nextDoc());
+ r.close();
+ dir.close();
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testGeneral() throws java.io.IOException
+ public virtual void testGeneral()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new MockTokenizer(new java.io.StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter source = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.ToString()), MockTokenizer.WHITESPACE, false));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TokenStream sink1 = source.newSinkTokenStream();
+ TokenStream sink1 = source.newSinkTokenStream();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TokenStream sink2 = source.newSinkTokenStream(theFilter);
+ TokenStream sink2 = source.newSinkTokenStream(theFilter);
+
+ source.addAttribute(typeof(CheckClearAttributesAttribute));
+ sink1.addAttribute(typeof(CheckClearAttributesAttribute));
+ sink2.addAttribute(typeof(CheckClearAttributesAttribute));
+
+ assertTokenStreamContents(source, tokens1);
+ assertTokenStreamContents(sink1, tokens1);
+ assertTokenStreamContents(sink2, new string[]{"The", "the"});
+ }
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void testMultipleSources() throws Exception
+ public virtual void testMultipleSources()
+ {
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new java.io.StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.ToString()), MockTokenizer.WHITESPACE, false));
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
+ TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
+ TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
+ tee1.reset();
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TokenStream source1 = new CachingTokenFilter(tee1);
+ TokenStream source1 = new CachingTokenFilter(tee1);
+
+ tee1.addAttribute(typeof(CheckClearAttributesAttribute));
+ dogDetector.addAttribute(typeof(CheckClearAttributesAttribute));
+ theDetector.addAttribute(typeof(CheckClearAttributesAttribute));
+
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new java.io.StringReader(buffer2.toString()), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer2.ToString()), MockTokenizer.WHITESPACE, false));
+ tee2.addSinkTokenStream(dogDetector);
+ tee2.addSinkTokenStream(theDetector);
+//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
+//ORIGINAL LINE: final TokenStream source2 = tee2;
+ TokenStream source2 = tee2;
+
+ assertTokenStreamContents(source1, tokens1);
+ assertTokenStreamContents(source2, tokens2);
+
+ assertTokenStreamContents(theDetector, new string[]{"The", "the", "The", "the"});
+ assertTokenStreamContents(dogDetector, new string[]{"Dogs", "Dogs"});
+
+ source1.reset();
+ TokenStream lowerCasing = new LowerCaseFilter(TEST_VERSION_CURRENT, source1);
+ string[] lowerCaseTokens = new string[tokens1.Length];
+ for (int i = 0; i < tokens1.Length; i++)
+ {
+ lowerCaseTokens[i] = tokens1[i].ToLower(Locale.ROOT);
+ }
+ assertTokenStreamContents(lowerCasing, lowerCaseTokens);
+ }
+
+ /// <summary>
+ /// Not an explicit test, just useful to print out some info on performance
+ /// </summary>
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void performance() throws Exception
+ public virtual void performance()
+ {
+ int[] tokCount = new int[] {100, 500, 1000, 2000, 5000, 10000};
+ int[] modCounts = new int[] {1, 2, 5, 10, 20, 50, 100, 200, 500};
+ for (int k = 0; k < tokCount.Length; k++)
+ {
+ StringBuilder buffer = new StringBuilder();
+ Console.WriteLine("-----Tokens: " + tokCount[k] + "-----");
+ for (int i = 0; i < tokCount[k]; i++)
+ {
+ buffer.Append(English.intToEnglish(i).toUpperCase(Locale.ROOT)).Append(' ');
+ }
+ //make sure we produce the same tokens
+ TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))));
+ TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(this, 100));
+ teeStream.consumeAllTokens();
+ TokenStream stream = new ModuloTokenFilter(this, new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))), 100);
+ CharTermAttribute tfTok = stream.addAttribute(typeof(CharTermAttribute));
+ CharTermAttribute sinkTok = sink.addAttribute(typeof(CharTermAttribute));
+ for (int i = 0; stream.incrementToken(); i++)
+ {
+ assertTrue(sink.incrementToken());
+ assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.Equals(sinkTok) == true);
+ }
+
+ //simulate two fields, each being analyzed once, for 20 documents
+ for (int j = 0; j < modCounts.Length; j++)
+ {
+ int tfPos = 0;
+ long start = DateTimeHelperClass.CurrentUnixTimeMillis();
+ for (int i = 0; i < 20; i++)
+ {
+ stream = new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString())));
+ PositionIncrementAttribute posIncrAtt = stream.getAttribute(typeof(PositionIncrementAttribute));
+ while (stream.incrementToken())
+ {
+ tfPos += posIncrAtt.PositionIncrement;
+ }
+ stream = new ModuloTokenFilter(this, new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))), modCounts[j]);
+ posIncrAtt = stream.getAttribute(typeof(PositionIncrementAttribute));
+ while (stream.incrementToken())
+ {
+ tfPos += posIncrAtt.PositionIncrement;
+ }
+ }
+ long finish = DateTimeHelperClass.CurrentUnixTimeMillis();
+ Console.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
+ int sinkPos = 0;
+ //simulate one field with one sink
+ start = DateTimeHelperClass.CurrentUnixTimeMillis();
+ for (int i = 0; i < 20; i++)
+ {
+ teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.ToString()))));
+ sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(this, modCounts[j]));
+ PositionIncrementAttribute posIncrAtt = teeStream.getAttribute(typeof(PositionIncrementAttribute));
+ while (teeStream.incrementToken())
+ {
+ sinkPos += posIncrAtt.PositionIncrement;
+ }
+ //System.out.println("Modulo--------");
+ posIncrAtt = sink.getAttribute(typeof(PositionIncrementAttribute));
+ while (sink.incrementToken())
+ {
+ sinkPos += posIncrAtt.PositionIncrement;
+ }
+ }
+ finish = DateTimeHelperClass.CurrentUnixTimeMillis();
+ Console.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
+ assertTrue(sinkPos + " does not equal: " + tfPos, sinkPos == tfPos);
+
+ }
+ Console.WriteLine("- End Tokens: " + tokCount[k] + "-----");
+ }
+
+ }
+
+
+ internal class ModuloTokenFilter : TokenFilter
+ {
+ private readonly TestTeeSinkTokenFilter outerInstance;
+
+
+ internal int modCount;
+
+ internal ModuloTokenFilter(TestTeeSinkTokenFilter outerInstance, TokenStream input, int mc) : base(input)
+ {
+ this.outerInstance = outerInstance;
+ modCount = mc;
+ }
+
+ internal int count = 0;
+
+ //return every 100 tokens
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: @Override public boolean incrementToken() throws java.io.IOException
+ public override bool incrementToken()
+ {
+ bool hasNext;
+ for (hasNext = input.incrementToken(); hasNext && count % modCount != 0; hasNext = input.incrementToken())
+ {
+ count++;
+ }
+ count++;
+ return hasNext;
+ }
+ }
+
+ internal class ModuloSinkFilter : TeeSinkTokenFilter.SinkFilter
+ {
+ private readonly TestTeeSinkTokenFilter outerInstance;
+
+ internal int count = 0;
+ internal int modCount;
+
+ internal ModuloSinkFilter(TestTeeSinkTokenFilter outerInstance, int mc)
+ {
+ this.outerInstance = outerInstance;
+ modCount = mc;
+ }
+
+ public override bool accept(AttributeSource a)
+ {
+ bool b = (a != null && count % modCount == 0);
+ count++;
+ return b;
+ }
+
+ }
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c64856a7/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TokenRangeSinkTokenizerTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TokenRangeSinkTokenizerTest.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TokenRangeSinkTokenizerTest.cs
new file mode 100644
index 0000000..5628c39
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Sinks/TokenRangeSinkTokenizerTest.cs
@@ -0,0 +1,62 @@
+namespace org.apache.lucene.analysis.sinks
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using Test = org.junit.Test;
+
+ public class TokenRangeSinkTokenizerTest : BaseTokenStreamTestCase
+ {
+
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+//ORIGINAL LINE: public void test() throws java.io.IOException
+ public virtual void test()
+ {
+ TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
+ string test = "The quick red fox jumped over the lazy brown dogs";
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.newSinkTokenStream(sinkFilter);
+
+ int count = 0;
+ tee.reset();
+ while (tee.incrementToken())
+ {
+ count++;
+ }
+
+ int sinkCount = 0;
+ rangeToks.reset();
+ while (rangeToks.incrementToken())
+ {
+ sinkCount++;
+ }
+
+ assertTrue(count + " does not equal: " + 10, count == 10);
+ assertTrue("rangeToks Size: " + sinkCount + " is not: " + 2, sinkCount == 2);
+ }
+
+//JAVA TO C# CONVERTER TODO TASK: Most Java annotations will not have direct .NET equivalent attributes:
+//ORIGINAL LINE: @Test(expected = IllegalArgumentException.class) public void testIllegalArguments() throws Exception
+//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ public virtual void testIllegalArguments()
+ {
+ new TokenRangeSinkFilter(4, 2);
+ }
+ }
+}
\ No newline at end of file