You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/07/23 17:36:32 UTC
[07/13] lucenenet git commit: Ported Lucene.Net.Analysis.Kuromoji +
tests
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Analysis.Kuromoji/Util/ToStringUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Kuromoji/Util/ToStringUtil.cs b/src/Lucene.Net.Analysis.Kuromoji/Util/ToStringUtil.cs
new file mode 100644
index 0000000..95e2703
--- /dev/null
+++ b/src/Lucene.Net.Analysis.Kuromoji/Util/ToStringUtil.cs
@@ -0,0 +1,1401 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Lucene.Net.Analysis.Ja.Util
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Utility class for english translations of morphological data,
+ /// used only for debugging.
+ /// </summary>
+ public static class ToStringUtil
+ {
+ // a translation map for parts of speech, only used for reflectWith
+ private static readonly IDictionary<string, string> posTranslations = new Dictionary<string, string>(StringComparer.Ordinal)
+ {
+ { "名詞", "noun"},
+ { "名詞-一般", "noun-common" },
+ { "名詞-固有名詞", "noun-proper" },
+ { "名詞-固有名詞-一般", "noun-proper-misc" },
+ { "名詞-固有名詞-人名", "noun-proper-person" },
+ { "名詞-固有名詞-人名-一般", "noun-proper-person-misc" },
+ { "名詞-固有名詞-人名-姓", "noun-proper-person-surname" },
+ { "名詞-固有名詞-人名-名", "noun-proper-person-given_name" },
+ { "名詞-固有名詞-組織", "noun-proper-organization" },
+ { "名詞-固有名詞-地域", "noun-proper-place" },
+ { "名詞-固有名詞-地域-一般", "noun-proper-place-misc" },
+ { "名詞-固有名詞-地域-国", "noun-proper-place-country" },
+ { "名詞-代名詞", "noun-pronoun" },
+ { "名詞-代名詞-一般", "noun-pronoun-misc" },
+ { "名詞-代名詞-縮約", "noun-pronoun-contraction" },
+ { "名詞-副詞可能", "noun-adverbial" },
+ { "名詞-サ変接続", "noun-verbal" },
+ { "名詞-形容動詞語幹", "noun-adjective-base" },
+ { "名詞-数", "noun-numeric" },
+ { "名詞-非自立", "noun-affix" },
+ { "名詞-非自立-一般", "noun-affix-misc" },
+ { "名詞-非自立-副詞可能", "noun-affix-adverbial" },
+ { "名詞-非自立-助動詞語幹", "noun-affix-aux" },
+ { "名詞-非自立-形容動詞語幹", "noun-affix-adjective-base" },
+ { "名詞-特殊", "noun-special" },
+ { "名詞-特殊-助動詞語幹", "noun-special-aux" },
+ { "名詞-接尾", "noun-suffix" },
+ { "名詞-接尾-一般", "noun-suffix-misc" },
+ { "名詞-接尾-人名", "noun-suffix-person" },
+ { "名詞-接尾-地域", "noun-suffix-place" },
+ { "名詞-接尾-サ変接続", "noun-suffix-verbal" },
+ { "名詞-接尾-助動詞語幹", "noun-suffix-aux" },
+ { "名詞-接尾-形容動詞語幹", "noun-suffix-adjective-base" },
+ { "名詞-接尾-副詞可能", "noun-suffix-adverbial" },
+ { "名詞-接尾-助数詞", "noun-suffix-classifier" },
+ { "名詞-接尾-特殊", "noun-suffix-special" },
+ { "名詞-接続詞的", "noun-suffix-conjunctive" },
+ { "名詞-動詞非自立的", "noun-verbal_aux" },
+ { "名詞-引用文字列", "noun-quotation" },
+ { "名詞-ナイ形容詞語幹", "noun-nai_adjective" },
+ { "接頭詞", "prefix" },
+ { "接頭詞-名詞接続", "prefix-nominal" },
+ { "接頭詞-動詞接続", "prefix-verbal" },
+ { "接頭詞-形容詞接続", "prefix-adjectival" },
+ { "接頭詞-数接続", "prefix-numerical" },
+ { "動詞", "verb" },
+ { "動詞-自立", "verb-main" },
+ { "動詞-非自立", "verb-auxiliary" },
+ { "動詞-接尾", "verb-suffix" },
+ { "形容詞", "adjective" },
+ { "形容詞-自立", "adjective-main" },
+ { "形容詞-非自立", "adjective-auxiliary" },
+ { "形容詞-接尾", "adjective-suffix" },
+ { "副詞", "adverb" },
+ { "副詞-一般", "adverb-misc" },
+ { "副詞-助詞類接続", "adverb-particle_conjunction" },
+ { "連体詞", "adnominal" },
+ { "接続詞", "conjunction" },
+ { "助詞", "particle" },
+ { "助詞-格助詞", "particle-case" },
+ { "助詞-格助詞-一般", "particle-case-misc" },
+ { "助詞-格助詞-引用", "particle-case-quote" },
+ { "助詞-格助詞-連語", "particle-case-compound" },
+ { "助詞-接続助詞", "particle-conjunctive" },
+ { "助詞-係助詞", "particle-dependency" },
+ { "助詞-副助詞", "particle-adverbial" },
+ { "助詞-間投助詞", "particle-interjective" },
+ { "助詞-並立助詞", "particle-coordinate" },
+ { "助詞-終助詞", "particle-final" },
+ { "助詞-副助詞／並立助詞／終助詞", "particle-adverbial/conjunctive/final" },
+ { "助詞-連体化", "particle-adnominalizer" },
+ { "助詞-副詞化", "particle-adnominalizer" },
+ { "助詞-特殊", "particle-special" },
+ { "助動詞", "auxiliary-verb" },
+ { "感動詞", "interjection" },
+ { "記号", "symbol" },
+ { "記号-一般", "symbol-misc" },
+ { "記号-句点", "symbol-period" },
+ { "記号-読点", "symbol-comma" },
+ { "記号-空白", "symbol-space" },
+ { "記号-括弧開", "symbol-open_bracket" },
+ { "記号-括弧閉", "symbol-close_bracket" },
+ { "記号-アルファベット", "symbol-alphabetic" },
+ { "その他", "other" },
+ { "その他-間投", "other-interjection" },
+ { "フィラー", "filler" },
+ { "非言語音", "non-verbal" },
+ { "語断片", "fragment" },
+ { "未知語", "unknown" }
+ };
+
+
+ /// <summary>
+ /// Get the english form of a POS tag
+ /// </summary>
+ public static string GetPOSTranslation(string s)
+ {
+ string result;
+ posTranslations.TryGetValue(s, out result);
+ return result;
+ }
+
+ // a translation map for inflection types, only used for reflectWith
+ private static readonly IDictionary<string, string> inflTypeTranslations = new Dictionary<string, string>(StringComparer.Ordinal)
+ {
+ { "*", "*" },
+ { "形容詞・アウオ段", "adj-group-a-o-u" },
+ { "形容詞・イ段", "adj-group-i" },
+ { "形容詞・イイ", "adj-group-ii" },
+ { "不変化型", "non-inflectional" },
+ { "特殊・タ", "special-da" },
+ { "特殊・ダ", "special-ta" },
+ { "文語・ゴトシ", "classical-gotoshi" },
+ { "特殊・ジャ", "special-ja" },
+ { "特殊・ナイ", "special-nai" },
+ { "五段・ラ行特殊", "5-row-cons-r-special" },
+ { "特殊・ヌ", "special-nu" },
+ { "文語・キ", "classical-ki" },
+ { "特殊・タイ", "special-tai" },
+ { "文語・ベシ", "classical-beshi" },
+ { "特殊・ヤ", "special-ya" },
+ { "文語・マジ", "classical-maji" },
+ { "下二・タ行", "2-row-lower-cons-t" },
+ { "特殊・デス", "special-desu" },
+ { "特殊・マス", "special-masu" },
+ { "五段・ラ行アル", "5-row-aru" },
+ { "文語・ナリ", "classical-nari" },
+ { "文語・リ", "classical-ri" },
+ { "文語・ケリ", "classical-keri" },
+ { "文語・ル", "classical-ru" },
+ { "五段・カ行イ音便", "5-row-cons-k-i-onbin" },
+ { "五段・サ行", "5-row-cons-s" },
+ { "一段", "1-row" },
+ { "五段・ワ行促音便", "5-row-cons-w-cons-onbin" },
+ { "五段・マ行", "5-row-cons-m" },
+ { "五段・タ行", "5-row-cons-t" },
+ { "五段・ラ行", "5-row-cons-r" },
+ { "サ変・−スル", "irregular-suffix-suru" },
+ { "五段・ガ行", "5-row-cons-g" },
+ { "サ変・−ズル", "irregular-suffix-zuru" },
+ { "五段・バ行", "5-row-cons-b" },
+ { "五段・ワ行ウ音便", "5-row-cons-w-u-onbin" },
+ { "下二・ダ行", "2-row-lower-cons-d" },
+ { "五段・カ行促音便ユク", "5-row-cons-k-cons-onbin-yuku" },
+ { "上二・ダ行", "2-row-upper-cons-d" },
+ { "五段・カ行促音便", "5-row-cons-k-cons-onbin" },
+ { "一段・得ル", "1-row-eru" },
+ { "四段・タ行", "4-row-cons-t" },
+ { "五段・ナ行", "5-row-cons-n" },
+ { "下二・ハ行", "2-row-lower-cons-h" },
+ { "四段・ハ行", "4-row-cons-h" },
+ { "四段・バ行", "4-row-cons-b" },
+ { "サ変・スル", "irregular-suru" },
+ { "上二・ハ行", "2-row-upper-cons-h" },
+ { "下二・マ行", "2-row-lower-cons-m" },
+ { "四段・サ行", "4-row-cons-s" },
+ { "下二・ガ行", "2-row-lower-cons-g" },
+ { "カ変・来ル", "kuru-kanji" },
+ { "一段・クレル", "1-row-kureru" },
+ { "下二・得", "2-row-lower-u" },
+ { "カ変・クル", "kuru-kana" },
+ { "ラ変", "irregular-cons-r" },
+ { "下二・カ行", "2-row-lower-cons-k" },
+ };
+
+
+ /// <summary>
+ /// Get the english form of inflection type
+ /// </summary>
+ public static string GetInflectionTypeTranslation(string s)
+ {
+ string result;
+ inflTypeTranslations.TryGetValue(s, out result);
+ return result;
+ }
+
+ // a translation map for inflection forms, only used for reflectWith
+ private static readonly IDictionary<string, string> inflFormTranslations = new Dictionary<string, string>(StringComparer.Ordinal)
+ {
+ { "*", "*" },
+ { "基本形", "base" },
+ { "文語基本形", "classical-base" },
+ { "未然ヌ接続", "imperfective-nu-connection" },
+ { "未然ウ接続", "imperfective-u-connection" },
+ { "連用タ接続", "conjunctive-ta-connection" },
+ { "連用テ接続", "conjunctive-te-connection" },
+ { "連用ゴザイ接続", "conjunctive-gozai-connection" },
+ { "体言接続", "uninflected-connection" },
+ { "仮定形", "subjunctive" },
+ { "命令ｅ", "imperative-e" },
+ { "仮定縮約１", "conditional-contracted-1" },
+ { "仮定縮約２", "conditional-contracted-2" },
+ { "ガル接続", "garu-connection" },
+ { "未然形", "imperfective" },
+ { "連用形", "conjunctive" },
+ { "音便基本形", "onbin-base" },
+ { "連用デ接続", "conjunctive-de-connection" },
+ { "未然特殊", "imperfective-special" },
+ { "命令ｉ", "imperative-i" },
+ { "連用ニ接続", "conjunctive-ni-connection" },
+ { "命令ｙｏ", "imperative-yo" },
+ { "体言接続特殊", "adnominal-special" },
+ { "命令ｒｏ", "imperative-ro" },
+ { "体言接続特殊２", "uninflected-special-connection-2" },
+ { "未然レル接続", "imperfective-reru-connection" },
+ { "現代基本形", "modern-base" },
+ { "基本形-促音便", "base-onbin" }, // not sure about this
+ };
+
+
+ /// <summary>
+ /// Get the english form of inflected form
+ /// </summary>
+ public static string GetInflectedFormTranslation(string s)
+ {
+ string result;
+ inflFormTranslations.TryGetValue(s, out result);
+ return result;
+ }
+
+ /// <summary>
+ /// Romanize katakana with modified hepburn
+ /// </summary>
+ public static string GetRomanization(string s)
+ {
+ StringBuilder result = new StringBuilder();
+ try
+ {
+ GetRomanization(result, s);
+ }
+ catch (IOException bogus)
+ {
+ throw new Exception(bogus.ToString(), bogus);
+ }
+ return result.ToString();
+ }
+
+ /// <summary>
+ /// Romanize katakana with modified hepburn
+ /// </summary>
+ // TODO: now that this is used by readingsfilter and not just for
+ // debugging, fix this to really be a scheme that works best with IMEs
+ public static void GetRomanization(StringBuilder builder, string s)
+ {
+ int len = s.Length;
+ for (int i = 0; i < len; i++)
+ {
+ // maximum lookahead: 3
+ char ch = s[i];
+ char ch2 = (i < len - 1) ? s[i + 1] : (char)0;
+ char ch3 = (i < len - 2) ? s[i + 2] : (char)0;
+
+ //main:
+ switch (ch)
+ {
+
+ case 'ッ':
+ switch (ch2)
+ {
+ case 'カ':
+ case 'キ':
+ case 'ク':
+ case 'ケ':
+ case 'コ':
+ builder.Append('k');
+ goto break_main;
+ case 'サ':
+ case 'シ':
+ case 'ス':
+ case 'セ':
+ case 'ソ':
+ builder.Append('s');
+ goto break_main;
+ case 'タ':
+ case 'チ':
+ case 'ツ':
+ case 'テ':
+ case 'ト':
+ builder.Append('t');
+ goto break_main;
+ case 'パ':
+ case 'ピ':
+ case 'プ':
+ case 'ペ':
+ case 'ポ':
+ builder.Append('p');
+ goto break_main;
+ }
+ break;
+ case 'ア':
+ builder.Append('a');
+ break;
+ case 'イ':
+ if (ch2 == 'ィ')
+ {
+ builder.Append("yi");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("ye");
+ i++;
+ }
+ else
+ {
+ builder.Append('i');
+ }
+ break;
+ case 'ウ':
+ switch (ch2)
+ {
+ case 'ァ':
+ builder.Append("wa");
+ i++;
+ break;
+ case 'ィ':
+ builder.Append("wi");
+ i++;
+ break;
+ case 'ゥ':
+ builder.Append("wu");
+ i++;
+ break;
+ case 'ェ':
+ builder.Append("we");
+ i++;
+ break;
+ case 'ォ':
+ builder.Append("wo");
+ i++;
+ break;
+ case 'ュ':
+ builder.Append("wyu");
+ i++;
+ break;
+ default:
+ builder.Append('u');
+ break;
+ }
+ break;
+ case 'エ':
+ builder.Append('e');
+ break;
+ case 'オ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append('ō');
+ i++;
+ }
+ else
+ {
+ builder.Append('o');
+ }
+ break;
+ case 'カ':
+ builder.Append("ka");
+ break;
+ case 'キ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("kyō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("kyū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("kya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("kyo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("kyu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("kye");
+ i++;
+ }
+ else
+ {
+ builder.Append("ki");
+ }
+ break;
+ case 'ク':
+ switch (ch2)
+ {
+ case 'ァ':
+ builder.Append("kwa");
+ i++;
+ break;
+ case 'ィ':
+ builder.Append("kwi");
+ i++;
+ break;
+ case 'ェ':
+ builder.Append("kwe");
+ i++;
+ break;
+ case 'ォ':
+ builder.Append("kwo");
+ i++;
+ break;
+ case 'ヮ':
+ builder.Append("kwa");
+ i++;
+ break;
+ default:
+ builder.Append("ku");
+ break;
+ }
+ break;
+ case 'ケ':
+ builder.Append("ke");
+ break;
+ case 'コ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("kō");
+ i++;
+ }
+ else
+ {
+ builder.Append("ko");
+ }
+ break;
+ case 'サ':
+ builder.Append("sa");
+ break;
+ case 'シ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("shō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("shū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("sha");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("sho");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("shu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("she");
+ i++;
+ }
+ else
+ {
+ builder.Append("shi");
+ }
+ break;
+ case 'ス':
+ if (ch2 == 'ィ')
+ {
+ builder.Append("si");
+ i++;
+ }
+ else
+ {
+ builder.Append("su");
+ }
+ break;
+ case 'セ':
+ builder.Append("se");
+ break;
+ case 'ソ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("sō");
+ i++;
+ }
+ else
+ {
+ builder.Append("so");
+ }
+ break;
+ case 'タ':
+ builder.Append("ta");
+ break;
+ case 'チ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("chō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("chū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("cha");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("cho");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("chu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("che");
+ i++;
+ }
+ else
+ {
+ builder.Append("chi");
+ }
+ break;
+ case 'ツ':
+ if (ch2 == 'ァ')
+ {
+ builder.Append("tsa");
+ i++;
+ }
+ else if (ch2 == 'ィ')
+ {
+ builder.Append("tsi");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("tse");
+ i++;
+ }
+ else if (ch2 == 'ォ')
+ {
+ builder.Append("tso");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("tsyu");
+ i++;
+ }
+ else
+ {
+ builder.Append("tsu");
+ }
+ break;
+ case 'テ':
+ if (ch2 == 'ィ')
+ {
+ builder.Append("ti");
+ i++;
+ }
+ else if (ch2 == 'ゥ')
+ {
+ builder.Append("tu");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("tyu");
+ i++;
+ }
+ else
+ {
+ builder.Append("te");
+ }
+ break;
+ case 'ト':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("tō");
+ i++;
+ }
+ else if (ch2 == 'ゥ')
+ {
+ builder.Append("tu");
+ i++;
+ }
+ else
+ {
+ builder.Append("to");
+ }
+ break;
+ case 'ナ':
+ builder.Append("na");
+ break;
+ case 'ニ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("nyō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("nyū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("nya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("nyo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("nyu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("nye");
+ i++;
+ }
+ else
+ {
+ builder.Append("ni");
+ }
+ break;
+ case 'ヌ':
+ builder.Append("nu");
+ break;
+ case 'ネ':
+ builder.Append("ne");
+ break;
+ case 'ノ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("nō");
+ i++;
+ }
+ else
+ {
+ builder.Append("no");
+ }
+ break;
+ case 'ハ':
+ builder.Append("ha");
+ break;
+ case 'ヒ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("hyō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("hyū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("hya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("hyo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("hyu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("hye");
+ i++;
+ }
+ else
+ {
+ builder.Append("hi");
+ }
+ break;
+ case 'フ':
+ if (ch2 == 'ャ')
+ {
+ builder.Append("fya");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("fyu");
+ i++;
+ }
+ else if (ch2 == 'ィ' && ch3 == 'ェ')
+ {
+ builder.Append("fye");
+ i += 2;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("fyo");
+ i++;
+ }
+ else if (ch2 == 'ァ')
+ {
+ builder.Append("fa");
+ i++;
+ }
+ else if (ch2 == 'ィ')
+ {
+ builder.Append("fi");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("fe");
+ i++;
+ }
+ else if (ch2 == 'ォ')
+ {
+ builder.Append("fo");
+ i++;
+ }
+ else
+ {
+ builder.Append("fu");
+ }
+ break;
+ case 'ヘ':
+ builder.Append("he");
+ break;
+ case 'ホ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("hō");
+ i++;
+ }
+ else if (ch2 == 'ゥ')
+ {
+ builder.Append("hu");
+ i++;
+ }
+ else
+ {
+ builder.Append("ho");
+ }
+ break;
+ case 'マ':
+ builder.Append("ma");
+ break;
+ case 'ミ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("myō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("myū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("mya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("myo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("myu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("mye");
+ i++;
+ }
+ else
+ {
+ builder.Append("mi");
+ }
+ break;
+ case 'ム':
+ builder.Append("mu");
+ break;
+ case 'メ':
+ builder.Append("me");
+ break;
+ case 'モ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("mō");
+ i++;
+ }
+ else
+ {
+ builder.Append("mo");
+ }
+ break;
+ case 'ヤ':
+ builder.Append("ya");
+ break;
+ case 'ユ':
+ builder.Append("yu");
+ break;
+ case 'ヨ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("yō");
+ i++;
+ }
+ else
+ {
+ builder.Append("yo");
+ }
+ break;
+ case 'ラ':
+ if (ch2 == '゜')
+ {
+ builder.Append("la");
+ i++;
+ }
+ else
+ {
+ builder.Append("ra");
+ }
+ break;
+ case 'リ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("ryō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("ryū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("rya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("ryo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("ryu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("rye");
+ i++;
+ }
+ else if (ch2 == '゜')
+ {
+ builder.Append("li");
+ i++;
+ }
+ else
+ {
+ builder.Append("ri");
+ }
+ break;
+ case 'ル':
+ if (ch2 == '゜')
+ {
+ builder.Append("lu");
+ i++;
+ }
+ else
+ {
+ builder.Append("ru");
+ }
+ break;
+ case 'レ':
+ if (ch2 == '゜')
+ {
+ builder.Append("le");
+ i++;
+ }
+ else
+ {
+ builder.Append("re");
+ }
+ break;
+ case 'ロ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("rō");
+ i++;
+ }
+ else if (ch2 == '゜')
+ {
+ builder.Append("lo");
+ i++;
+ }
+ else
+ {
+ builder.Append("ro");
+ }
+ break;
+ case 'ワ':
+ builder.Append("wa");
+ break;
+ case 'ヰ':
+ builder.Append("i");
+ break;
+ case 'ヱ':
+ builder.Append("e");
+ break;
+ case 'ヲ':
+ builder.Append("o");
+ break;
+ case 'ン':
+ switch (ch2)
+ {
+ case 'バ':
+ case 'ビ':
+ case 'ブ':
+ case 'ベ':
+ case 'ボ':
+ case 'パ':
+ case 'ピ':
+ case 'プ':
+ case 'ペ':
+ case 'ポ':
+ case 'マ':
+ case 'ミ':
+ case 'ム':
+ case 'メ':
+ case 'モ':
+ builder.Append('m');
+ goto break_main;
+ case 'ヤ':
+ case 'ユ':
+ case 'ヨ':
+ case 'ア':
+ case 'イ':
+ case 'ウ':
+ case 'エ':
+ case 'オ':
+ builder.Append("n'");
+ goto break_main;
+ default:
+ builder.Append("n");
+ goto break_main;
+ }
+ case 'ガ':
+ builder.Append("ga");
+ break;
+ case 'ギ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("gyō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("gyū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("gya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("gyo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("gyu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("gye");
+ i++;
+ }
+ else
+ {
+ builder.Append("gi");
+ }
+ break;
+ case 'グ':
+ switch (ch2)
+ {
+ case 'ァ':
+ builder.Append("gwa");
+ i++;
+ break;
+ case 'ィ':
+ builder.Append("gwi");
+ i++;
+ break;
+ case 'ェ':
+ builder.Append("gwe");
+ i++;
+ break;
+ case 'ォ':
+ builder.Append("gwo");
+ i++;
+ break;
+ case 'ヮ':
+ builder.Append("gwa");
+ i++;
+ break;
+ default:
+ builder.Append("gu");
+ break;
+ }
+ break;
+ case 'ゲ':
+ builder.Append("ge");
+ break;
+ case 'ゴ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("gō");
+ i++;
+ }
+ else
+ {
+ builder.Append("go");
+ }
+ break;
+ case 'ザ':
+ builder.Append("za");
+ break;
+ case 'ジ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("jō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("jū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("ja");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("jo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("ju");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("je");
+ i++;
+ }
+ else
+ {
+ builder.Append("ji");
+ }
+ break;
+ case 'ズ':
+ if (ch2 == 'ィ')
+ {
+ builder.Append("zi");
+ i++;
+ }
+ else
+ {
+ builder.Append("zu");
+ }
+ break;
+ case 'ゼ':
+ builder.Append("ze");
+ break;
+ case 'ゾ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("zō");
+ i++;
+ }
+ else
+ {
+ builder.Append("zo");
+ }
+ break;
+ case 'ダ':
+ builder.Append("da");
+ break;
+ case 'ヂ':
+ // TODO: investigate all this
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("jō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("jū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("ja");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("jo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("ju");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("je");
+ i++;
+ }
+ else
+ {
+ builder.Append("ji");
+ }
+ break;
+ case 'ヅ':
+ builder.Append("zu");
+ break;
+ case 'デ':
+ if (ch2 == 'ィ')
+ {
+ builder.Append("di");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("dyu");
+ i++;
+ }
+ else
+ {
+ builder.Append("de");
+ }
+ break;
+ case 'ド':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("dō");
+ i++;
+ }
+ else if (ch2 == 'ゥ')
+ {
+ builder.Append("du");
+ i++;
+ }
+ else
+ {
+ builder.Append("do");
+ }
+ break;
+ case 'バ':
+ builder.Append("ba");
+ break;
+ case 'ビ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("byō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("byū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("bya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("byo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("byu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("bye");
+ i++;
+ }
+ else
+ {
+ builder.Append("bi");
+ }
+ break;
+ case 'ブ':
+ builder.Append("bu");
+ break;
+ case 'ベ':
+ builder.Append("be");
+ break;
+ case 'ボ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("bō");
+ i++;
+ }
+ else
+ {
+ builder.Append("bo");
+ }
+ break;
+ case 'パ':
+ builder.Append("pa");
+ break;
+ case 'ピ':
+ if (ch2 == 'ョ' && ch3 == 'ウ')
+ {
+ builder.Append("pyō");
+ i += 2;
+ }
+ else if (ch2 == 'ュ' && ch3 == 'ウ')
+ {
+ builder.Append("pyū");
+ i += 2;
+ }
+ else if (ch2 == 'ャ')
+ {
+ builder.Append("pya");
+ i++;
+ }
+ else if (ch2 == 'ョ')
+ {
+ builder.Append("pyo");
+ i++;
+ }
+ else if (ch2 == 'ュ')
+ {
+ builder.Append("pyu");
+ i++;
+ }
+ else if (ch2 == 'ェ')
+ {
+ builder.Append("pye");
+ i++;
+ }
+ else
+ {
+ builder.Append("pi");
+ }
+ break;
+ case 'プ':
+ builder.Append("pu");
+ break;
+ case 'ペ':
+ builder.Append("pe");
+ break;
+ case 'ポ':
+ if (ch2 == 'ウ')
+ {
+ builder.Append("pō");
+ i++;
+ }
+ else
+ {
+ builder.Append("po");
+ }
+ break;
+ case 'ヷ':
+ builder.Append("va");
+ break;
+ case 'ヸ':
+ builder.Append("vi");
+ break;
+ case 'ヹ':
+ builder.Append("ve");
+ break;
+ case 'ヺ':
+ builder.Append("vo");
+ break;
+ case 'ヴ':
+ if (ch2 == 'ィ' && ch3 == 'ェ')
+ {
+ builder.Append("vye");
+ i += 2;
+ }
+ else
+ {
+ builder.Append('v');
+ }
+ break;
+ case 'ァ':
+ builder.Append('a');
+ break;
+ case 'ィ':
+ builder.Append('i');
+ break;
+ case 'ゥ':
+ builder.Append('u');
+ break;
+ case 'ェ':
+ builder.Append('e');
+ break;
+ case 'ォ':
+ builder.Append('o');
+ break;
+ case 'ヮ':
+ builder.Append("wa");
+ break;
+ case 'ャ':
+ builder.Append("ya");
+ break;
+ case 'ュ':
+ builder.Append("yu");
+ break;
+ case 'ョ':
+ builder.Append("yo");
+ break;
+ case 'ー':
+ break;
+ default:
+ builder.Append(ch);
+ break;
+ }
+ break_main: { }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Analysis.Kuromoji/project.json
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Kuromoji/project.json b/src/Lucene.Net.Analysis.Kuromoji/project.json
new file mode 100644
index 0000000..937b9bf
--- /dev/null
+++ b/src/Lucene.Net.Analysis.Kuromoji/project.json
@@ -0,0 +1,60 @@
+{
+ "version": "4.8.0",
+ "title": "Lucene.Net.Analysis.Kuromoji",
+ "description": "Japanese Morphological Analyzer for the Lucene.Net full-text search engine library from The Apache Software Foundation.",
+ "authors": [ "The Apache Software Foundation" ],
+ "packOptions": {
+ "projectUrl": "http://lucenenet.apache.org/",
+ "licenseUrl": "https://github.com/apache/lucenenet/blob/master/LICENSE.txt",
+ "iconUrl": "https://github.com/apache/lucenenet/blob/master/branding/logo/lucene-net-icon-128x128.png?raw=true",
+ "owners": [ "The Apache Software Foundation" ],
+ "repository": { "url": "https://github.com/apache/lucenenet" },
+ "tags": [ "lucene.net", "core", "text", "search", "information", "retrieval", "lucene", "apache", "analysis", "index", "query", "japanese" ],
+ "releaseNotes": "This package depends on a temporary version of icu.net hosted on MyGet until official .NET Core support is added. To install, copy the NuGet.config file from https://github.com/apache/lucenenet/blob/master/NuGet.config into your project and then install this package via Package Manager Console as usual."
+ },
+ "buildOptions": {
+ "compile": {
+ "includeFiles": [ "../CommonAssemblyInfo.cs" ]
+ },
+ "embed": {
+ "includeFiles": [
+ "stoptags.txt",
+ "stopwords.txt",
+ "Dict/CharacterDefinition.dat",
+ "Dict/ConnectionCosts.dat",
+ "Dict/TokenInfoDictionary$buffer.dat",
+ "Dict/TokenInfoDictionary$fst.dat",
+ "Dict/TokenInfoDictionary$posDict.dat",
+ "Dict/TokenInfoDictionary$targetMap.dat",
+ "Dict/UnknownDictionary$buffer.dat",
+ "Dict/UnknownDictionary$posDict.dat",
+ "Dict/UnknownDictionary$targetMap.dat"
+ ]
+ },
+ "nowarn": [ "1591", "1573" ]
+ },
+ "dependencies": {
+ "Lucene.Net": "4.8.0",
+ "Lucene.Net.Analysis.Common": "4.8.0"
+ },
+ "frameworks": {
+ "netstandard1.5": {
+ "imports": "dnxcore50",
+ "buildOptions": {
+ "debugType": "portable",
+ "define": [ "NETSTANDARD" ]
+ },
+ "dependencies": {
+ "NETStandard.Library": "1.6.0",
+ "System.Globalization.Extensions": "4.3.0",
+ "System.Text.Encoding.CodePages": "4.4.0-preview1-25305-02"
+ }
+ },
+ "net451": {
+ "buildOptions": {
+ "debugType": "full",
+ "define": [ "FEATURE_SERIALIZABLE" ]
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Analysis.Kuromoji/stoptags.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Kuromoji/stoptags.txt b/src/Lucene.Net.Analysis.Kuromoji/stoptags.txt
new file mode 100644
index 0000000..71b7508
--- /dev/null
+++ b/src/Lucene.Net.Analysis.Kuromoji/stoptags.txt
@@ -0,0 +1,420 @@
+#
+# This file defines a Japanese stoptag set for JapanesePartOfSpeechStopFilter.
+#
+# Any token with a part-of-speech tag that exactly matches those defined in this
+# file are removed from the token stream.
+#
+# Set your own stoptags by uncommenting the lines below. Note that comments are
+# not allowed on the same line as a stoptag. See LUCENE-3745 for frequency lists,
+# etc. that can be useful for building you own stoptag set.
+#
+# The entire possible tagset is provided below for convenience.
+#
+#####
+# noun: unclassified nouns
+#名詞
+#
+# noun-common: Common nouns or nouns where the sub-classification is undefined
+#名詞-一般
+#
+# noun-proper: Proper nouns where the sub-classification is undefined
+#名詞-固有名詞
+#
+# noun-proper-misc: miscellaneous proper nouns
+#名詞-固有名詞-一般
+#
+# noun-proper-person: Personal names where the sub-classification is undefined
+#名詞-固有名詞-人名
+#
+# noun-proper-person-misc: names that cannot be divided into surname and
+# given name; foreign names; names where the surname or given name is unknown.
+# e.g. お市の方
+#名詞-固有名詞-人名-一般
+#
+# noun-proper-person-surname: Mainly Japanese surnames.
+# e.g. 山田
+#名詞-固有名詞-人名-姓
+#
+# noun-proper-person-given_name: Mainly Japanese given names.
+# e.g. 太郎
+#名詞-固有名詞-人名-名
+#
+# noun-proper-organization: Names representing organizations.
+# e.g. 通産省, NHK
+#名詞-固有名詞-組織
+#
+# noun-proper-place: Place names where the sub-classification is undefined
+#名詞-固有名詞-地域
+#
+# noun-proper-place-misc: Place names excluding countries.
+# e.g. アジア, バルセロナ, 京都
+#名詞-固有名詞-地域-一般
+#
+# noun-proper-place-country: Country names.
+# e.g. 日本, オーストラリア
+#名詞-固有名詞-地域-国
+#
+# noun-pronoun: Pronouns where the sub-classification is undefined
+#名詞-代名詞
+#
+# noun-pronoun-misc: miscellaneous pronouns:
+# e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ
+#名詞-代名詞-一般
+#
+# noun-pronoun-contraction: Spoken language contraction made by combining a
+# pronoun and the particle 'wa'.
+# e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ
+#名詞-代名詞-縮約
+#
+# noun-adverbial: Temporal nouns such as names of days or months that behave
+# like adverbs. Nouns that represent amount or ratios and can be used adverbially,
+# e.g. 金曜, 一月, 午後, 少量
+#名詞-副詞可能
+#
+# noun-verbal: Nouns that take arguments with case and can appear followed by
+# 'suru' and related verbs (する, できる, なさる, くださる)
+# e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り
+#名詞-サ変接続
+#
+# noun-adjective-base: The base form of adjectives, words that appear before な ("na")
+# e.g. 健康, 安易, 駄目, だめ
+#名詞-形容動詞語幹
+#
+# noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数.
+# e.g. 0, 1, 2, 何, 数, 幾
+#名詞-数
+#
+# noun-affix: noun affixes where the sub-classification is undefined
+#名詞-非自立
+#
+# noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that
+# attach to the base form of inflectional words, words that cannot be classified
+# into any of the other categories below. This category includes indefinite nouns.
+# e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第,
+# 順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み,
+# 拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳,
+# わり, 割り, 割, ん-口語/, もん-口語/
+#名詞-非自立-一般
+#
+# noun-affix-adverbial: noun affixes that that can behave as adverbs.
+# e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ,
+# 上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか,
+# 最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所,
+# とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま,
+# 儘, 侭, みぎり, 矢先
+#名詞-非自立-副詞可能
+#
+# noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars
+# with the stem よう(だ) ("you(da)").
+# e.g. よう, やう, 様 (よう)
+#名詞-非自立-助動詞語幹
+#
+# noun-affix-adjective-base: noun affixes that can connect to the indeclinable
+# connection form な (aux "da").
+# e.g. みたい, ふう
+#名詞-非自立-形容動詞語幹
+#
+# noun-special: special nouns where the sub-classification is undefined.
+#名詞-特殊
+#
+# noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is
+# treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base
+# form of inflectional words.
+# e.g. そう
+#名詞-特殊-助動詞語幹
+#
+# noun-suffix: noun suffixes where the sub-classification is undefined.
+#名詞-接尾
+#
+# noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect
+# to ガル or タイ and can combine into compound nouns, words that cannot be classified into
+# any of the other categories below. In general, this category is more inclusive than
+# 接尾語 ("suffix") and is usually the last element in a compound noun.
+# e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み,
+# よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用
+#名詞-接尾-一般
+#
+# noun-suffix-person: Suffixes that form nouns and attach to person names more often
+# than other nouns.
+# e.g. 君, 様, 著
+#名詞-接尾-人名
+#
+# noun-suffix-place: Suffixes that form nouns and attach to place names more often
+# than other nouns.
+# e.g. 町, 市, 県
+#名詞-接尾-地域
+#
+# noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that
+# can appear before スル ("suru").
+# e.g. 化, 視, 分け, 入り, 落ち, 買い
+#名詞-接尾-サ変接続
+#
+# noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions,
+# is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the
+# conjunctive form of inflectional words.
+# e.g. そう
+#名詞-接尾-助動詞語幹
+#
+# noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive
+# form of inflectional words and appear before the copula だ ("da").
+# e.g. 的, げ, がち
+#名詞-接尾-形容動詞語幹
+#
+# noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs.
+# e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ)
+#名詞-接尾-副詞可能
+#
+# noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category
+# is more inclusive than 助数詞 ("classifier") and includes common nouns that attach
+# to numbers.
+# e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半
+#名詞-接尾-助数詞
+#
+# noun-suffix-special: Special suffixes that mainly attach to inflecting words.
+# e.g. (楽し) さ, (考え) 方
+#名詞-接尾-特殊
+#
+# noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words
+# together.
+# e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦)
+#名詞-接続詞的
+#
+# noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are
+# semantically verb-like.
+# e.g. ごらん, ご覧, 御覧, 頂戴
+#名詞-動詞非自立的
+#
+# noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry,
+# dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation")
+# is いわく ("iwaku").
+#名詞-引用文字列
+#
+# noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and
+# behave like an adjective.
+# e.g. 申し訳, 仕方, とんでも, 違い
+#名詞-ナイ形容詞語幹
+#
+#####
+# prefix: unclassified prefixes
+#接頭詞
+#
+# prefix-nominal: Prefixes that attach to nouns (including adjective stem forms)
+# excluding numerical expressions.
+# e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派)
+#接頭詞-名詞接続
+#
+# prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb
+# in conjunctive form followed by なる/なさる/くださる.
+# e.g. お (読みなさい), お (座り)
+#接頭詞-動詞接続
+#
+# prefix-adjectival: Prefixes that attach to adjectives.
+# e.g. お (寒いですねえ), バカ (でかい)
+#接頭詞-形容詞接続
+#
+# prefix-numerical: Prefixes that attach to numerical expressions.
+# e.g. 約, およそ, 毎時
+#接頭詞-数接続
+#
+#####
+# verb: unclassified verbs
+#動詞
+#
+# verb-main:
+#動詞-自立
+#
+# verb-auxiliary:
+#動詞-非自立
+#
+# verb-suffix:
+#動詞-接尾
+#
+#####
+# adjective: unclassified adjectives
+#形容詞
+#
+# adjective-main:
+#形容詞-自立
+#
+# adjective-auxiliary:
+#形容詞-非自立
+#
+# adjective-suffix:
+#形容詞-接尾
+#
+#####
+# adverb: unclassified adverbs
+#副詞
+#
+# adverb-misc: Words that can be segmented into one unit and where adnominal
+# modification is not possible.
+# e.g. あいかわらず, 多分
+#副詞-一般
+#
+# adverb-particle_conjunction: Adverbs that can be followed by の, は, に,
+# な, する, だ, etc.
+# e.g. こんなに, そんなに, あんなに, なにか, なんでも
+#副詞-助詞類接続
+#
+#####
+# adnominal: Words that only have noun-modifying forms.
+# e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう,
+# どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした,
+# 「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き
+#連体詞
+#
+#####
+# conjunction: Conjunctions that can occur independently.
+# e.g. が, けれども, そして, じゃあ, それどころか
+接続詞
+#
+#####
+# particle: unclassified particles.
+助詞
+#
+# particle-case: case particles where the subclassification is undefined.
+助詞-格助詞
+#
+# particle-case-misc: Case particles.
+# e.g. から, が, で, と, に, へ, より, を, の, にて
+助詞-格助詞-一般
+#
+# particle-case-quote: the "to" that appears after nouns, a person’s speech,
+# quotation marks, expressions of decisions from a meeting, reasons, judgements,
+# conjectures, etc.
+# e.g. ( だ) と (述べた.), ( である) と (して執行猶予...)
+助詞-格助詞-引用
+#
+# particle-case-compound: Compounds of particles and verbs that mainly behave
+# like case particles.
+# e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って,
+# にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける,
+# にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し,
+# に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして,
+# に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって,
+# にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る,
+# にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる,
+# って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ
+助詞-格助詞-連語
+#
+# particle-conjunctive:
+# e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども,
+# ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/,
+# (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/
+助詞-接続助詞
+#
+# particle-dependency:
+# e.g. こそ, さえ, しか, すら, は, も, ぞ
+助詞-係助詞
+#
+# particle-adverbial:
+# e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/,
+# (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/,
+# (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに,
+# (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/,
+# ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」)
+助詞-副助詞
+#
+# particle-interjective: particles with interjective grammatical roles.
+# e.g. (松島) や
+助詞-間投助詞
+#
+# particle-coordinate:
+# e.g. と, たり, だの, だり, とか, なり, や, やら
+助詞-並立助詞
+#
+# particle-final:
+# e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ,
+# ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/
+助詞-終助詞
+#
+# particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is
+# adverbial, conjunctive, or sentence final. For example:
+# (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」
+# (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」
+# 「(祈りが届いたせい) か (, 試験に合格した.)」
+# (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」
+# e.g. か
+助詞-副助詞／並立助詞／終助詞
+#
+# particle-adnominalizer: The "no" that attaches to nouns and modifies
+# non-inflectional words.
+助詞-連体化
+#
+# particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs
+# that are giongo, giseigo, or gitaigo.
+# e.g. に, と
+助詞-副詞化
+#
+# particle-special: A particle that does not fit into one of the above classifications.
+# This includes particles that are used in Tanka, Haiku, and other poetry.
+# e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家)
+助詞-特殊
+#
+#####
+# auxiliary-verb:
+助動詞
+#
+#####
+# interjection: Greetings and other exclamations.
+# e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます,
+# いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい
+#感動詞
+#
+#####
+# symbol: unclassified Symbols.
+記号
+#
+# symbol-misc: A general symbol not in one of the categories below.
+# e.g. [○◎@$〒→+]
+記号-一般
+#
+# symbol-comma: Commas
+# e.g. [,、]
+記号-読点
+#
+# symbol-period: Periods and full stops.
+# e.g. [.．。]
+記号-句点
+#
+# symbol-space: Full-width whitespace.
+記号-空白
+#
+# symbol-open_bracket:
+# e.g. [({‘“『【]
+記号-括弧開
+#
+# symbol-close_bracket:
+# e.g. [)}’”』」】]
+記号-括弧閉
+#
+# symbol-alphabetic:
+#記号-アルファベット
+#
+#####
+# other: unclassified other
+#その他
+#
+# other-interjection: Words that are hard to classify as noun-suffixes or
+# sentence-final particles.
+# e.g. (だ)ァ
+その他-間投
+#
+#####
+# filler: Aizuchi that occurs during a conversation or sounds inserted as filler.
+# e.g. あの, うんと, えと
+フィラー
+#
+#####
+# non-verbal: non-verbal sound.
+非言語音
+#
+#####
+# fragment:
+#語断片
+#
+#####
+# unknown: unknown part of speech.
+#未知語
+#
+##### End of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Analysis.Kuromoji/stopwords.txt
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Kuromoji/stopwords.txt b/src/Lucene.Net.Analysis.Kuromoji/stopwords.txt
new file mode 100644
index 0000000..d4321be
--- /dev/null
+++ b/src/Lucene.Net.Analysis.Kuromoji/stopwords.txt
@@ -0,0 +1,127 @@
+#
+# This file defines a stopword set for Japanese.
+#
+# This set is made up of hand-picked frequent terms from segmented Japanese Wikipedia.
+# Punctuation characters and frequent kanji have mostly been left out. See LUCENE-3745
+# for frequency lists, etc. that can be useful for making your own set (if desired)
+#
+# Note that there is an overlap between these stopwords and the terms stopped when used
+# in combination with the JapanesePartOfSpeechStopFilter. When editing this file, note
+# that comments are not allowed on the same line as stopwords.
+#
+# Also note that stopping is done in a case-insensitive manner. Change your StopFilter
+# configuration if you need case-sensitive stopping. Lastly, note that stopping is done
+# using the same character width as the entries in this file. Since this StopFilter is
+# normally done after a CJKWidthFilter in your chain, you would usually want your romaji
+# entries to be in half-width and your kana entries to be in full-width.
+#
+の
+に
+は
+を
+た
+が
+で
+て
+と
+し
+れ
+さ
+ある
+いる
+も
+する
+から
+な
+こと
+として
+い
+や
+れる
+など
+なっ
+ない
+この
+ため
+その
+あっ
+よう
+また
+もの
+という
+あり
+まで
+られ
+なる
+へ
+か
+だ
+これ
+によって
+により
+おり
+より
+による
+ず
+なり
+られる
+において
+ば
+なかっ
+なく
+しかし
+について
+せ
+だっ
+その後
+できる
+それ
+う
+ので
+なお
+のみ
+でき
+き
+つ
+における
+および
+いう
+さらに
+でも
+ら
+たり
+その他
+に関する
+たち
+ます
+ん
+なら
+に対して
+特に
+せる
+及び
+これら
+とき
+では
+にて
+ほか
+ながら
+うち
+そして
+とともに
+ただし
+かつて
+それぞれ
+または
+お
+ほど
+ものの
+に対する
+ほとんど
+と共に
+といった
+です
+とも
+ところ
+ここ
+##### End of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/TestTokenInfoDictionary.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/TestTokenInfoDictionary.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/TestTokenInfoDictionary.cs
new file mode 100644
index 0000000..dd305a4
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/TestTokenInfoDictionary.cs
@@ -0,0 +1,114 @@
+using Lucene.Net.Analysis.Ja.Util;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Fst;
+using NUnit.Framework;
+using System;
+
+namespace Lucene.Net.Analysis.Ja.Dict
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class TestTokenInfoDictionary : LuceneTestCase
+ {
+ /// <summary>enumerates the entire FST/lookup data and just does basic sanity checks</summary>
+ [Test]
+ public void TestEnumerateAll()
+ {
+ // just for debugging
+ int numTerms = 0;
+ int numWords = 0;
+ int lastWordId = -1;
+ int lastSourceId = -1;
+ TokenInfoDictionary tid = TokenInfoDictionary.GetInstance();
+ ConnectionCosts matrix = ConnectionCosts.GetInstance();
+ FST<long?> fst = tid.FST.InternalFST;
+ Int32sRefFSTEnum<long?> fstEnum = new Int32sRefFSTEnum<long?>(fst);
+ Int32sRefFSTEnum.InputOutput<long?> mapping;
+ Int32sRef scratch = new Int32sRef();
+ while ((mapping = fstEnum.Next()) != null)
+ {
+ numTerms++;
+ Int32sRef input = mapping.Input;
+ char[] chars = new char[input.Length];
+ for (int i = 0; i < chars.Length; i++)
+ {
+ chars[i] = (char)input.Int32s[input.Offset + i];
+ }
+ assertTrue(UnicodeUtil.ValidUTF16String(new string(chars)));
+
+ long? output = mapping.Output;
+ int sourceId = (int)output.Value;
+ // we walk in order, terms, sourceIds, and wordIds should always be increasing
+ assertTrue(sourceId > lastSourceId);
+ lastSourceId = sourceId;
+ tid.LookupWordIds(sourceId, scratch);
+ for (int i = 0; i < scratch.Length; i++)
+ {
+ numWords++;
+ int wordId = scratch.Int32s[scratch.Offset + i];
+ assertTrue(wordId > lastWordId);
+ lastWordId = wordId;
+
+ String baseForm = tid.GetBaseForm(wordId, chars, 0, chars.Length);
+ assertTrue(baseForm == null || UnicodeUtil.ValidUTF16String(baseForm));
+
+ String inflectionForm = tid.GetInflectionForm(wordId);
+ assertTrue(inflectionForm == null || UnicodeUtil.ValidUTF16String(inflectionForm));
+ if (inflectionForm != null)
+ {
+ // check that its actually an ipadic inflection form
+ assertNotNull(ToStringUtil.GetInflectedFormTranslation(inflectionForm));
+ }
+
+ String inflectionType = tid.GetInflectionType(wordId);
+ assertTrue(inflectionType == null || UnicodeUtil.ValidUTF16String(inflectionType));
+ if (inflectionType != null)
+ {
+ // check that its actually an ipadic inflection type
+ assertNotNull(ToStringUtil.GetInflectionTypeTranslation(inflectionType));
+ }
+
+ int leftId = tid.GetLeftId(wordId);
+ int rightId = tid.GetRightId(wordId);
+
+ matrix.Get(rightId, leftId);
+
+ tid.GetWordCost(wordId);
+
+ String pos = tid.GetPartOfSpeech(wordId);
+ assertNotNull(pos);
+ assertTrue(UnicodeUtil.ValidUTF16String(pos));
+ // check that its actually an ipadic pos tag
+ assertNotNull(ToStringUtil.GetPOSTranslation(pos));
+
+ String pronunciation = tid.GetPronunciation(wordId, chars, 0, chars.Length);
+ assertNotNull(pronunciation);
+ assertTrue(UnicodeUtil.ValidUTF16String(pronunciation));
+
+ String reading = tid.GetReading(wordId, chars, 0, chars.Length);
+ assertNotNull(reading);
+ assertTrue(UnicodeUtil.ValidUTF16String(reading));
+ }
+ }
+ if (VERBOSE)
+ {
+ Console.WriteLine("checked " + numTerms + " terms, " + numWords + " words.");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/UserDictionaryTest.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/UserDictionaryTest.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/UserDictionaryTest.cs
new file mode 100644
index 0000000..f899476
--- /dev/null
+++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Dict/UserDictionaryTest.cs
@@ -0,0 +1,90 @@
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+
+namespace Lucene.Net.Analysis.Ja.Dict
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ public class UserDictionaryTest : LuceneTestCase
+ {
+ [Test]
+ public void TestLookup()
+ {
+ UserDictionary dictionary = TestJapaneseTokenizer.ReadDict();
+ String s = "関西国際空港に行った";
+ int[][] dictionaryEntryResult = dictionary.Lookup(s.toCharArray(), 0, s.Length);
+ // Length should be three 関西, 国際, 空港
+ assertEquals(3, dictionaryEntryResult.Length);
+
+ // Test positions
+ assertEquals(0, dictionaryEntryResult[0][1]); // index of 関西
+ assertEquals(2, dictionaryEntryResult[1][1]); // index of 国際
+ assertEquals(4, dictionaryEntryResult[2][1]); // index of 空港
+
+ // Test lengths
+ assertEquals(2, dictionaryEntryResult[0][2]); // length of 関西
+ assertEquals(2, dictionaryEntryResult[1][2]); // length of 国際
+ assertEquals(2, dictionaryEntryResult[2][2]); // length of 空港
+
+ s = "関西国際空港と関西国際空港に行った";
+ int[][] dictionaryEntryResult2 = dictionary.Lookup(s.toCharArray(), 0, s.Length);
+ // Length should be six
+ assertEquals(6, dictionaryEntryResult2.Length);
+ }
+
+ [Test]
+ public void TestReadings()
+ {
+ UserDictionary dictionary = TestJapaneseTokenizer.ReadDict();
+ int[]
+ []
+ result = dictionary.Lookup("日本経済新聞".toCharArray(), 0, 6);
+ assertEquals(3, result.Length);
+ int wordIdNihon = result[0]
+ [0]; // wordId of 日本 in 日本経済新聞
+ assertEquals("ニホン", dictionary.GetReading(wordIdNihon, "日本".toCharArray(), 0, 2));
+
+ result = dictionary.Lookup("朝青龍".toCharArray(), 0, 3);
+ assertEquals(1, result.Length);
+ int wordIdAsashoryu = result[0]
+ [0]; // wordId for 朝青龍
+ assertEquals("アサショウリュウ", dictionary.GetReading(wordIdAsashoryu, "朝青龍".toCharArray(), 0, 3));
+ }
+
+ [Test]
+ public void TestPartOfSpeech()
+ {
+ UserDictionary dictionary = TestJapaneseTokenizer.ReadDict();
+ int[]
+ []
+ result = dictionary.Lookup("日本経済新聞".toCharArray(), 0, 6);
+ assertEquals(3, result.Length);
+ int wordIdKeizai = result[1]
+ [0]; // wordId of 経済 in 日本経済新聞
+ assertEquals("カスタム名詞", dictionary.GetPartOfSpeech(wordIdKeizai));
+ }
+
+ [Test]
+ public void TestRead()
+ {
+ UserDictionary dictionary = TestJapaneseTokenizer.ReadDict();
+ assertNotNull(dictionary);
+ }
+ }
+}