svn commit: r1230748 [4/5] - in /lucene/dev/trunk: dev-tools/eclipse/ lucene/contrib/ modules/analysis/ modules/analysis/common/src/java/org/apache/lucene/analysis/util/ modules/analysis/common/src/test/org/apache/lucene/analysis/util/ modules/analysis...

[rm...@apache.org]

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stoptags.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stoptags.txt?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stoptags.txt (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stoptags.txt Thu Jan 12 20:10:48 2012
@@ -0,0 +1,410 @@
+# set of default stop tags:
+# uncomment a part of speech to treat those words as stopwords.
+# the entire tagset is provided here for convenience.
+#
+#####
+#  noun: unclassified nouns
+#名詞
+#
+#  noun-common: Common nouns or nouns where the sub-classification is undefined
+#名詞-一般
+#
+#  noun-proper: Proper nouns where the sub-classification is undefined 
+#名詞-固有名詞
+#
+#  noun-proper-misc: miscellaneous proper nouns
+#名詞-固有名詞-一般
+#
+#  noun-proper-person: Personal names where the sub-classification is undefined
+#名詞-固有名詞-人名
+#
+#  noun-proper-person-misc: names that cannot be divided into surname and 
+#  given name; foreign names; names where the surname or given name is unknown.
+#  e.g. お市の方
+#名詞-固有名詞-人名-一般
+#
+#  noun-proper-person-surname: Mainly Japanese surnames.
+#  e.g. 山田
+#名詞-固有名詞-人名-姓
+#
+#  noun-proper-person-given_name: Mainly Japanese given names.
+#  e.g. 太郎
+#名詞-固有名詞-人名-名
+#
+#  noun-proper-organization: Names representing organizations.
+#  e.g. 通産省, NHK
+#名詞-固有名詞-組織
+#
+#  noun-proper-place: Place names where the sub-classification is undefined
+#名詞-固有名詞-地域
+#
+#  noun-proper-place-misc: Place names excluding countries.
+#  e.g. アジア, バルセロナ, 京都
+#名詞-固有名詞-地域-一般
+#
+#  noun-proper-place-country: Country names. 
+#  e.g. 日本, オーストラリア
+#名詞-固有名詞-地域-国
+#
+#  noun-pronoun: Pronouns where the sub-classification is undefined
+#名詞-代名詞
+#
+#  noun-pronoun-misc: miscellaneous pronouns: 
+#  e.g. それ, ここ, あいつ, あなた, あちこち, いくつ, どこか, なに, みなさん, みんな, わたくし, われわれ
+#名詞-代名詞-一般
+#
+#  noun-pronoun-contraction: Spoken language contraction made by combining a 
+#  pronoun and the particle 'wa'.
+#  e.g. ありゃ, こりゃ, こりゃあ, そりゃ, そりゃあ 
+#名詞-代名詞-縮約
+#
+#  noun-adverbial: Temporal nouns such as names of days or months that behave 
+#  like adverbs. Nouns that represent amount or ratios and can be used adverbially,
+#  e.g. 金曜, 一月, 午後, 少量
+#名詞-副詞可能
+#
+#  noun-verbal: Nouns that take arguments with case and can appear followed by 
+#  'suru' and related verbs (する, できる, なさる, くださる)
+#  e.g. インプット, 愛着, 悪化, 悪戦苦闘, 一安心, 下取り
+#名詞-サ変接続
+#
+#  noun-adjective-base: The base form of adjectives, words that appear before な ("na")
+#  e.g. 健康, 安易, 駄目, だめ
+#名詞-形容動詞語幹
+#
+#  noun-numeric: Arabic numbers, Chinese numerals, and counters like 何 (回), 数.
+#  e.g. 0, 1, 2, 何, 数, 幾
+#名詞-数
+#
+#  noun-affix: noun affixes where the sub-classification is undefined
+#名詞-非自立
+#
+#  noun-affix-misc: Of adnominalizers, the case-marker の ("no"), and words that 
+#  attach to the base form of inflectional words, words that cannot be classified 
+#  into any of the other categories below. This category includes indefinite nouns.
+#  e.g. あかつき, 暁, かい, 甲斐, 気, きらい, 嫌い, くせ, 癖, こと, 事, ごと, 毎, しだい, 次第, 
+#       順, せい, 所為, ついで, 序で, つもり, 積もり, 点, どころ, の, はず, 筈, はずみ, 弾み, 
+#       拍子, ふう, ふり, 振り, ほう, 方, 旨, もの, 物, 者, ゆえ, 故, ゆえん, 所以, わけ, 訳,
+#       わり, 割り, 割, ん-口語/, もん-口語/
+#名詞-非自立-一般
+#
+#  noun-affix-adverbial: noun affixes that that can behave as adverbs.
+#  e.g. あいだ, 間, あげく, 挙げ句, あと, 後, 余り, 以外, 以降, 以後, 以上, 以前, 一方, うえ, 
+#       上, うち, 内, おり, 折り, かぎり, 限り, きり, っきり, 結果, ころ, 頃, さい, 際, 最中, さなか, 
+#       最中, じたい, 自体, たび, 度, ため, 為, つど, 都度, とおり, 通り, とき, 時, ところ, 所, 
+#       とたん, 途端, なか, 中, のち, 後, ばあい, 場合, 日, ぶん, 分, ほか, 他, まえ, 前, まま, 
+#       儘, 侭, みぎり, 矢先
+#名詞-非自立-副詞可能
+#
+#  noun-affix-aux: noun affixes treated as 助動詞 ("auxiliary verb") in school grammars 
+#  with the stem よう(だ) ("you(da)").
+#  e.g.  よう, やう, 様 (よう)
+#名詞-非自立-助動詞語幹
+#  
+#  noun-affix-adjective-base: noun affixes that can connect to the indeclinable
+#  connection form な (aux "da").
+#  e.g. みたい, ふう
+#名詞-非自立-形容動詞語幹
+#
+#  noun-special: special nouns where the sub-classification is undefined.
+#名詞-特殊
+#
+#  noun-special-aux: The そうだ ("souda") stem form that is used for reporting news, is 
+#  treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the base 
+#  form of inflectional words.
+#  e.g. そう
+#名詞-特殊-助動詞語幹
+#
+#  noun-suffix: noun suffixes where the sub-classification is undefined.
+#名詞-接尾
+#
+#  noun-suffix-misc: Of the nouns or stem forms of other parts of speech that connect 
+#  to ガル or タイ and can combine into compound nouns, words that cannot be classified into
+#  any of the other categories below. In general, this category is more inclusive than 
+#  接尾語 ("suffix") and is usually the last element in a compound noun.
+#  e.g. おき, かた, 方, 甲斐 (がい), がかり, ぎみ, 気味, ぐるみ, (～した) さ, 次第, 済 (ず) み,
+#       よう, (でき)っこ, 感, 観, 性, 学, 類, 面, 用
+#名詞-接尾-一般
+#
+#  noun-suffix-person: Suffixes that form nouns and attach to person names more often
+#  than other nouns.
+#  e.g. 君, 様, 著
+#名詞-接尾-人名
+#
+#  noun-suffix-place: Suffixes that form nouns and attach to place names more often 
+#  than other nouns.
+#  e.g. 町, 市, 県
+#名詞-接尾-地域
+#
+#  noun-suffix-verbal: Of the suffixes that attach to nouns and form nouns, those that 
+#  can appear before スル ("suru").
+#  e.g. 化, 視, 分け, 入り, 落ち, 買い
+#名詞-接尾-サ変接続
+#
+#  noun-suffix-aux: The stem form of そうだ (様態) that is used to indicate conditions, 
+#  is treated as 助動詞 ("auxiliary verb") in school grammars, and attach to the 
+#  conjunctive form of inflectional words.
+#  e.g. そう
+#名詞-接尾-助動詞語幹
+#
+#  noun-suffix-adjective-base: Suffixes that attach to other nouns or the conjunctive 
+#  form of inflectional words and appear before the copula だ ("da").
+#  e.g. 的, げ, がち
+#名詞-接尾-形容動詞語幹
+#
+#  noun-suffix-adverbial: Suffixes that attach to other nouns and can behave as adverbs.
+#  e.g. 後 (ご), 以後, 以降, 以前, 前後, 中, 末, 上, 時 (じ)
+#名詞-接尾-副詞可能
+#
+#  noun-suffix-classifier: Suffixes that attach to numbers and form nouns. This category 
+#  is more inclusive than 助数詞 ("classifier") and includes common nouns that attach 
+#  to numbers.
+#  e.g. 個, つ, 本, 冊, パーセント, cm, kg, カ月, か国, 区画, 時間, 時半
+#名詞-接尾-助数詞
+#
+#  noun-suffix-special: Special suffixes that mainly attach to inflecting words.
+#  e.g. (楽し) さ, (考え) 方
+#名詞-接尾-特殊
+#
+#  noun-suffix-conjunctive: Nouns that behave like conjunctions and join two words 
+#  together.
+#  e.g. (日本) 対 (アメリカ), 対 (アメリカ), (3) 対 (5), (女優) 兼 (主婦)
+#名詞-接続詞的
+#
+#  noun-verbal_aux: Nouns that attach to the conjunctive particle て ("te") and are 
+#  semantically verb-like.
+#  e.g. ごらん, ご覧, 御覧, 頂戴
+#名詞-動詞非自立的
+#
+#  noun-quotation: text that cannot be segmented into words, proverbs, Chinese poetry, 
+#  dialects, English, etc. Currently, the only entry for 名詞 引用文字列 ("noun quotation") 
+#  is いわく ("iwaku").
+#名詞-引用文字列
+#
+#  noun-nai_adjective: Words that appear before the auxiliary verb ない ("nai") and
+#  behave like an adjective.
+#  e.g. 申し訳, 仕方, とんでも, 違い
+#名詞-ナイ形容詞語幹
+#
+#####
+#  prefix: unclassified prefixes
+接頭詞
+#
+#  prefix-nominal: Prefixes that attach to nouns (including adjective stem forms) 
+#  excluding numerical expressions.
+#  e.g. お (水), 某 (氏), 同 (社), 故 (～氏), 高 (品質), お (見事), ご (立派)
+接頭詞-名詞接続
+#
+#  prefix-verbal: Prefixes that attach to the imperative form of a verb or a verb
+#  in conjunctive form followed by なる/なさる/くださる.
+#  e.g. お (読みなさい), お (座り)
+接頭詞-動詞接続
+#
+#  prefix-adjectival: Prefixes that attach to adjectives.
+#  e.g. お (寒いですねえ), バカ (でかい)
+接頭詞-形容詞接続
+#
+#  prefix-numerical: Prefixes that attach to numerical expressions.
+#  e.g. 約, およそ, 毎時
+接頭詞-数接続
+#
+#####
+#  verb: unclassified verbs
+#動詞
+#
+#  verb-main:
+#動詞-自立
+#
+#  verb-auxiliary:
+動詞-非自立
+#
+#  verb-suffix:
+#動詞-接尾
+#
+#####
+#  adjective: unclassified adjectives
+#形容詞
+#
+#  adjective-main:
+#形容詞-自立
+#
+#  adjective-auxiliary:
+#形容詞-非自立
+#
+#  adjective-suffix:
+#形容詞-接尾
+#
+#####
+#  adverb: unclassified adverbs
+#副詞
+#
+#  adverb-misc: Words that can be segmented into one unit and where adnominal 
+#  modification is not possible.
+#  e.g. あいかわらず, 多分
+#副詞-一般
+#
+#  adverb-particle_conjunction: Adverbs that can be followed by の, は, に, 
+#  な, する, だ, etc.
+#  e.g. こんなに, そんなに, あんなに, なにか, なんでも
+#副詞-助詞類接続
+#
+#####
+#  adnominal: Words that only have noun-modifying forms.
+#  e.g. この, その, あの, どの, いわゆる, なんらかの, 何らかの, いろんな, こういう, そういう, ああいう, 
+#       どういう, こんな, そんな, あんな, どんな, 大きな, 小さな, おかしな, ほんの, たいした, 
+#       「(, も) さる (ことながら)」, 微々たる, 堂々たる, 単なる, いかなる, 我が」「同じ, 亡き
+#連体詞
+#
+#####
+#  conjunction: Conjunctions that can occur independently.
+#  e.g. が, けれども, そして, じゃあ, それどころか
+接続詞
+#
+#####
+#  particle: unclassified particles.
+助詞
+#
+#  particle-case: case particles where the subclassification is undefined.
+助詞-格助詞
+#
+#  particle-case-misc: Case particles.
+#  e.g. から, が, で, と, に, へ, より, を, の, にて
+助詞-格助詞-一般
+#
+#  particle-case-quote: the "to" that appears after nouns, a person’s speech, 
+#  quotation marks, expressions of decisions from a meeting, reasons, judgements,
+#  conjectures, etc.
+#  e.g. ( だ) と (述べた.), ( である) と (して執行猶予...)
+助詞-格助詞-引用
+#
+#  particle-case-compound: Compounds of particles and verbs that mainly behave 
+#  like case particles.
+#  e.g. という, といった, とかいう, として, とともに, と共に, でもって, にあたって, に当たって, に当って,
+#       にあたり, に当たり, に当り, に当たる, にあたる, において, に於いて,に於て, における, に於ける, 
+#       にかけ, にかけて, にかんし, に関し, にかんして, に関して, にかんする, に関する, に際し, 
+#       に際して, にしたがい, に従い, に従う, にしたがって, に従って, にたいし, に対し, にたいして, 
+#       に対して, にたいする, に対する, について, につき, につけ, につけて, につれ, につれて, にとって,
+#       にとり, にまつわる, によって, に依って, に因って, により, に依り, に因り, による, に依る, に因る, 
+#       にわたって, にわたる, をもって, を以って, を通じ, を通じて, を通して, をめぐって, をめぐり, をめぐる,
+#       って-口語/, ちゅう-関西弁「という」/, (何) ていう (人)-口語/, っていう-口語/, といふ, とかいふ
+助詞-格助詞-連語
+#
+#  particle-conjunctive:
+#  e.g. から, からには, が, けれど, けれども, けど, し, つつ, て, で, と, ところが, どころか, とも, ども, 
+#       ながら, なり, ので, のに, ば, ものの, や ( した), やいなや, (ころん) じゃ(いけない)-口語/, 
+#       (行っ) ちゃ(いけない)-口語/, (言っ) たって (しかたがない)-口語/, (それがなく)ったって (平気)-口語/
+助詞-接続助詞
+#
+#  particle-dependency:
+#  e.g. こそ, さえ, しか, すら, は, も, ぞ
+助詞-係助詞
+#
+#  particle-adverbial:
+#  e.g. がてら, かも, くらい, 位, ぐらい, しも, (学校) じゃ(これが流行っている)-口語/, 
+#       (それ)じゃあ (よくない)-口語/, ずつ, (私) なぞ, など, (私) なり (に), (先生) なんか (大嫌い)-口語/,
+#       (私) なんぞ, (先生) なんて (大嫌い)-口語/, のみ, だけ, (私) だって-口語/, だに, 
+#       (彼)ったら-口語/, (お茶) でも (いかが), 等 (とう), (今後) とも, ばかり, ばっか-口語/, ばっかり-口語/,
+#       ほど, 程, まで, 迄, (誰) も (が)([助詞-格助詞] および [助詞-係助詞] の前に位置する「も」)
+助詞-副助詞
+#
+#  particle-interjective: particles with interjective grammatical roles.
+#  e.g. (松島) や
+助詞-間投助詞
+#
+#  particle-coordinate:
+#  e.g. と, たり, だの, だり, とか, なり, や, やら
+助詞-並立助詞
+#
+#  particle-final:
+#  e.g. かい, かしら, さ, ぜ, (だ)っけ-口語/, (とまってる) で-方言/, な, ナ, なあ-口語/, ぞ, ね, ネ, 
+#       ねぇ-口語/, ねえ-口語/, ねん-方言/, の, のう-口語/, や, よ, ヨ, よぉ-口語/, わ, わい-口語/
+助詞-終助詞
+#
+#  particle-adverbial/conjunctive/final: The particle "ka" when unknown whether it is 
+#  adverbial, conjunctive, or sentence final. For example:
+#       (a) 「A か B か」. Ex:「(国内で運用する) か,(海外で運用する) か (.)」
+#       (b) Inside an adverb phrase. Ex:「(幸いという) か (, 死者はいなかった.)」
+#           「(祈りが届いたせい) か (, 試験に合格した.)」
+#       (c) 「かのように」. Ex:「(何もなかった) か (のように振る舞った.)」
+#  e.g. か
+助詞-副助詞／並立助詞／終助詞
+#
+#  particle-adnominalizer: The "no" that attaches to nouns and modifies 
+#  non-inflectional words.
+助詞-連体化
+#
+#  particle-adnominalizer: The "ni" and "to" that appear following nouns and adverbs 
+#  that are giongo, giseigo, or gitaigo.
+#  e.g. に, と
+助詞-副詞化
+#
+#  particle-special: A particle that does not fit into one of the above classifications. 
+#  This includes particles that are used in Tanka, Haiku, and other poetry.
+#  e.g. かな, けむ, ( しただろう) に, (あんた) にゃ(わからん), (俺) ん (家)
+助詞-特殊
+#
+#####
+#  auxiliary-verb:
+助動詞
+#
+#####
+#  interjection: Greetings and other exclamations.
+#  e.g. おはよう, おはようございます, こんにちは, こんばんは, ありがとう, どうもありがとう, ありがとうございます, 
+#       いただきます, ごちそうさま, さよなら, さようなら, はい, いいえ, ごめん, ごめんなさい
+感動詞
+#
+#####
+#  symbol: unclassified Symbols.
+#記号
+#
+#  symbol-misc: A general symbol not in one of the categories below.
+#  e.g. [○◎@$〒→+]
+記号-一般
+#
+#  symbol-comma: Commas
+#  e.g. [,、]
+記号-読点
+#
+#  symbol-period: Periods and full stops.
+#  e.g. [.．。]
+記号-句点
+#
+#  symbol-space: Full-width whitespace.
+記号-空白
+#
+#  symbol-open_bracket:
+#  e.g. [({‘“『【]
+記号-括弧開
+#
+#  symbol-close_bracket:
+#  e.g. [)}’”』」】]
+記号-括弧閉
+#
+#  symbol-alphabetic:
+#記号-アルファベット
+#
+#####
+#  other: unclassified other
+#その他
+#
+#  other-interjection: Words that are hard to classify as noun-suffixes or 
+#  sentence-final particles.
+#  e.g. (だ)ァ
+その他-間投
+#
+#####
+#  filler: Aizuchi that occurs during a conversation or sounds inserted as filler.
+#  e.g. あの, うんと, えと
+フィラー
+#
+#####
+#  non-verbal: non-verbal sound.
+非言語音
+#
+#####
+#  fragment:
+#語断片
+#
+#####
+#  unknown: unknown part of speech.
+#未知語

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stopwords.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stopwords.txt?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stopwords.txt (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/stopwords.txt Thu Jan 12 20:10:48 2012
@@ -0,0 +1,13 @@
+# short set of japanese stopwords
+いう
+する
+人物
+さま
+すること
+ため
+もの
+おいて
+なる
+できる
+おく
+ある

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,231 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.util.List;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class SegmenterTest extends LuceneTestCase {
+  
+  private static Segmenter segmenter;
+  
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    segmenter = new Segmenter();
+  }
+  
+  @AfterClass
+  public static void afterClass() throws Exception {
+    segmenter = null;
+  }
+  
+  @Test
+  public void testSegmentation() {
+    // Skip tests for Michelle Kwan -- UniDic segments Kwan as ク ワン
+    //		String input = "ミシェル・クワンが優勝しました。スペースステーションに行きます。うたがわしい。";
+    //		String[] surfaceForms = {
+    //				"ミシェル", "・", "クワン", "が", "優勝", "し", "まし", "た", "。",
+    //				"スペース", "ステーション", "に", "行き", "ます", "。",
+    //				"うたがわしい", "。"
+    //		};
+    String input = "スペースステーションに行きます。うたがわしい。";
+    String[] surfaceForms = {
+        "スペース", "ステーション", "に", "行き", "ます", "。",
+        "うたがわしい", "。"
+    };
+    List<Token> tokens = segmenter.tokenize(input);
+    assertTrue(tokens.size() == surfaceForms.length);
+    for (int i = 0; i < tokens.size(); i++) {
+      assertEquals(surfaceForms[i], tokens.get(i).getSurfaceFormString());
+    }
+  }
+  
+  @Test
+  public void testReadings() {
+    List<Token> tokens = segmenter.tokenize("寿司が食べたいです。");
+    assertEquals(6, tokens.size());
+    assertEquals("スシ", tokens.get(0).getReading());
+    assertEquals("ガ",    tokens.get(1).getReading());
+    assertEquals("タベ", tokens.get(2).getReading());
+    assertEquals("タイ",  tokens.get(3).getReading());
+    assertEquals("デス", tokens.get(4).getReading());
+    assertEquals("。", tokens.get(5).getReading());
+  }
+  
+  @Test
+  public void testReadings2() {
+    List<Token> tokens = segmenter.tokenize("多くの学生が試験に落ちた。");
+    assertEquals(9, tokens.size());
+    assertEquals("オオク", tokens.get(0).getReading());
+    assertEquals("ノ", tokens.get(1).getReading());
+    assertEquals("ガクセイ", tokens.get(2).getReading());
+    assertEquals("ガ", tokens.get(3).getReading());
+    assertEquals("シケン", tokens.get(4).getReading());
+    assertEquals("ニ", tokens.get(5).getReading());
+    assertEquals("オチ", tokens.get(6).getReading());
+    assertEquals("タ", tokens.get(7).getReading());
+    assertEquals("。", tokens.get(8).getReading());
+  }
+  
+  @Test
+  public void testPronunciations() {
+    List<Token> tokens = segmenter.tokenize("寿司が食べたいです。");
+    assertEquals(6, tokens.size());
+    assertEquals("スシ", tokens.get(0).getPronunciation());
+    assertEquals("ガ",    tokens.get(1).getPronunciation());
+    assertEquals("タベ", tokens.get(2).getPronunciation());
+    assertEquals("タイ",  tokens.get(3).getPronunciation());
+    assertEquals("デス", tokens.get(4).getPronunciation());
+    assertEquals("。", tokens.get(5).getPronunciation());
+  }
+  
+  @Test
+  public void testPronunciations2() {
+    List<Token> tokens = segmenter.tokenize("多くの学生が試験に落ちた。");
+    assertEquals(9, tokens.size());
+    // pronunciation differs from reading here
+    assertEquals("オーク", tokens.get(0).getPronunciation());
+    assertEquals("ノ", tokens.get(1).getPronunciation());
+    assertEquals("ガクセイ", tokens.get(2).getPronunciation());
+    assertEquals("ガ", tokens.get(3).getPronunciation());
+    assertEquals("シケン", tokens.get(4).getPronunciation());
+    assertEquals("ニ", tokens.get(5).getPronunciation());
+    assertEquals("オチ", tokens.get(6).getPronunciation());
+    assertEquals("タ", tokens.get(7).getPronunciation());
+    assertEquals("。", tokens.get(8).getPronunciation());
+  }
+  
+  @Test
+  public void testBasicForms() {
+    List<Token> tokens = segmenter.tokenize("それはまだ実験段階にあります。");
+    assertEquals(9, tokens.size());
+    assertNull(tokens.get(0).getBaseForm());
+    assertNull(tokens.get(1).getBaseForm());
+    assertNull(tokens.get(2).getBaseForm());
+    assertNull(tokens.get(3).getBaseForm());
+    assertNull(tokens.get(4).getBaseForm());
+    assertNull(tokens.get(5).getBaseForm());
+    assertEquals(tokens.get(6).getBaseForm(), "ある");
+    assertNull(tokens.get(7).getBaseForm());
+    assertNull(tokens.get(8).getBaseForm());
+  }
+  
+  @Test
+  public void testInflectionTypes() {
+    List<Token> tokens = segmenter.tokenize("それはまだ実験段階にあります。");
+    assertEquals(9, tokens.size());
+    assertNull(tokens.get(0).getInflectionType());
+    assertNull(tokens.get(1).getInflectionType());
+    assertNull(tokens.get(2).getInflectionType());
+    assertNull(tokens.get(3).getInflectionType());
+    assertNull(tokens.get(4).getInflectionType());
+    assertNull(tokens.get(5).getInflectionType());
+    assertEquals("五段・ラ行", tokens.get(6).getInflectionType());
+    assertEquals("特殊・マス", tokens.get(7).getInflectionType());
+    assertNull(tokens.get(8).getInflectionType());
+  }
+  
+  @Test
+  public void testInflectionForms() {
+    List<Token> tokens = segmenter.tokenize("それはまだ実験段階にあります。");
+    assertEquals(9, tokens.size());
+    assertNull(tokens.get(0).getInflectionForm());
+    assertNull(tokens.get(1).getInflectionForm());
+    assertNull(tokens.get(2).getInflectionForm());
+    assertNull(tokens.get(3).getInflectionForm());
+    assertNull(tokens.get(4).getInflectionForm());
+    assertNull(tokens.get(5).getInflectionForm());
+    assertEquals("連用形", tokens.get(6).getInflectionForm());
+    assertEquals("基本形", tokens.get(7).getInflectionForm());
+    assertNull(tokens.get(8).getInflectionForm());
+  }
+  
+  @Test
+  public void testPartOfSpeech() {
+    List<Token> tokens = segmenter.tokenize("それはまだ実験段階にあります。");
+    assertEquals(9, tokens.size());
+    assertEquals("名詞-代名詞-一般",  tokens.get(0).getPartOfSpeech());
+    assertEquals("助詞-係助詞",    tokens.get(1).getPartOfSpeech());
+    assertEquals("副詞-助詞類接続", tokens.get(2).getPartOfSpeech());
+    assertEquals("名詞-サ変接続",   tokens.get(3).getPartOfSpeech());
+    assertEquals("名詞-一般",      tokens.get(4).getPartOfSpeech());
+    assertEquals("助詞-格助詞-一般",  tokens.get(5).getPartOfSpeech());
+    assertEquals("動詞-自立",      tokens.get(6).getPartOfSpeech());
+    assertEquals("助動詞",       tokens.get(7).getPartOfSpeech());
+    assertEquals("記号-句点",      tokens.get(8).getPartOfSpeech());
+  }
+
+  // TODO: the next 2 tests are no longer using the first/last word ids, maybe lookup the words and fix?
+  // do we have a possibility to actually lookup the first and last word from dictionary?
+  public void testYabottai() {
+    List<Token> tokens = segmenter.tokenize("やぼったい");
+    assertEquals(1, tokens.size());
+    assertEquals("やぼったい", tokens.get(0).getSurfaceFormString());
+  }
+
+  public void testTsukitosha() {
+    List<Token> tokens = segmenter.tokenize("突き通しゃ");
+    assertEquals(1, tokens.size());
+    assertEquals("突き通しゃ", tokens.get(0).getSurfaceFormString());
+  }
+
+  public void testBocchan() throws Exception {
+    doTestBocchan(1);
+  }
+  
+  @Test @Nightly
+  public void testBocchanBig() throws Exception {
+    doTestBocchan(100);
+  }
+  
+  private void doTestBocchan(int numIterations) throws Exception {
+    LineNumberReader reader = new LineNumberReader(new InputStreamReader(
+        this.getClass().getResourceAsStream("bocchan.utf-8")));
+    
+    String line = reader.readLine();
+    reader.close();
+    
+    if (VERBOSE) {
+      System.out.println("Test for Bocchan without pre-splitting sentences");
+    }
+    long totalStart = System.currentTimeMillis();
+    for (int i = 0; i < numIterations; i++){
+      segmenter.tokenize(line);
+    }
+    if (VERBOSE) {
+      System.out.println("Total time : " + (System.currentTimeMillis() - totalStart));
+      System.out.println("Test for Bocchan with pre-splitting sentences");
+    }
+    String[] sentences = line.split("、|。");
+    totalStart = System.currentTimeMillis();
+    for (int i = 0; i < numIterations; i++) {
+      for (String sentence: sentences) {
+        segmenter.tokenize(sentence);       
+      }
+    }
+    if (VERBOSE) {
+      System.out.println("Total time : " + (System.currentTimeMillis() - totalStart));
+    }
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestExtendedMode.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestExtendedMode.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestExtendedMode.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,63 @@
+package org.apache.lucene.analysis.kuromoji;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util._TestUtil;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class TestExtendedMode extends BaseTokenStreamTestCase {
+  private final Segmenter segmenter = new Segmenter(Mode.EXTENDED);
+  private final Analyzer analyzer = new Analyzer() {
+    
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      Tokenizer tokenizer = new KuromojiTokenizer(segmenter, reader);
+      return new TokenStreamComponents(tokenizer, tokenizer);
+    }
+  };
+  
+  /** simple test for supplementary characters */
+  public void testSurrogates() throws IOException {
+    assertAnalyzesTo(analyzer, "𩬅艱鍟䇹愯瀛",
+      new String[] { "𩬅", "艱", "鍟", "䇹", "愯", "瀛" });
+  }
+  
+  /** random test ensuring we don't ever split supplementaries */
+  public void testSurrogates2() throws IOException {
+    int numIterations = atLeast(10000);
+    for (int i = 0; i < numIterations; i++) {
+      String s = _TestUtil.randomUnicodeString(random, 100);
+      TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+      ts.reset();
+      while (ts.incrementToken()) {
+        assertTrue(UnicodeUtil.validUTF16String(termAtt));
+      }
+    }
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,51 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+
+public class TestKuromojiAnalyzer extends BaseTokenStreamTestCase {
+  /** This test fails with NPE when the 
+   * stopwords file is missing in classpath */
+  public void testResourcesAvailable() {
+    new KuromojiAnalyzer(TEST_VERSION_CURRENT);
+  }
+  
+  /**
+   * An example sentence, test removal of particles, etc by POS,
+   * lemmatization with the basic form, and that position increments
+   * and offsets are correct.
+   */
+  public void testBasics() throws IOException {
+    assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "多くの学生が試験に落ちた。",
+        new String[] { "多く", "学生", "試験", "落ちる" },
+        new int[] { 0, 3, 6,  9 },
+        new int[] { 2, 5, 8, 11 },
+        new int[] { 1, 2, 2,  2 }
+      );
+  }
+  
+  /**
+   * blast random strings against the analyzer
+   */
+  public void testRandom() throws IOException {
+    checkRandomData(random, new KuromojiAnalyzer(TEST_VERSION_CURRENT), atLeast(10000));
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiBaseFormFilter.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiBaseFormFilter.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiBaseFormFilter.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,50 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Tokenizer;
+
+public class TestKuromojiBaseFormFilter extends BaseTokenStreamTestCase {
+  private Analyzer analyzer = new Analyzer() {
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      Tokenizer tokenizer = new KuromojiTokenizer(reader);
+      return new TokenStreamComponents(tokenizer, new KuromojiBaseFormFilter(tokenizer));
+    }
+  };
+  
+  public void testBasics() throws IOException {
+    assertAnalyzesTo(analyzer, "それはまだ実験段階にあります",
+        new String[] { "それ", "は", "まだ", "実験", "段階", "に", "ある", "ます"  }
+    );
+  }
+  
+  public void testEnglish() throws IOException {
+    assertAnalyzesTo(analyzer, "this atest",
+        new String[] { "this", "atest" });
+  }
+  
+  public void testRandomStrings() throws IOException {
+    checkRandomData(random, analyzer, atLeast(10000));
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,150 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util._TestUtil;
+
+public class TestKuromojiTokenizer extends BaseTokenStreamTestCase {
+  private Analyzer analyzer = new Analyzer() {
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      Tokenizer tokenizer = new KuromojiTokenizer(reader);
+      return new TokenStreamComponents(tokenizer, tokenizer);
+    }
+  };
+  
+  public void testDecomposition1() throws Exception {
+    assertAnalyzesTo(analyzer, "本来は、貧困層の女性や子供に医療保護を提供するために創設された制度である、" +
+                         "アメリカ低所得者医療援助制度が、今日では、その予算の約３分の１を老人に費やしている。",
+     new String[] { "本来", "は",  "貧困", "層", "の", "女性", "や", "子供", "に", "医療", "保護", "を",      
+                    "提供", "する", "ため", "に", "創設", "さ", "れ", "た", "制度", "で", "ある",  "アメリカ", 
+                    "低", "所得", "者", "医療", "援助", "制度", "が",  "今日", "で", "は",  "その",
+                    "予算", "の", "約", "３", "分の", "１", "を", "老人", "に", "費やし", "て", "いる" },
+     new int[] { 0, 2, 4, 6, 7,  8, 10, 11, 13, 14, 16, 18, 19, 21, 23, 25, 26, 28, 29, 30, 
+                 31, 33, 34, 37, 41, 42, 44, 45, 47, 49, 51, 53, 55, 56, 58, 60,
+                 62, 63, 64, 65, 67, 68, 69, 71, 72, 75, 76 },
+     new int[] { 2, 3, 6, 7, 8, 10, 11, 13, 14, 16, 18, 19, 21, 23, 25, 26, 28, 29, 30, 31,
+                 33, 34, 36, 41, 42, 44, 45, 47, 49, 51, 52, 55, 56, 57, 60, 62,
+                 63, 64, 65, 67, 68, 69, 71, 72, 75, 76, 78 }
+    );
+  }
+  
+  public void testDecomposition2() throws Exception {
+    assertAnalyzesTo(analyzer, "麻薬の密売は根こそぎ絶やさなければならない",
+      new String[] { "麻薬", "の", "密売", "は", "根こそぎ", "絶やさ", "なけれ", "ば", "なら", "ない" },
+      new int[] { 0, 2, 3, 5, 6,  10, 13, 16, 17, 19 },
+      new int[] { 2, 3, 5, 6, 10, 13, 16, 17, 19, 21 }
+    );
+  }
+  
+  public void testDecomposition3() throws Exception {
+    assertAnalyzesTo(analyzer, "魔女狩大将マシュー・ホプキンス。",
+      new String[] { "魔女", "狩", "大将", "マシュー",  "ホプキンス" },
+      new int[] { 0, 2, 3, 5, 10 },
+      new int[] { 2, 3, 5, 9, 15 }
+    );
+  }
+
+  public void testDecomposition4() throws Exception {
+    assertAnalyzesTo(analyzer, "これは本ではない",
+      new String[] { "これ", "は", "本", "で", "は", "ない" },
+      new int[] { 0, 2, 3, 4, 5, 6 },
+      new int[] { 2, 3, 4, 5, 6, 8 }
+    );
+  }
+
+  public void testDecomposition5() throws Exception {
+    assertAnalyzesTo(analyzer, "くよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよくよ",
+      new String[] { "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ", "くよくよ" },
+      new int[] { 0, 4, 8, 12, 16, 20, 24, 28, 32, 36},
+      new int[] { 4, 8, 12, 16, 20, 24, 28, 32, 36, 40 }
+    );
+  }
+
+  /** Tests that sentence offset is incorporated into the resulting offsets */
+  public void testTwoSentences() throws Exception {
+    assertAnalyzesTo(analyzer, "魔女狩大将マシュー・ホプキンス。 魔女狩大将マシュー・ホプキンス。",
+      new String[] { "魔女", "狩", "大将", "マシュー", "ホプキンス",  "魔女", "狩", "大将", "マシュー",  "ホプキンス"  },
+      new int[] { 0, 2, 3, 5, 10, 17, 19, 20, 22, 27 },
+      new int[] { 2, 3, 5, 9, 15, 19, 20, 22, 26, 32 }
+    );
+  }
+
+  /** blast some random strings through the analyzer */
+  public void testRandomStrings() throws Exception {
+    checkRandomData(random, analyzer, 10000*RANDOM_MULTIPLIER);
+  }
+  
+  public void testLargeDocReliability() throws Exception {
+    for (int i = 0; i < 100; i++) {
+      String s = _TestUtil.randomUnicodeString(random, 10000);
+      TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+      ts.reset();
+      while (ts.incrementToken()) {
+      }
+    }
+  }
+  
+  /** simple test for supplementary characters */
+  public void testSurrogates() throws IOException {
+    assertAnalyzesTo(analyzer, "𩬅艱鍟䇹愯瀛",
+      new String[] { "𩬅", "艱", "鍟", "䇹", "愯", "瀛" });
+  }
+  
+  /** random test ensuring we don't ever split supplementaries */
+  public void testSurrogates2() throws IOException {
+    int numIterations = atLeast(10000);
+    for (int i = 0; i < numIterations; i++) {
+      String s = _TestUtil.randomUnicodeString(random, 100);
+      TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+      CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
+      ts.reset();
+      while (ts.incrementToken()) {
+        assertTrue(UnicodeUtil.validUTF16String(termAtt));
+      }
+    }
+  }
+  
+  // note: test is kinda silly since kuromoji emits punctuation tokens.
+  // but, when/if we filter these out it will be useful.
+  public void testEnd() throws Exception {
+    assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("これは本ではない")),
+        new String[] { "これ", "は", "本", "で", "は", "ない" },
+        new int[] { 0, 2, 3, 4, 5, 6 },
+        new int[] { 2, 3, 4, 5, 6, 8 },
+        new Integer(8)
+    );
+    
+    assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("これは本ではない    ")),
+        new String[] { "これ", "は", "本", "で", "は", "ない"  },
+        new int[] { 0, 2, 3, 4, 5, 6, 8 },
+        new int[] { 2, 3, 4, 5, 6, 8, 9 },
+        new Integer(12)
+    );
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/bocchan.utf-8
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/bocchan.utf-8?rev=1230748&view=auto
==============================================================================
Binary file - no diff available.

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/TestTokenInfoDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/TestTokenInfoDictionary.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/TestTokenInfoDictionary.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/TestTokenInfoDictionary.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,107 @@
+package org.apache.lucene.analysis.kuromoji.dict;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.kuromoji.util.ToStringUtil;
+import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.UnicodeUtil;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.IntsRefFSTEnum;
+import org.apache.lucene.util.fst.IntsRefFSTEnum.InputOutput;
+
+public class TestTokenInfoDictionary extends LuceneTestCase {
+
+  /** enumerates the entire FST/lookup data and just does basic sanity checks */
+  public void testEnumerateAll() throws Exception {
+    // just for debugging
+    int numTerms = 0;
+    int numWords = 0;
+    int lastWordId = -1;
+    int lastSourceId = -1;
+    TokenInfoDictionary tid = TokenInfoDictionary.getInstance();
+    ConnectionCosts matrix = ConnectionCosts.getInstance();
+    FST<Long> fst = tid.getFST().getInternalFST();
+    IntsRefFSTEnum<Long> fstEnum = new IntsRefFSTEnum<Long>(fst);
+    InputOutput<Long> mapping;
+    IntsRef scratch = new IntsRef();
+    while ((mapping = fstEnum.next()) != null) {
+      numTerms++;
+      IntsRef input = mapping.input;
+      char chars[] = new char[input.length];
+      for (int i = 0; i < chars.length; i++) {
+        chars[i] = (char)input.ints[input.offset+i];
+      }
+      assertTrue(UnicodeUtil.validUTF16String(new String(chars)));
+      
+      Long output = mapping.output;
+      int sourceId = output.intValue();
+      // we walk in order, terms, sourceIds, and wordIds should always be increasing
+      assertTrue(sourceId > lastSourceId);
+      lastSourceId = sourceId;
+      tid.lookupWordIds(sourceId, scratch);
+      for (int i = 0; i < scratch.length; i++) {
+        numWords++;
+        int wordId = scratch.ints[scratch.offset+i];
+        assertTrue(wordId > lastWordId);
+        lastWordId = wordId;
+         
+        String baseForm = tid.getBaseForm(wordId);
+        assertTrue(baseForm == null || UnicodeUtil.validUTF16String(baseForm));
+        
+        String inflectionForm = tid.getInflectionForm(wordId);
+        assertTrue(inflectionForm == null || UnicodeUtil.validUTF16String(inflectionForm));
+        if (inflectionForm != null) {
+          // check that its actually an ipadic inflection form
+          assertNotNull(ToStringUtil.getInflectedFormTranslation(inflectionForm));          
+        }
+        
+        String inflectionType = tid.getInflectionType(wordId);
+        assertTrue(inflectionType == null || UnicodeUtil.validUTF16String(inflectionType));
+        if (inflectionType != null) {
+          // check that its actually an ipadic inflection type
+          assertNotNull(ToStringUtil.getInflectionTypeTranslation(inflectionType));
+        }
+        
+        int leftId = tid.getLeftId(wordId);
+        int rightId = tid.getRightId(wordId);
+        
+        matrix.get(rightId, leftId);
+        
+        tid.getWordCost(wordId);
+        
+        String pos = tid.getPartOfSpeech(wordId);
+        assertNotNull(pos);
+        assertTrue(UnicodeUtil.validUTF16String(pos));
+        // check that its actually an ipadic pos tag
+        assertNotNull(ToStringUtil.getPOSTranslation(pos));
+        
+        String pronunciation = tid.getPronunciation(wordId);
+        assertNotNull(pronunciation);
+        assertTrue(UnicodeUtil.validUTF16String(pronunciation));
+        
+        String reading = tid.getReading(wordId);
+        assertNotNull(reading);
+        assertTrue(UnicodeUtil.validUTF16String(reading));
+      }
+    }
+    if (VERBOSE) {
+      System.out.println("checked " + numTerms + " terms, " + numWords + " words.");
+    }
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/UserDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/UserDictionaryTest.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/UserDictionaryTest.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/dict/UserDictionaryTest.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,98 @@
+package org.apache.lucene.analysis.kuromoji.dict;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.kuromoji.SegmenterTest;
+import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
+public class UserDictionaryTest extends LuceneTestCase {
+
+  private UserDictionary readDict() throws IOException {
+    InputStream is = SegmenterTest.class.getResourceAsStream("userdict.txt");
+    if (is == null)
+      throw new FileNotFoundException("Cannot find userdict.txt in test classpath!");
+    try {
+      Reader reader = new InputStreamReader(is, IOUtils.CHARSET_UTF_8);
+      return new UserDictionary(reader);
+    } finally {
+      is.close();
+    }
+  }
+  
+  @Test
+  public void testLookup() throws IOException {
+    UserDictionary dictionary = readDict();
+    String s = "関西国際空港に行った";
+    int[][] dictionaryEntryResult = dictionary.lookup(s.toCharArray(), 0, s.length());
+    // Length should be three 関西, 国際, 空港
+    assertEquals(3, dictionaryEntryResult.length);
+    
+    // Test positions
+    assertEquals(0, dictionaryEntryResult[0][1]); // index of 関西
+    assertEquals(2, dictionaryEntryResult[1][1]); // index of 国際
+    assertEquals(4, dictionaryEntryResult[2][1]); // index of 空港
+    
+    // Test lengths
+    assertEquals(2, dictionaryEntryResult[0][2]); // length of 関西
+    assertEquals(2, dictionaryEntryResult[1][2]); // length of 国際
+    assertEquals(2, dictionaryEntryResult[2][2]); // length of 空港
+    
+    s = "関西国際空港と関西国際空港に行った";
+    int[][] dictionaryEntryResult2 = dictionary.lookup(s.toCharArray(), 0, s.length());
+    // Length should be six 
+    assertEquals(6, dictionaryEntryResult2.length);
+  }
+  
+  @Test
+  public void testReadings() throws IOException {
+    UserDictionary dictionary = readDict();
+    int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);
+    assertEquals(3, result.length);
+    int wordIdNihon = result[0][0]; // wordId of 日本 in 日本経済新聞
+    assertEquals("ニホン", dictionary.getReading(wordIdNihon));
+    
+    result = dictionary.lookup("朝青龍".toCharArray(), 0, 3);
+    assertEquals(1, result.length);
+    int wordIdAsashoryu = result[0][0]; // wordId for 朝青龍
+    assertEquals("アサショウリュウ", dictionary.getReading(wordIdAsashoryu));
+  }
+  
+  @Test
+  public void testPartOfSpeech() throws IOException {
+    UserDictionary dictionary = readDict();
+    int[][] result = dictionary.lookup("日本経済新聞".toCharArray(), 0, 6);
+    assertEquals(3, result.length);
+    int wordIdKeizai = result[1][0]; // wordId of 経済 in 日本経済新聞
+    assertEquals("カスタム名詞", dictionary.getPartOfSpeech(wordIdKeizai));
+  }
+  
+  @Test
+  public void testRead() throws IOException {
+    UserDictionary dictionary = readDict();
+    assertNotNull(dictionary);		
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/userdict.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/userdict.txt?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/userdict.txt (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/userdict.txt Thu Jan 12 20:10:48 2012
@@ -0,0 +1,6 @@
+# Custom segmentation for long entries
+日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞
+関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,テスト名詞
+
+# Custom reading for sumo wrestler
+朝青龍,朝青龍,アサショウリュウ,カスタム人名

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/util/TestToStringUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/util/TestToStringUtil.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/util/TestToStringUtil.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/util/TestToStringUtil.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,34 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestToStringUtil extends LuceneTestCase {
+  public void testPOS() {
+    assertEquals("noun-suffix-verbal", ToStringUtil.getPOSTranslation("名詞-接尾-サ変接続"));
+  }
+  
+  public void testHepburn() {
+    assertEquals("majan", ToStringUtil.getRomanization("マージャン"));
+    assertEquals("uroncha", ToStringUtil.getRomanization("ウーロンチャ"));
+    assertEquals("chahan", ToStringUtil.getRomanization("チャーハン"));
+    assertEquals("chashu", ToStringUtil.getRomanization("チャーシュー"));
+    assertEquals("shumai", ToStringUtil.getRomanization("シューマイ"));
+  }
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/BinaryDictionaryWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/BinaryDictionaryWriter.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/BinaryDictionaryWriter.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/BinaryDictionaryWriter.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,371 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.CodecUtil;
+
+import org.apache.lucene.analysis.kuromoji.dict.BinaryDictionary;
+
+public abstract class BinaryDictionaryWriter {
+  protected final Class<? extends BinaryDictionary> implClazz;
+  protected ByteBuffer buffer;
+  private int targetMapEndOffset = 0, lastWordId = -1, lastSourceId = -1;
+  private int[] targetMap = new int[8192];
+  private int[] targetMapOffsets = new int[8192];
+  private final List<String> posDict = new ArrayList<String>();
+  private final Map<String,Integer> posDictLookup = new HashMap<String,Integer>();
+  
+  private final List<String> inflDict = new ArrayList<String>();
+  private final Map<String,Integer> inflDictLookup = new HashMap<String,Integer>();
+
+  public BinaryDictionaryWriter(Class<? extends BinaryDictionary> implClazz, int size) {
+    this.implClazz = implClazz;
+    buffer = ByteBuffer.allocate(size);
+  }
+  
+  /**
+   * put the entry in map
+   * @return current position of buffer, which will be wordId of next entry
+   */
+  public int put(String[] entry) {
+    short leftId = Short.parseShort(entry[1]);
+    short rightId = Short.parseShort(entry[2]);
+    short wordCost = Short.parseShort(entry[3]);
+    
+    StringBuilder sb = new StringBuilder();
+    
+    // build up the POS string
+    for (int i = 4; i < 8; i++) {
+      String part = entry[i];
+      assert part.length() > 0;
+      if (!"*".equals(part)) {
+        if (sb.length() > 0) {
+          sb.append('-');
+        }
+        sb.append(part);
+      }
+    }
+    String pos = sb.toString();
+    Integer posIndex = posDictLookup.get(pos);
+    if (posIndex == null) {
+      posIndex = posDict.size();
+      posDict.add(pos);
+      posDictLookup.put(pos, posIndex);
+      assert posDict.size() == posDictLookup.size();
+    }
+    
+    sb.setLength(0);
+    sb.append(CSVUtil.quoteEscape(entry[8]));
+    sb.append(',');
+    sb.append(CSVUtil.quoteEscape(entry[9]));
+    String inflData = sb.toString();
+    
+    Integer inflIndex = Integer.MAX_VALUE;
+    int hasInflData;
+    if ("*,*".equals(inflData)) {
+      hasInflData = 0; // no inflection data
+    } else {
+      hasInflData = 1;
+      inflIndex = inflDictLookup.get(inflData);
+      if (inflIndex == null) {
+        inflIndex = inflDict.size();
+        inflDict.add(inflData);
+        inflDictLookup.put(inflData, inflIndex);
+        assert inflDict.size() == inflDictLookup.size();
+      }
+    }
+    
+    String baseForm = entry[10];
+    String reading = entry[11];
+    String pronunciation = entry[12];
+    
+    // extend buffer if necessary
+    int left = buffer.remaining();
+    // worst case: three short, 4 bytes, one vint and features (all as utf-16)
+    int worstCase = 6 + 4 + 2 + 2*(baseForm.length() + reading.length() + pronunciation.length());
+    if (worstCase > left) {
+      ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + worstCase - left, 1));
+      buffer.flip();
+      newBuffer.put(buffer);
+      buffer = newBuffer;
+    }
+    
+    buffer.putShort(leftId);
+    buffer.putShort(rightId);
+    buffer.putShort(wordCost);
+    assert posIndex.intValue() < 128;
+    buffer.put((byte) (posIndex.intValue() << 1 | hasInflData));
+    
+    int pronunciationIsReading = pronunciation.equals(reading) ? 1 : 0;
+    
+    if ("*".equals(baseForm) || baseForm.equals(entry[0])) {
+      buffer.put((byte)pronunciationIsReading); // base form is the same as surface form
+    } else {
+      assert baseForm.length() < 128;
+      buffer.put((byte)(baseForm.length() << 1 | pronunciationIsReading));
+      for (int i = 0; i < baseForm.length(); i++) {
+        buffer.putChar(baseForm.charAt(i));
+      }
+    }
+    
+    if (isKatakana(reading)) {
+      buffer.put((byte) (reading.length() << 1 | 1));
+      writeKatakana(reading);
+    } else {
+      buffer.put((byte) (reading.length() << 1));
+      for (int i = 0; i < reading.length(); i++) {
+        buffer.putChar(reading.charAt(i));
+      }
+    }
+    
+    if (pronunciationIsReading == 0) {
+      if (isKatakana(pronunciation)) {
+        buffer.put((byte) (pronunciation.length() << 1 | 1));
+        writeKatakana(pronunciation);
+      } else {
+        buffer.put((byte) (pronunciation.length() << 1));
+        for (int i = 0; i < pronunciation.length(); i++) {
+          buffer.putChar(pronunciation.charAt(i));
+        }
+      }
+    }
+    
+    if (hasInflData > 0) {
+      int key = inflIndex.intValue();
+      assert key < 32768; // note there are really like 300 of these...
+      if (key < 128) {
+        buffer.put((byte) key);
+      } else {
+        buffer.put((byte) ((key & 0x7f) | 0x80));
+        buffer.put((byte) (key >>> 7));
+      }
+    }
+    
+    return buffer.position();
+  }
+  
+  private boolean isKatakana(String s) {
+    for (int i = 0; i < s.length(); i++) {
+      char ch = s.charAt(i);
+      if (ch < 0x30A0 || ch > 0x30FF) {
+        return false;
+      }
+    }
+    return true;
+  }
+  
+  private void writeKatakana(String s) {
+    for (int i = 0; i < s.length(); i++) {
+      buffer.put((byte) (s.charAt(i) - 0x30A0));
+    }
+  }
+  
+  public void addMapping(int sourceId, int wordId) {
+    assert wordId > lastWordId : "words out of order: " + wordId + " vs lastID: " + lastWordId;
+    
+    if (sourceId > lastSourceId) {
+      assert sourceId > lastSourceId : "source ids out of order: lastSourceId=" + lastSourceId + " vs sourceId=" + sourceId;
+      targetMapOffsets = ArrayUtil.grow(targetMapOffsets, sourceId + 1);
+      for (int i = lastSourceId + 1; i <= sourceId; i++) {
+        targetMapOffsets[i] = targetMapEndOffset;
+      }
+    } else {
+      assert sourceId == lastSourceId;
+    }
+
+    targetMap = ArrayUtil.grow(targetMap, targetMapEndOffset + 1);
+    targetMap[targetMapEndOffset] = wordId;
+    targetMapEndOffset++;
+
+    lastSourceId = sourceId;
+    lastWordId = wordId;
+  }
+
+  protected final String getBaseFileName(String baseDir) throws IOException {
+    return baseDir + File.separator + implClazz.getName().replace('.', File.separatorChar);
+  }
+  
+  /**
+   * Write dictionary in file
+   * Dictionary format is:
+   * [Size of dictionary(int)], [entry:{left id(short)}{right id(short)}{word cost(short)}{length of pos info(short)}{pos info(char)}], [entry...], [entry...].....
+   * @throws IOException
+   */
+  public void write(String baseDir) throws IOException {
+    final String baseName = getBaseFileName(baseDir);
+    writeDictionary(baseName + BinaryDictionary.DICT_FILENAME_SUFFIX);
+    writeTargetMap(baseName + BinaryDictionary.TARGETMAP_FILENAME_SUFFIX);
+    writePosDict(baseName + BinaryDictionary.POSDICT_FILENAME_SUFFIX);
+    writeInflDict(baseName + BinaryDictionary.INFLDICT_FILENAME_SUFFIX);
+  }
+  
+  // TODO: maybe this int[] should instead be the output to the FST...
+  protected void writeTargetMap(String filename) throws IOException {
+    new File(filename).getParentFile().mkdirs();
+    OutputStream os = new FileOutputStream(filename);
+    try {
+      os = new BufferedOutputStream(os);
+      final DataOutput out = new OutputStreamDataOutput(os);
+      CodecUtil.writeHeader(out, BinaryDictionary.TARGETMAP_HEADER, BinaryDictionary.VERSION);
+      
+      final int numSourceIds = lastSourceId + 1;
+      out.writeVInt(targetMapEndOffset); // <-- size of main array
+      out.writeVInt(numSourceIds + 1); // <-- size of offset array (+ 1 more entry)
+      int prev = 0, sourceId = 0;
+      for (int ofs = 0; ofs < targetMapEndOffset; ofs++) {
+        final int val = targetMap[ofs], delta = val - prev;
+        assert delta >= 0;
+        if (ofs == targetMapOffsets[sourceId]) {
+          out.writeVInt((delta << 1) | 0x01);
+          sourceId++;
+        } else {
+          out.writeVInt((delta << 1));
+        }
+        prev += delta;
+      }
+      assert sourceId == numSourceIds : "sourceId:"+sourceId+" != numSourceIds:"+numSourceIds;
+    } finally {
+      os.close();
+    }
+  }
+  
+  protected void writePosDict(String filename) throws IOException {
+    new File(filename).getParentFile().mkdirs();
+    OutputStream os = new FileOutputStream(filename);
+    try {
+      os = new BufferedOutputStream(os);
+      final DataOutput out = new OutputStreamDataOutput(os);
+      CodecUtil.writeHeader(out, BinaryDictionary.POSDICT_HEADER, BinaryDictionary.VERSION);
+      out.writeVInt(posDict.size());
+      for (String s : posDict) {
+        out.writeString(s);
+      }
+    } finally {
+      os.close();
+    }
+  }
+  
+  protected void writeInflDict(String filename) throws IOException {
+    new File(filename).getParentFile().mkdirs();
+    OutputStream os = new FileOutputStream(filename);
+    try {
+      os = new BufferedOutputStream(os);
+      final DataOutput out = new OutputStreamDataOutput(os);
+      CodecUtil.writeHeader(out, BinaryDictionary.INFLDICT_HEADER, BinaryDictionary.VERSION);
+      out.writeVInt(inflDict.size());
+      for (String s : inflDict) {
+        String data[] = CSVUtil.parse(s);
+        assert data.length == 2 : "malformed inflection: " + s;
+        out.writeString(data[0]);
+        out.writeString(data[1]);
+      }
+    } finally {
+      os.close();
+    }
+  }
+  
+  protected void writeDictionary(String filename) throws IOException {
+    new File(filename).getParentFile().mkdirs();
+    final FileOutputStream os = new FileOutputStream(filename);
+    try {
+      final DataOutput out = new OutputStreamDataOutput(os);
+      CodecUtil.writeHeader(out, BinaryDictionary.DICT_HEADER, BinaryDictionary.VERSION);
+      out.writeVInt(buffer.position());
+      final WritableByteChannel channel = Channels.newChannel(os);
+      // Write Buffer
+      buffer.flip();  // set position to 0, set limit to current position
+      channel.write(buffer);
+      assert buffer.remaining() == 0L;
+    } finally {
+      os.close();
+    }
+  }
+  
+  // TODO: the below is messy, but makes the dictionary smaller.
+  // we track frequencies of inflections so the highest-freq ones have smaller indexes.
+
+  /** optional: notes inflection seen in the data up front */
+  public void noteInflection(String entry[]) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(CSVUtil.quoteEscape(entry[8]));
+    sb.append(',');
+    sb.append(CSVUtil.quoteEscape(entry[9]));
+    String s = sb.toString();
+    if ("*,*".equals(s)) {
+      return; // no inflection data
+    }
+    Integer freq = notedInflections.get(s);
+    if (freq == null) {
+      freq = 0;
+    }
+    notedInflections.put(s, freq+1);
+  }
+  
+  /** prepopulates inflection mapping by frequency */
+  public void finalizeInflections() {
+    InflectionAndFreq freqs[] = new InflectionAndFreq[notedInflections.size()];
+    int upto = 0;
+    for (Map.Entry<String,Integer> e : notedInflections.entrySet()) {
+      freqs[upto++] = new InflectionAndFreq(e.getKey(), e.getValue());
+    }
+    Arrays.sort(freqs, Collections.reverseOrder());
+    for (int i = 0; i < upto; i++) {
+      inflDict.add(freqs[i].inflection);
+      inflDictLookup.put(freqs[i].inflection, i);
+    }
+  }
+  
+  static class InflectionAndFreq implements Comparable<InflectionAndFreq> {
+    String inflection;
+    int freq;
+    
+    InflectionAndFreq(String s, int i) {
+      this.inflection = s;
+      this.freq = i;
+    }
+    
+    public int compareTo(InflectionAndFreq other) {
+      int cmp = freq - other.freq;
+      if (cmp == 0) {
+        return inflection.compareTo(other.inflection);
+      } else {
+        return cmp;
+      }
+    }
+  }
+  
+  private HashMap<String,Integer> notedInflections = new HashMap<String,Integer>();
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,95 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.lucene.analysis.kuromoji.dict.CharacterDefinition;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.util.CodecUtil;
+
+public final class CharacterDefinitionWriter {
+
+  private final byte[] characterCategoryMap = new byte[0x10000];
+  
+  private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];
+  private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];
+    
+  /**
+   * Constructor for building. TODO: remove write access
+   */
+  public CharacterDefinitionWriter() {
+    Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);
+  }
+  
+  /**
+   * Put mapping from unicode code point to character class.
+   * 
+   * @param codePoint
+   *            code point
+   * @param characterClassName character class name
+   */
+  public void putCharacterCategory(int codePoint, String characterClassName) {
+    characterClassName = characterClassName.split(" ")[0]; // use first
+    // category
+    // class
+    
+    // Override Nakaguro
+    if (codePoint == 0x30FB) {
+      characterClassName = "SYMBOL";
+    }
+    characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);
+  }
+  
+  public void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
+    final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);
+    invokeMap[characterClass] = invoke == 1;
+    groupMap[characterClass] = group == 1;
+    // TODO: length def ignored
+  }
+  
+  public void write(String baseDir) throws IOException {
+    String filename = baseDir + File.separator +
+      CharacterDefinition.class.getName().replace('.', File.separatorChar) + CharacterDefinition.FILENAME_SUFFIX;
+    new File(filename).getParentFile().mkdirs();
+    OutputStream os = new FileOutputStream(filename);
+    try {
+      os = new BufferedOutputStream(os);
+      final DataOutput out = new OutputStreamDataOutput(os);
+      CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
+      out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
+      for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
+        final byte b = (byte) (
+          (invokeMap[i] ? 0x01 : 0x00) | 
+          (groupMap[i] ? 0x02 : 0x00)
+        );
+        out.writeByte(b);
+      }
+    } finally {
+      os.close();
+    }
+  }
+  
+}

Added: lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java?rev=1230748&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java Thu Jan 12 20:10:48 2012
@@ -0,0 +1,67 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+
+public class ConnectionCostsBuilder {
+  
+  private ConnectionCostsBuilder() {
+  }
+  
+  public static ConnectionCostsWriter build(String filename) throws IOException {
+    FileInputStream inputStream = new FileInputStream(filename);
+    Charset cs = Charset.forName("US-ASCII");
+    CharsetDecoder decoder = cs.newDecoder()
+        .onMalformedInput(CodingErrorAction.REPORT)
+        .onUnmappableCharacter(CodingErrorAction.REPORT);
+    InputStreamReader streamReader = new InputStreamReader(inputStream, decoder);
+    LineNumberReader lineReader = new LineNumberReader(streamReader);
+    
+    String line = lineReader.readLine();
+    String[] dimensions = line.split("\\s+");
+    
+    assert dimensions.length == 2;
+    
+    int forwardSize = Integer.parseInt(dimensions[0]);
+    int backwardSize = Integer.parseInt(dimensions[1]);
+    
+    assert forwardSize > 0 && backwardSize > 0;
+    
+    ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);
+    
+    while ((line = lineReader.readLine()) != null) {
+      String[] fields = line.split("\\s+");
+      
+      assert fields.length == 3;
+      
+      int forwardId = Integer.parseInt(fields[0]);
+      int backwardId = Integer.parseInt(fields[1]);
+      int cost = Integer.parseInt(fields[2]);
+      
+      costs.add(forwardId, backwardId, cost);
+    }
+    return costs;
+  }
+}