You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2012/03/26 12:31:52 UTC
svn commit: r1305297 [1/3] - in /lucene/dev/trunk: modules/analysis/kuromoji/
modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/
modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/
modules/analysis/kuromoji/src/java/or...
Author: cm
Date: Mon Mar 26 10:31:48 2012
New Revision: 1305297
URL: http://svn.apache.org/viewvc?rev=1305297&view=rev
Log:
Move and rename Kuromoji (LUCENE-3909)
Added:
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/
- copied from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiBaseFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiPartOfSpeechStopFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiReadingFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/ja/
- copied from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/
- copied from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiBaseFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiReadingFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/
- copied from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/ja/
- copied from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/JapaneseBaseFormFilterFactory.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiBaseFormFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/JapaneseKatakanaStemFilterFactory.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/JapanesePartOfSpeechStopFilterFactory.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiPartOfSpeechStopFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/JapaneseReadingFormFilterFactory.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiReadingFormFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/JapaneseTokenizerFactory.java (contents, props changed)
- copied, changed from r1305205, lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java
Removed:
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiAnalyzer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiBaseFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiKatakanaStemFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiPartOfSpeechStopFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiReadingFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/KuromojiTokenizer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestKuromojiAnalyzer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestKuromojiBaseFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestKuromojiKatakanaStemFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestKuromojiReadingFormFilter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestKuromojiTokenizer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiBaseFormFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiPartOfSpeechStopFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiReadingFormFilterFactory.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiTokenizerFactory.java
Modified:
lucene/dev/trunk/modules/analysis/kuromoji/build.xml
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttribute.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttributeImpl.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttribute.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttributeImpl.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttribute.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttributeImpl.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttribute.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttributeImpl.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ToStringUtil.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/UserDictionaryTest.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/util/TestToStringUtil.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/BinaryDictionaryWriter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/CharacterDefinitionWriter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/ConnectionCostsBuilder.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/ConnectionCostsWriter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/DictionaryBuilder.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryBuilder.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/UnknownDictionaryWriter.java
lucene/dev/trunk/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/ja/dict/UnknownDictionaryTest.java
lucene/dev/trunk/solr/build.xml
lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestKuromojiBaseFormFilterFactory.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestKuromojiPartOfSpeechStopFilterFactory.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/analysis/TestKuromojiTokenizerFactory.java
lucene/dev/trunk/solr/example/solr/conf/schema.xml
Modified: lucene/dev/trunk/modules/analysis/kuromoji/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/build.xml?rev=1305297&r1=1305296&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/build.xml (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/build.xml Mon Mar 26 10:31:48 2012
@@ -78,10 +78,10 @@
<target name="build-dict" depends="compile-tools, download-dict">
<sequential>
<delete verbose="true">
- <fileset dir="src/resources/org/apache/lucene/analysis/kuromoji/dict" includes="**/*"/>
+ <fileset dir="src/resources/org/apache/lucene/analysis/ja/dict" includes="**/*"/>
</delete>
<!-- TODO: optimize the dictionary construction a bit so that you don't need 1G -->
- <java fork="true" failonerror="true" maxmemory="1g" classname="org.apache.lucene.analysis.kuromoji.util.DictionaryBuilder">
+ <java fork="true" failonerror="true" maxmemory="1g" classname="org.apache.lucene.analysis.ja.util.DictionaryBuilder">
<classpath>
<path refid="tools.classpath"/>
<pathelement path="${build.dir}/classes/tools"/>
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/GraphvizFormatter.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -18,14 +18,12 @@ package org.apache.lucene.analysis.kurom
*/
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Position;
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Type;
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.WrappedPositionArray;
-import org.apache.lucene.analysis.kuromoji.dict.ConnectionCosts;
-import org.apache.lucene.analysis.kuromoji.dict.Dictionary;
+import org.apache.lucene.analysis.ja.JapaneseTokenizer.Position;
+import org.apache.lucene.analysis.ja.JapaneseTokenizer.WrappedPositionArray;
+import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
+import org.apache.lucene.analysis.ja.dict.Dictionary;
// TODO: would be nice to show 2nd best path in a diff't
@@ -62,7 +60,7 @@ public class GraphvizFormatter {
}
// Backtraces another incremental fragment:
- void onBacktrace(KuromojiTokenizer tok, WrappedPositionArray positions, int lastBackTracePos, Position endPosData, int fromIDX, char[] fragment, boolean isEnd) {
+ void onBacktrace(JapaneseTokenizer tok, WrappedPositionArray positions, int lastBackTracePos, Position endPosData, int fromIDX, char[] fragment, boolean isEnd) {
setBestPathMap(positions, lastBackTracePos, endPosData, fromIDX);
sb.append(formatNodes(tok, positions, lastBackTracePos, endPosData, fragment));
if (isEnd) {
@@ -96,7 +94,7 @@ public class GraphvizFormatter {
}
}
- private String formatNodes(KuromojiTokenizer tok, WrappedPositionArray positions, int startPos, Position endPosData, char[] fragment) {
+ private String formatNodes(JapaneseTokenizer tok, WrappedPositionArray positions, int startPos, Position endPosData, char[] fragment) {
StringBuilder sb = new StringBuilder();
// Output nodes
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,26 +27,26 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.cjk.CJKWidthFilter;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Mode;
-import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
+import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
+import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
/**
* Analyzer for Japanese that uses morphological analysis.
- * @see KuromojiTokenizer
+ * @see JapaneseTokenizer
*/
-public class KuromojiAnalyzer extends StopwordAnalyzerBase {
+public class JapaneseAnalyzer extends StopwordAnalyzerBase {
private final Mode mode;
private final Set<String> stoptags;
private final UserDictionary userDict;
- public KuromojiAnalyzer(Version matchVersion) {
- this(matchVersion, null, KuromojiTokenizer.DEFAULT_MODE, DefaultSetHolder.DEFAULT_STOP_SET, DefaultSetHolder.DEFAULT_STOP_TAGS);
+ public JapaneseAnalyzer(Version matchVersion) {
+ this(matchVersion, null, JapaneseTokenizer.DEFAULT_MODE, DefaultSetHolder.DEFAULT_STOP_SET, DefaultSetHolder.DEFAULT_STOP_TAGS);
}
- public KuromojiAnalyzer(Version matchVersion, UserDictionary userDict, Mode mode, CharArraySet stopwords, Set<String> stoptags) {
+ public JapaneseAnalyzer(Version matchVersion, UserDictionary userDict, Mode mode, CharArraySet stopwords, Set<String> stoptags) {
super(matchVersion, stopwords);
this.userDict = userDict;
this.mode = mode;
@@ -71,8 +71,8 @@ public class KuromojiAnalyzer extends St
static {
try {
- DEFAULT_STOP_SET = loadStopwordSet(true, KuromojiAnalyzer.class, "stopwords.txt", "#"); // ignore case
- final CharArraySet tagset = loadStopwordSet(false, KuromojiAnalyzer.class, "stoptags.txt", "#");
+ DEFAULT_STOP_SET = loadStopwordSet(true, JapaneseAnalyzer.class, "stopwords.txt", "#"); // ignore case
+ final CharArraySet tagset = loadStopwordSet(false, JapaneseAnalyzer.class, "stoptags.txt", "#");
DEFAULT_STOP_TAGS = new HashSet<String>();
for (Object element : tagset) {
char chars[] = (char[]) element;
@@ -87,12 +87,12 @@ public class KuromojiAnalyzer extends St
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KuromojiTokenizer(reader, userDict, true, mode);
- TokenStream stream = new KuromojiBaseFormFilter(tokenizer);
- stream = new KuromojiPartOfSpeechStopFilter(true, stream, stoptags);
+ Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
+ TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
+ stream = new JapanesePartOfSpeechStopFilter(true, stream, stoptags);
stream = new CJKWidthFilter(stream);
stream = new StopFilter(matchVersion, stream, stopwords);
- stream = new KuromojiKatakanaStemFilter(stream);
+ stream = new JapaneseKatakanaStemFilter(stream);
stream = new LowerCaseFilter(matchVersion, stream);
return new TokenStreamComponents(tokenizer, stream);
}
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiBaseFormFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiBaseFormFilter.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiBaseFormFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.kuromoji.tokenattributes.BaseFormAttribute;
+import org.apache.lucene.analysis.ja.tokenattributes.BaseFormAttribute;
import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
@@ -36,12 +36,12 @@ import org.apache.lucene.analysis.tokena
* the {@link KeywordAttribute} before this {@link TokenStream}.
* </p>
*/
-public final class KuromojiBaseFormFilter extends TokenFilter {
+public final class JapaneseBaseFormFilter extends TokenFilter {
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final BaseFormAttribute basicFormAtt = addAttribute(BaseFormAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
- public KuromojiBaseFormFilter(TokenStream input) {
+ public JapaneseBaseFormFilter(TokenStream input) {
super(input);
}
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -41,7 +41,7 @@ import java.io.IOException;
* </p>
*/
-public final class KuromojiKatakanaStemFilter extends TokenFilter {
+public final class JapaneseKatakanaStemFilter extends TokenFilter {
public final static int DEFAULT_MINIMUM_LENGTH = 4;
private final static char HIRAGANA_KATAKANA_PROLONGED_SOUND_MARK = '\u30fc';
@@ -49,12 +49,12 @@ public final class KuromojiKatakanaStemF
private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
private final int minimumKatakanaLength;
- public KuromojiKatakanaStemFilter(TokenStream input, int minimumLength) {
+ public JapaneseKatakanaStemFilter(TokenStream input, int minimumLength) {
super(input);
this.minimumKatakanaLength = minimumLength;
}
- public KuromojiKatakanaStemFilter(TokenStream input) {
+ public JapaneseKatakanaStemFilter(TokenStream input) {
this(input, DEFAULT_MINIMUM_LENGTH);
}
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiPartOfSpeechStopFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiPartOfSpeechStopFilter.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiPartOfSpeechStopFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapanesePartOfSpeechStopFilter.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,18 +20,18 @@ package org.apache.lucene.analysis.kurom
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.analysis.kuromoji.tokenattributes.PartOfSpeechAttribute;
+import org.apache.lucene.analysis.ja.tokenattributes.PartOfSpeechAttribute;
import org.apache.lucene.analysis.util.FilteringTokenFilter;
import org.apache.lucene.analysis.TokenStream;
/**
* Removes tokens that match a set of part-of-speech tags.
*/
-public final class KuromojiPartOfSpeechStopFilter extends FilteringTokenFilter {
+public final class JapanesePartOfSpeechStopFilter extends FilteringTokenFilter {
private final Set<String> stopTags;
private final PartOfSpeechAttribute posAtt = addAttribute(PartOfSpeechAttribute.class);
- public KuromojiPartOfSpeechStopFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTags) {
+ public JapanesePartOfSpeechStopFilter(boolean enablePositionIncrements, TokenStream input, Set<String> stopTags) {
super(enablePositionIncrements, input);
this.stopTags = stopTags;
}
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiReadingFormFilter.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiReadingFormFilter.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiReadingFormFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseReadingFormFilter.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,8 +20,8 @@ package org.apache.lucene.analysis.kurom
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.kuromoji.tokenattributes.ReadingAttribute;
-import org.apache.lucene.analysis.kuromoji.util.ToStringUtil;
+import org.apache.lucene.analysis.ja.tokenattributes.ReadingAttribute;
+import org.apache.lucene.analysis.ja.util.ToStringUtil;
import java.io.IOException;
@@ -31,18 +31,18 @@ import java.io.IOException;
* The default reading form is katakana.
*/
-public final class KuromojiReadingFormFilter extends TokenFilter {
+public final class JapaneseReadingFormFilter extends TokenFilter {
private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
private final ReadingAttribute readingAttr = addAttribute(ReadingAttribute.class);
private boolean useRomaji;
- public KuromojiReadingFormFilter(TokenStream input, boolean useRomaji) {
+ public JapaneseReadingFormFilter(TokenStream input, boolean useRomaji) {
super(input);
this.useRomaji = useRomaji;
}
- public KuromojiReadingFormFilter(TokenStream input) {
+ public JapaneseReadingFormFilter(TokenStream input) {
this(input, false);
}
Copied: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java (from r1305205, lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java?p2=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java&p1=lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer.java&r1=1305205&r2=1305297&rev=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -26,14 +26,14 @@ import java.util.EnumMap;
import java.util.List;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.kuromoji.dict.CharacterDefinition;
-import org.apache.lucene.analysis.kuromoji.dict.ConnectionCosts;
-import org.apache.lucene.analysis.kuromoji.dict.Dictionary;
-import org.apache.lucene.analysis.kuromoji.dict.TokenInfoDictionary;
-import org.apache.lucene.analysis.kuromoji.dict.TokenInfoFST;
-import org.apache.lucene.analysis.kuromoji.dict.UnknownDictionary;
-import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
-import org.apache.lucene.analysis.kuromoji.tokenattributes.*;
+import org.apache.lucene.analysis.ja.dict.CharacterDefinition;
+import org.apache.lucene.analysis.ja.dict.ConnectionCosts;
+import org.apache.lucene.analysis.ja.dict.Dictionary;
+import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
+import org.apache.lucene.analysis.ja.dict.TokenInfoFST;
+import org.apache.lucene.analysis.ja.dict.UnknownDictionary;
+import org.apache.lucene.analysis.ja.dict.UserDictionary;
+import org.apache.lucene.analysis.ja.tokenattributes.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -69,7 +69,7 @@ import org.apache.lucene.util.fst.FST;
* {@link Mode#SEARCH}, we output the alternate segmentation
* as well.
*/
-public final class KuromojiTokenizer extends Tokenizer {
+public final class JapaneseTokenizer extends Tokenizer {
/**
* Tokenization mode: this determines how the tokenizer handles
@@ -174,14 +174,14 @@ public final class KuromojiTokenizer ext
private final InflectionAttribute inflectionAtt = addAttribute(InflectionAttribute.class);
/**
- * Create a new KuromojiTokenizer.
+ * Create a new JapaneseTokenizer.
*
* @param input Reader containing text
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
* @param mode tokenization mode.
*/
- public KuromojiTokenizer(Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
+ public JapaneseTokenizer(Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
super(input);
dictionary = TokenInfoDictionary.getInstance();
fst = dictionary.getFST();
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/Token.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Type;
-import org.apache.lucene.analysis.kuromoji.dict.Dictionary;
+import org.apache.lucene.analysis.ja.JapaneseTokenizer.Type;
+import org.apache.lucene.analysis.ja.dict.Dictionary;
/**
* Analyzed token with morphological data from its dictionary.
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/CharacterDefinition.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/Dictionary.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoFST.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UnknownDictionary.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.dict;
+package org.apache.lucene.analysis.ja.dict;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -27,8 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
-import org.apache.lucene.analysis.kuromoji.dict.Dictionary;
-import org.apache.lucene.analysis.kuromoji.util.CSVUtil;
+import org.apache.lucene.analysis.ja.util.CSVUtil;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.FST;
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttribute.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttribute.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttribute.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
+import org.apache.lucene.analysis.ja.Token;
import org.apache.lucene.util.Attribute;
/**
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttributeImpl.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttributeImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/BaseFormAttributeImpl.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
+import org.apache.lucene.analysis.ja.Token;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttribute.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttribute.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttribute.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
+import org.apache.lucene.analysis.ja.Token;
import org.apache.lucene.util.Attribute;
/**
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttributeImpl.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttributeImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/InflectionAttributeImpl.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
-import org.apache.lucene.analysis.kuromoji.util.ToStringUtil;
+import org.apache.lucene.analysis.ja.Token;
+import org.apache.lucene.analysis.ja.util.ToStringUtil;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttribute.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttribute.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttribute.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
+import org.apache.lucene.analysis.ja.Token;
import org.apache.lucene.util.Attribute;
/**
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttributeImpl.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttributeImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/PartOfSpeechAttributeImpl.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
-import org.apache.lucene.analysis.kuromoji.util.ToStringUtil;
+import org.apache.lucene.analysis.ja.Token;
+import org.apache.lucene.analysis.ja.util.ToStringUtil;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttribute.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttribute.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttribute.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,7 +17,7 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
+import org.apache.lucene.analysis.ja.Token;
import org.apache.lucene.util.Attribute;
/**
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttributeImpl.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttributeImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/tokenattributes/ReadingAttributeImpl.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.tokenattributes;
+package org.apache.lucene.analysis.ja.tokenattributes;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import org.apache.lucene.analysis.kuromoji.Token;
-import org.apache.lucene.analysis.kuromoji.util.ToStringUtil;
+import org.apache.lucene.analysis.ja.Token;
+import org.apache.lucene.analysis.ja.util.ToStringUtil;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.AttributeReflector;
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/CSVUtil.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.util;
+package org.apache.lucene.analysis.ja.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ToStringUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ToStringUtil.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ToStringUtil.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/ToStringUtil.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji.util;
+package org.apache.lucene.analysis.ja.util;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1305297&r1=1305205&r2=1305297&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Mon Mar 26 10:31:48 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.analysis.kuromoji;
+package org.apache.lucene.analysis.ja;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Mode;
+import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
@@ -35,7 +35,7 @@ public class TestExtendedMode extends Ba
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KuromojiTokenizer(reader, null, true, Mode.EXTENDED);
+ Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};