You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2012/03/24 07:38:54 UTC
svn commit: r1304719 - in /lucene/dev/trunk: lucene/contrib/
modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/
modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/
solr/core/src/java/org/apache/solr/analysis/ so...
Author: cm
Date: Sat Mar 24 06:38:53 2012
New Revision: 1304719
URL: http://svn.apache.org/viewvc?rev=1304719&view=rev
Log:
Added katakana stem filter (LUCENE-3901)
Added:
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java (with props)
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java (with props)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java (with props)
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java
lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
lucene/dev/trunk/solr/example/solr/conf/schema.xml
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1304719&r1=1304718&r2=1304719&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Sat Mar 24 06:38:53 2012
@@ -172,6 +172,10 @@ New Features
* LUCENE-3767: Kuromoji tokenizer/analyzer produces both compound words
and the segmentation of that compound in Mode.SEARCH. (Robert Muir, Mike McCandless via Christian Moen)
+ * LUCENE-3901: Added katakana stem filter to normalize common spelling variants
+ with/without trailing long vowel marks. The filter is used in both KuromojiAnalyzer
+ and the "text_ja" field type in schema.xml. (Christian Moen)
+
* LUCENE-3685: Add ToChildBlockJoinQuery and renamed previous
BlockJoinQuery to ToParentBlockJoinQuery, so that you can now do
joins in both parent to child and child to parent directions.
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java?rev=1304719&r1=1304718&r2=1304719&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiAnalyzer.java Sat Mar 24 06:38:53 2012
@@ -92,6 +92,7 @@ public class KuromojiAnalyzer extends St
stream = new KuromojiPartOfSpeechStopFilter(true, stream, stoptags);
stream = new CJKWidthFilter(stream);
stream = new StopFilter(matchVersion, stream, stopwords);
+ stream = new KuromojiKatakanaStemFilter(stream);
stream = new LowerCaseFilter(matchVersion, stream);
return new TokenStreamComponents(tokenizer, stream);
}
Added: lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java?rev=1304719&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiKatakanaStemFilter.java Sat Mar 24 06:38:53 2012
@@ -0,0 +1,98 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+import java.io.IOException;
+
+/**
+ * A {@link TokenFilter} that normalizes common katakana spelling variations
+ * ending in a long sound character by removing this character (U+30FC). Only
+ * katakana words longer than a minimum length are stemmed (default is four).
+ * <p>
+ * Note that only full-width katakana characters are supported. Please use a
+ * {@link org.apache.lucene.analysis.cjk.CJKWidthFilter} to convert half-width
+ * katakana to full-width before using this filter.
+ * </p>
+ * <p>
+ * In order to prevent terms from being stemmed, use an instance of
+ * {@link org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter}
+ * or a custom {@link TokenFilter} that sets the {@link KeywordAttribute}
+ * before this {@link TokenStream}.
+ * </p>
+ */
+
+public final class KuromojiKatakanaStemFilter extends TokenFilter {
+ public final static int DEFAULT_MINIMUM_LENGTH = 4;
+ private final static char HIRAGANA_KATAKANA_PROLONGED_SOUND_MARK = '\u30fc';
+
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
+ private final int minimumKatakanaLength;
+
+ public KuromojiKatakanaStemFilter(TokenStream input, int minimumLength) {
+ super(input);
+ this.minimumKatakanaLength = minimumLength;
+ }
+
+ public KuromojiKatakanaStemFilter(TokenStream input) {
+ this(input, DEFAULT_MINIMUM_LENGTH);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ if (!keywordAttr.isKeyword()) {
+ termAttr.setLength(stem(termAttr.buffer(), termAttr.length()));
+ }
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private int stem(char[] term, int length) {
+ if (length < minimumKatakanaLength) {
+ return length;
+ }
+
+ if (! isKatakana(term, length)) {
+ return length;
+ }
+
+ if (term[length - 1] == HIRAGANA_KATAKANA_PROLONGED_SOUND_MARK) {
+ return length - 1;
+ }
+
+ return length;
+ }
+
+ private boolean isKatakana(char[] term, int length) {
+ for (int i = 0; i < length; i++) {
+ // NOTE: Test only identifies full-width characters -- half-widths are supported
+ if (Character.UnicodeBlock.of(term[i]) != Character.UnicodeBlock.KATAKANA) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
Modified: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java?rev=1304719&r1=1304718&r2=1304719&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java Sat Mar 24 06:38:53 2012
@@ -24,6 +24,9 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.kuromoji.KuromojiTokenizer.Mode;
+/**
+ * Test Kuromoji Japanese morphological analyzer
+ */
public class TestKuromojiAnalyzer extends BaseTokenStreamTestCase {
/** This test fails with NPE when the
* stopwords file is missing in classpath */
@@ -54,27 +57,26 @@ public class TestKuromojiAnalyzer extend
KuromojiAnalyzer.getDefaultStopSet(),
KuromojiAnalyzer.getDefaultStopTags());
- /*
- //TokenStream ts = a.tokenStream("foo", new StringReader("妹ã®å²åã§ãã俺ã¨å¹´åã§ãä»åé¨çã§ãã"));
- TokenStream ts = a.tokenStream("foo", new StringReader("�<!--\"<!--#<!--;?><!--#<!--#><!---->?>-->;"));
- ts.reset();
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- while(ts.incrementToken()) {
- System.out.println(" " + termAtt.toString());
- }
- System.out.println("DONE PARSE\n\n");
- */
-
// Senior software engineer:
assertAnalyzesToPositions(a, "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
new String[] { "ã·ãã¢",
- "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
+ "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢", // zero pos inc
"ã½ããã¦ã§ã¢",
"ã¨ã³ã¸ãã¢" },
new int[] { 1, 0, 1, 1},
new int[] { 1, 3, 1, 1}
);
+ // Senior project manager: also tests katakana spelling variation stemming
+ assertAnalyzesToPositions(a, "ã·ãã¢ããã¸ã§ã¯ãããã¼ã¸ã£ã¼",
+ new String[] { "ã·ãã¢",
+ "ã·ãã¢ããã¸ã§ã¯ãããã¼ã¸ã£", // trailing ã¼ removed by stemming, zero pos inc
+ "ããã¸ã§ã¯ã",
+ "ããã¼ã¸ã£"}, // trailing ã¼ removed by stemming
+ new int[]{1, 0, 1, 1},
+ new int[]{1, 3, 1, 1}
+ );
+
// Kansai International Airport:
assertAnalyzesToPositions(a, "é¢è¥¿å½é空港",
new String[] { "é¢è¥¿",
Added: lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java?rev=1304719&view=auto
==============================================================================
--- lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java (added)
+++ lucene/dev/trunk/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiKatakanaStemFilter.java Sat Mar 24 06:38:53 2012
@@ -0,0 +1,71 @@
+package org.apache.lucene.analysis.kuromoji;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Tokenizer;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Tests for {@link org.apache.lucene.analysis.kuromoji.KuromojiKatakanaStemFilter}
+ */
+public class TestKuromojiKatakanaStemFilter extends BaseTokenStreamTestCase {
+ private Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ // Use a MockTokenizer here since this filter doesn't really depend on Kuromoji
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new KuromojiKatakanaStemFilter(source));
+ }
+ };
+
+ /**
+ * Test a few common katakana spelling variations.
+ * <p>
+ * English translations are as follows:
+ * <ul>
+ * <li>copy</li>
+ * <li>coffee</li>
+ * <li>taxi</li>
+ * <li>party</li>
+ * <li>party (without long sound)</li>
+ * <li>center</li>
+ * </ul>
+ * Note that we remove a long sound in the case of "coffee" that is required.
+ * </p>
+ */
+ public void testStemVariants() throws IOException {
+ assertAnalyzesTo(analyzer, "ã³ãã¼ ã³ã¼ãã¼ ã¿ã¯ã·ã¼ ãã¼ãã£ã¼ ãã¼ã㣠ã»ã³ã¿ã¼",
+ new String[] { "ã³ãã¼", "ã³ã¼ã", "ã¿ã¯ã·", "ãã¼ãã£", "ãã¼ãã£", "ã»ã³ã¿" },
+ new int[] { 0, 4, 9, 14, 20, 25 },
+ new int[] { 3, 8, 13, 19, 24, 29 });
+ }
+
+ public void testUnsupportedHalfWidthVariants() throws IOException {
+ // The below result is expected since only full-width katakana is supported
+ assertAnalyzesTo(analyzer, "ï¾ï½¸ï½¼ï½°", new String[] { "ï¾ï½¸ï½¼ï½°" });
+ }
+
+ public void testRandomData() throws IOException {
+ checkRandomData(random, analyzer, 10000*RANDOM_MULTIPLIER);
+ }
+}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java?rev=1304719&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/analysis/KuromojiKatakanaStemFilterFactory.java Sat Mar 24 06:38:53 2012
@@ -0,0 +1,55 @@
+package org.apache.solr.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.kuromoji.KuromojiKatakanaStemFilter;
+import org.apache.solr.common.SolrException;
+
+import java.util.Map;
+
+/**
+ * Factory for {@link KuromojiKatakanaStemFilterFactory}.
+ * <pre class="prettyprint">
+ * <fieldType name="text_ja" class="solr.TextField">
+ * <analyzer>
+ * <tokenizer class="solr.KuromojiTokenizerFactory"/>
+ * <filter class="solr.KuromojiKatakanaStemFilterFactory"
+ * minimumLength="4"/>
+ * </analyzer>
+ * </fieldType>
+ * </pre>
+ */
+public class KuromojiKatakanaStemFilterFactory extends BaseTokenFilterFactory {
+ private static final String MINIMUM_LENGTH_PARAM = "minimumLength";
+ private int minimumLength;
+
+ @Override
+ public void init(Map<String, String> args) {
+ super.init(args);
+ minimumLength = getInt(MINIMUM_LENGTH_PARAM, KuromojiKatakanaStemFilter.DEFAULT_MINIMUM_LENGTH);
+ if (minimumLength < 2) {
+ throw new SolrException(SolrException.ErrorCode.UNKNOWN,
+ "Illegal " + MINIMUM_LENGTH_PARAM + " " + minimumLength + " (must be 2 or greater)");
+ }
+ }
+
+ public TokenStream create(TokenStream input) {
+ return new KuromojiKatakanaStemFilter(input, minimumLength);
+ }
+}
Modified: lucene/dev/trunk/solr/example/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/conf/schema.xml?rev=1304719&r1=1304718&r2=1304719&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/conf/schema.xml (original)
+++ lucene/dev/trunk/solr/example/solr/conf/schema.xml Sat Mar 24 06:38:53 2012
@@ -504,13 +504,13 @@
<!-- CJK bigram (see text_ja for a Japanese configuration using morphological analysis) -->
<fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
+ <analyzer>
<tokenizer class="solr.StandardTokenizerFactory"/>
<!-- normalize width before bigram, as e.g. half-width dakuten combine -->
<filter class="solr.CJKWidthFilterFactory"/>
<!-- for any non-CJK -->
<filter class="solr.LowerCaseFilterFactory"/>
- <filter class="solr.CJKBigramFilterFactory"/>
+ <filter class="solr.CJKBigramFilterFactory"/>
</analyzer>
</fieldType>
@@ -720,7 +720,9 @@
<filter class="solr.CJKWidthFilterFactory"/>
<!-- Removes common tokens typically not useful for search, but have a negative effect on ranking -->
<filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ja.txt" enablePositionIncrements="true" />
- <!-- Lower-case romaji characters -->
+ <!-- Normalizes common katakana spelling variations by removing any last long sound character (U+30FC) -->
+ <filter class="solr.KuromojiKatakanaStemFilterFactory" minimumLength="4"/>
+ <!-- Lower-cases romaji characters -->
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>