You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by to...@apache.org on 2020/02/01 06:23:50 UTC
[lucene-solr] branch branch_8x updated: LUCENE-9123: Add new
JapaneseTokenizer constructors with discardCompoundToken option to control
whether the tokenizer emits original tokens when the mode is not NORMAL.
This is an automated email from the ASF dual-hosted git repository.
tomoko pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 12242b5 LUCENE-9123: Add new JapaneseTokenizer constructors with discardCompoundToken option to control whether the tokenizer emits original tokens when the mode is not NORMAL.
12242b5 is described below
commit 12242b52e6d15a58d0af54d865c6af94e07fbf51
Author: Kazuaki Hiraga <ka...@gmail.com>
AuthorDate: Sat Feb 1 15:20:02 2020 +0900
LUCENE-9123: Add new JapaneseTokenizer constructors with discardCompoundToken option to control whether the tokenizer emits original tokens when the mode is not NORMAL.
---
lucene/CHANGES.txt | 3 ++
.../lucene/analysis/ja/JapaneseAnalyzer.java | 2 +-
.../lucene/analysis/ja/JapaneseTokenizer.java | 42 +++++++++++++++++++---
.../analysis/ja/JapaneseTokenizerFactory.java | 7 +++-
.../lucene/analysis/ja/TestJapaneseTokenizer.java | 37 +++++++++++++++++--
.../analysis/ja/TestJapaneseTokenizerFactory.java | 15 ++++++++
6 files changed, 97 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5e05216..5569d68 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -58,6 +58,9 @@ Improvements
* LUCENE-9141: Simplify LatLonShapeXQuery API by adding a new abstract class called LatLonGeometry. Queries are
executed with input objects that extend such interface. (Ignacio Vera)
+* LUCENE-9123: Add new JapaneseTokenizer constructors with discardCompoundToken option that controls whether
+ the tokenizer emits original (compound) tokens when the mode is not NORMAL. (Kazuaki Hiraga via Tomoko Uchida)
+
Optimizations
---------------------
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
index 82c71dc..8d09aee 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
@@ -87,7 +87,7 @@ public class JapaneseAnalyzer extends StopwordAnalyzerBase {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(userDict, true, mode);
+ Tokenizer tokenizer = new JapaneseTokenizer(userDict, true, false, mode);
TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
stream = new JapanesePartOfSpeechStopFilter(stream, stoptags);
stream = new CJKWidthFilter(stream);
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index b2b279b..4ce6393 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -198,7 +198,21 @@ public final class JapaneseTokenizer extends Tokenizer {
* @param mode tokenization mode.
*/
public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
- this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, discardPunctuation, mode);
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, discardPunctuation, false, mode);
+ }
+
+ /**
+ * Create a new JapaneseTokenizer.
+ * <p>
+ * Uses the default AttributeFactory.
+ *
+ * @param userDictionary Optional: if non-null, user dictionary.
+ * @param discardPunctuation true if punctuation tokens should be dropped from the output.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
+ * @param mode tokenization mode.
+ */
+ public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuation, boolean discardCompoundToken, Mode mode) {
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, discardPunctuation, discardCompoundToken, mode);
}
/**
@@ -215,7 +229,25 @@ public final class JapaneseTokenizer extends Tokenizer {
TokenInfoDictionary.getInstance(),
UnknownDictionary.getInstance(),
ConnectionCosts.getInstance(),
- userDictionary, discardPunctuation, mode);
+ userDictionary, discardPunctuation, false, mode);
+ }
+
+ /**
+ * Create a new JapaneseTokenizer using the system and unknown dictionaries shipped with Lucene.
+ *
+ * @param factory the AttributeFactory to use
+ * @param userDictionary Optional: if non-null, user dictionary.
+ * @param discardPunctuation true if punctuation tokens should be dropped from the output.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
+ * @param mode tokenization mode.
+ */
+ public JapaneseTokenizer
+ (AttributeFactory factory, UserDictionary userDictionary, boolean discardPunctuation, boolean discardCompoundToken, Mode mode) {
+ this(factory,
+ TokenInfoDictionary.getInstance(),
+ UnknownDictionary.getInstance(),
+ ConnectionCosts.getInstance(),
+ userDictionary, discardPunctuation, discardCompoundToken, mode);
}
/**
@@ -229,6 +261,7 @@ public final class JapaneseTokenizer extends Tokenizer {
* @param connectionCosts custom token transition costs
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
+ * @param discardCompoundToken true if compound tokens should be dropped from the output when tokenization mode is not NORMAL.
* @param mode tokenization mode.
* @lucene.experimental
*/
@@ -238,6 +271,7 @@ public final class JapaneseTokenizer extends Tokenizer {
ConnectionCosts connectionCosts,
UserDictionary userDictionary,
boolean discardPunctuation,
+ boolean discardCompoundToken,
Mode mode) {
super(factory);
this.dictionary = systemDictionary;
@@ -259,12 +293,12 @@ public final class JapaneseTokenizer extends Tokenizer {
case SEARCH:
searchMode = true;
extendedMode = false;
- outputCompounds = true;
+ outputCompounds = !discardCompoundToken;
break;
case EXTENDED:
searchMode = true;
extendedMode = true;
- outputCompounds = false;
+ outputCompounds = !discardCompoundToken;
break;
default:
searchMode = false;
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
index 1eaecd7..29dc44a 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
@@ -45,6 +45,7 @@ import org.apache.lucene.analysis.util.ResourceLoaderAware;
* userDictionary="user.txt"
* userDictionaryEncoding="UTF-8"
* discardPunctuation="true"
+ * discardCompoundToken="false"
* />
* <filter class="solr.JapaneseBaseFormFilterFactory"/>
* </analyzer>
@@ -93,6 +94,8 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
private static final String DISCARD_PUNCTUATION = "discardPunctuation"; // Expert option
+ private static final String DISCARD_COMPOUND_TOKEN = "discardCompoundToken"; // Expert option
+
private static final String NBEST_COST = "nBestCost";
private static final String NBEST_EXAMPLES = "nBestExamples";
@@ -101,6 +104,7 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
private final Mode mode;
private final boolean discardPunctuation;
+ private final boolean discardCompoundToken;
private final String userDictionaryPath;
private final String userDictionaryEncoding;
@@ -124,6 +128,7 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
userDictionaryPath = args.remove(USER_DICT_PATH);
userDictionaryEncoding = args.remove(USER_DICT_ENCODING);
discardPunctuation = getBoolean(args, DISCARD_PUNCTUATION, true);
+ discardCompoundToken = getBoolean(args, DISCARD_COMPOUND_TOKEN, false);
nbestCost = getInt(args, NBEST_COST, 0);
nbestExamples = args.remove(NBEST_EXAMPLES);
if (!args.isEmpty()) {
@@ -152,7 +157,7 @@ public class JapaneseTokenizerFactory extends TokenizerFactory implements Resour
@Override
public JapaneseTokenizer create(AttributeFactory factory) {
- JapaneseTokenizer t = new JapaneseTokenizer(factory, userDictionary, discardPunctuation, mode);
+ JapaneseTokenizer t = new JapaneseTokenizer(factory, userDictionary, discardPunctuation, discardCompoundToken, mode);
if (nbestExamples != null) {
nbestCost = Math.max(nbestCost, t.calcNBestCost(nbestExamples));
}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
index 516b98d..d6464d8 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
@@ -64,7 +64,7 @@ public class
}
}
- private Analyzer analyzer, analyzerNormal, analyzerNormalNBest, analyzerNoPunct, extendedModeAnalyzerNoPunct;
+ private Analyzer analyzer, analyzerNormal, analyzerNormalNBest, analyzerNoPunct, extendedModeAnalyzerNoPunct, analyzerNoCompound, extendedModeAnalyzerNoCompound;
private JapaneseTokenizer makeTokenizer(boolean discardPunctuation, Mode mode) {
return new JapaneseTokenizer(newAttributeFactory(), readDict(), discardPunctuation, mode);
@@ -118,11 +118,25 @@ public class
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
+ analyzerNoCompound = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, true, Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ extendedModeAnalyzerNoCompound = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, true, Mode.EXTENDED);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
}
@Override
public void tearDown() throws Exception {
- IOUtils.close(analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct);
+ IOUtils.close(analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct, analyzerNoCompound, extendedModeAnalyzerNoCompound);
super.tearDown();
}
@@ -477,7 +491,7 @@ public class
new TokenInfoDictionary(ResourceScheme.CLASSPATH, "org/apache/lucene/analysis/ja/dict/TokenInfoDictionary"),
new UnknownDictionary(ResourceScheme.CLASSPATH, "org/apache/lucene/analysis/ja/dict/UnknownDictionary"),
new ConnectionCosts(ResourceScheme.CLASSPATH, "org/apache/lucene/analysis/ja/dict/ConnectionCosts"),
- readDict(), true, Mode.SEARCH);
+ readDict(), true, false, Mode.SEARCH);
try (Analyzer a = makeAnalyzer(tokenizer)) {
assertTokenStreamContents(a.tokenStream("foo", "abcd"),
new String[] { "a", "b", "cd" },
@@ -895,4 +909,21 @@ public class
new int[]{0, 2},
new int[]{2, 4});
}
+
+ public void testNoCompoundToken() throws Exception {
+ assertAnalyzesTo(analyzerNormal, "株式会社とアカデミア",
+ new String[]{"株式会社", "と", "アカデミア"});
+
+ assertAnalyzesTo(analyzer, "株式会社とアカデミア",
+ new String[]{"株式", "株式会社", "会社", "と", "アカデミア"});
+
+ assertAnalyzesTo(analyzerNoCompound, "株式会社とアカデミア",
+ new String[]{"株式", "会社", "と", "アカデミア"});
+
+ assertAnalyzesTo(extendedModeAnalyzerNoPunct, "株式会社とアカデミア",
+ new String[]{"株式", "株式会社", "会社", "と", "ア", "カ", "デ", "ミ", "ア"});
+
+ assertAnalyzesTo(extendedModeAnalyzerNoCompound, "株式会社とアカデミア",
+ new String[]{"株式", "会社", "と", "ア", "カ", "デ", "ミ", "ア"});
+ }
}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
index bdf22cf..1ec4d91 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
@@ -109,6 +109,21 @@ public class TestJapaneseTokenizerFactory extends BaseTokenStreamTestCase {
);
}
+ /**
+ * Test discarding compound (original) token
+ */
+ public void testDiscardCompoundToken() throws IOException {
+ Map<String,String> args = new HashMap<>();
+ args.put("discardCompoundToken", "true");
+ JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
+ factory.inform(new StringMockResourceLoader(""));
+ TokenStream ts = factory.create(newAttributeFactory());
+ ((Tokenizer)ts).setReader(new StringReader("株式会社フーの上場のお知らせ"));
+ assertTokenStreamContents(ts,
+ new String[] { "株式", "会社", "フー", "の", "上場", "の", "お知らせ"}
+ );
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {