You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2020/11/23 08:13:04 UTC
[lucene-solr] branch branch_8x updated: LUCENE-9581: Japanese
tokenizer should discard the compound token instead of disabling the
decomposition of long tokens when discardCompoundToken is activated.
This is an automated email from the ASF dual-hosted git repository.
jimczi pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 24d7fe5 LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition of long tokens when discardCompoundToken is activated.
24d7fe5 is described below
commit 24d7fe5c62b4815d90a9e21ce295483e55a5460b
Author: jimczi <ji...@elastic.co>
AuthorDate: Mon Nov 23 08:55:36 2020 +0100
LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition of long tokens when discardCompoundToken is activated.
---
lucene/CHANGES.txt | 5 +-
.../lucene/analysis/ja/JapaneseTokenizer.java | 59 +++++++++++-----------
.../lucene/analysis/ja/TestJapaneseTokenizer.java | 6 +++
3 files changed, 40 insertions(+), 30 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b3fbab7..fc32c40 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,13 +41,16 @@ Optimizations
Bug Fixes
---------------------
-(No changes)
+
+* LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition
+ of long tokens when discardCompoundToken is activated. (Jim Ferenczi)
Other
---------------------
* SOLR-14995: Update Jetty to 9.4.34 (Mike Drob)
+
======================= Lucene 8.7.0 =======================
API Changes
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 4ce6393..96f64e4 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -507,7 +507,7 @@ public final class JapaneseTokenizer extends Tokenizer {
System.out.println(" + cost=" + leastCost + " wordID=" + wordID + " leftID=" + leftID + " leastIDX=" + leastIDX + " toPos=" + endPos + " toPos.idx=" + positions.get(endPos).count);
}
- if ((addPenalty || (!outputCompounds && searchMode)) && type != Type.USER) {
+ if (addPenalty && type != Type.USER) {
final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
if (VERBOSE) {
if (penalty > 0) {
@@ -1670,7 +1670,7 @@ public final class JapaneseTokenizer extends Tokenizer {
int backID = posData.backID[bestIDX];
int nextBestIDX = posData.backIndex[bestIDX];
- if (outputCompounds && searchMode && altToken == null && backType != Type.USER) {
+ if (searchMode && altToken == null && backType != Type.USER) {
// In searchMode, if best path had picked a too-long
// token, we use the "penalty" to compute the allowed
@@ -1764,34 +1764,35 @@ public final class JapaneseTokenizer extends Tokenizer {
assert offset >= 0;
if (altToken != null && altToken.getPosition() >= backPos) {
-
- // We've backtraced to the position where the
- // compound token starts; add it now:
-
- // The pruning we did when we created the altToken
- // ensures that the back trace will align back with
- // the start of the altToken:
- assert altToken.getPosition() == backPos: altToken.getPosition() + " vs " + backPos;
-
- // NOTE: not quite right: the compound token may
- // have had all punctuation back traced so far, but
- // then the decompounded token at this position is
- // not punctuation. In this case backCount is 0,
- // but we should maybe add the altToken anyway...?
-
- if (backCount > 0) {
- backCount++;
- altToken.setPositionLength(backCount);
- if (VERBOSE) {
- System.out.println(" add altToken=" + altToken);
- }
- pending.add(altToken);
- } else {
- // This means alt token was all punct tokens:
- if (VERBOSE) {
- System.out.println(" discard all-punctuation altToken=" + altToken);
+ if (outputCompounds) {
+ // We've backtraced to the position where the
+ // compound token starts; add it now:
+
+ // The pruning we did when we created the altToken
+ // ensures that the back trace will align back with
+ // the start of the altToken:
+ assert altToken.getPosition() == backPos : altToken.getPosition() + " vs " + backPos;
+
+ // NOTE: not quite right: the compound token may
+ // have had all punctuation back traced so far, but
+ // then the decompounded token at this position is
+ // not punctuation. In this case backCount is 0,
+ // but we should maybe add the altToken anyway...?
+
+ if (backCount > 0) {
+ backCount++;
+ altToken.setPositionLength(backCount);
+ if (VERBOSE) {
+ System.out.println(" add altToken=" + altToken);
+ }
+ pending.add(altToken);
+ } else {
+ // This means alt token was all punct tokens:
+ if (VERBOSE) {
+ System.out.println(" discard all-punctuation altToken=" + altToken);
+ }
+ assert discardPunctuation;
}
- assert discardPunctuation;
}
altToken = null;
}
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
index d6464d8..30836af 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
@@ -925,5 +925,11 @@ public class
assertAnalyzesTo(extendedModeAnalyzerNoCompound, "株式会社とアカデミア",
new String[]{"株式", "会社", "と", "ア", "カ", "デ", "ミ", "ア"});
+
+ assertAnalyzesTo(analyzer, "北海道日本ハムファイターズ",
+ new String[]{"北海道", "日本", "ハムファイターズ"});
+
+ assertAnalyzesTo(analyzerNoCompound, "北海道日本ハムファイターズ",
+ new String[]{"北海道", "日本", "ハムファイターズ"});
}
}