You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2020/11/23 08:13:04 UTC

[lucene-solr] branch branch_8x updated: LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition of long tokens when discardCompoundToken is activated.

This is an automated email from the ASF dual-hosted git repository.

jimczi pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/branch_8x by this push:
     new 24d7fe5  LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition of long tokens when discardCompoundToken is activated.
24d7fe5 is described below

commit 24d7fe5c62b4815d90a9e21ce295483e55a5460b
Author: jimczi <ji...@elastic.co>
AuthorDate: Mon Nov 23 08:55:36 2020 +0100

    LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition of long tokens when discardCompoundToken is activated.
---
 lucene/CHANGES.txt                                 |  5 +-
 .../lucene/analysis/ja/JapaneseTokenizer.java      | 59 +++++++++++-----------
 .../lucene/analysis/ja/TestJapaneseTokenizer.java  |  6 +++
 3 files changed, 40 insertions(+), 30 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index b3fbab7..fc32c40 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,13 +41,16 @@ Optimizations
 
 Bug Fixes
 ---------------------
-(No changes)
+
+* LUCENE-9581: Japanese tokenizer should discard the compound token instead of disabling the decomposition
+  of long tokens when discardCompoundToken is activated. (Jim Ferenczi)
 
 Other
 ---------------------
 
 * SOLR-14995: Update Jetty to 9.4.34 (Mike Drob)
 
+
 ======================= Lucene 8.7.0 =======================
 
 API Changes
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
index 4ce6393..96f64e4 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
@@ -507,7 +507,7 @@ public final class JapaneseTokenizer extends Tokenizer {
       System.out.println("      + cost=" + leastCost + " wordID=" + wordID + " leftID=" + leftID + " leastIDX=" + leastIDX + " toPos=" + endPos + " toPos.idx=" + positions.get(endPos).count);
     }
 
-    if ((addPenalty || (!outputCompounds && searchMode)) && type != Type.USER) {
+    if (addPenalty && type != Type.USER) {
       final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
       if (VERBOSE) {
         if (penalty > 0) {
@@ -1670,7 +1670,7 @@ public final class JapaneseTokenizer extends Tokenizer {
       int backID = posData.backID[bestIDX];
       int nextBestIDX = posData.backIndex[bestIDX];
 
-      if (outputCompounds && searchMode && altToken == null && backType != Type.USER) {
+      if (searchMode && altToken == null && backType != Type.USER) {
 
         // In searchMode, if best path had picked a too-long
         // token, we use the "penalty" to compute the allowed
@@ -1764,34 +1764,35 @@ public final class JapaneseTokenizer extends Tokenizer {
       assert offset >= 0;
 
       if (altToken != null && altToken.getPosition() >= backPos) {
-
-        // We've backtraced to the position where the
-        // compound token starts; add it now:
-
-        // The pruning we did when we created the altToken
-        // ensures that the back trace will align back with
-        // the start of the altToken:
-        assert altToken.getPosition() == backPos: altToken.getPosition() + " vs " + backPos;
-
-        // NOTE: not quite right: the compound token may
-        // have had all punctuation back traced so far, but
-        // then the decompounded token at this position is
-        // not punctuation.  In this case backCount is 0,
-        // but we should maybe add the altToken anyway...?
-
-        if (backCount > 0) {
-          backCount++;
-          altToken.setPositionLength(backCount);
-          if (VERBOSE) {
-            System.out.println("    add altToken=" + altToken);
-          }
-          pending.add(altToken);
-        } else {
-          // This means alt token was all punct tokens:
-          if (VERBOSE) {
-            System.out.println("    discard all-punctuation altToken=" + altToken);
+        if (outputCompounds) {
+          // We've backtraced to the position where the
+          // compound token starts; add it now:
+
+          // The pruning we did when we created the altToken
+          // ensures that the back trace will align back with
+          // the start of the altToken:
+          assert altToken.getPosition() == backPos : altToken.getPosition() + " vs " + backPos;
+
+          // NOTE: not quite right: the compound token may
+          // have had all punctuation back traced so far, but
+          // then the decompounded token at this position is
+          // not punctuation.  In this case backCount is 0,
+          // but we should maybe add the altToken anyway...?
+
+          if (backCount > 0) {
+            backCount++;
+            altToken.setPositionLength(backCount);
+            if (VERBOSE) {
+              System.out.println("    add altToken=" + altToken);
+            }
+            pending.add(altToken);
+          } else {
+            // This means alt token was all punct tokens:
+            if (VERBOSE) {
+              System.out.println("    discard all-punctuation altToken=" + altToken);
+            }
+            assert discardPunctuation;
           }
-          assert discardPunctuation;
         }
         altToken = null;
       }
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
index d6464d8..30836af 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
@@ -925,5 +925,11 @@ public class
 
     assertAnalyzesTo(extendedModeAnalyzerNoCompound, "株式会社とアカデミア",
         new String[]{"株式", "会社", "と", "ア", "カ", "デ", "ミ", "ア"});
+
+    assertAnalyzesTo(analyzer, "北海道日本ハムファイターズ",
+            new String[]{"北海道", "日本", "ハムファイターズ"});
+
+    assertAnalyzesTo(analyzerNoCompound, "北海道日本ハムファイターズ",
+            new String[]{"北海道", "日本", "ハムファイターズ"});
   }
 }