You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by br...@apache.org on 2024/06/11 10:04:41 UTC
(lucene) branch branch_9x updated: Replace TreeMap and use compiled Patterns in Japanese UserDictionary. (#13431)
This is an automated email from the ASF dual-hosted git repository.
broustant pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 3cce8507fbf Replace TreeMap and use compiled Patterns in Japanese UserDictionary. (#13431)
3cce8507fbf is described below
commit 3cce8507fbf4af5f128720dfabb00d7a8d0b7bfa
Author: Bruno Roustant <33...@users.noreply.github.com>
AuthorDate: Tue Jun 11 10:02:58 2024 +0200
Replace TreeMap and use compiled Patterns in Japanese UserDictionary. (#13431)
Replace TreeMap by a List of Match. Use compiled Pattern.
---
lucene/CHANGES.txt | 2 +
.../lucene/analysis/ja/dict/UserDictionary.java | 107 ++++++++++++---------
2 files changed, 62 insertions(+), 47 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d683d0b464b..ecc3a24e5c1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -30,6 +30,8 @@ Optimizations
* GITHUB#13454: MultiTermQuery returns null ScoreSupplier in cases where
no query terms are present in the index segment (Mayya Sharipova)
+* GITHUB#13431: Replace TreeMap and use compiled Patterns in Japanese UserDictionary. (Bruno Roustant)
+
Bug Fixes
---------------------
(No changes)
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
index 506b4de980f..e251a6b654e 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
@@ -23,8 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.util.CSVUtil;
import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.fst.FST;
@@ -34,6 +33,10 @@ import org.apache.lucene.util.fst.PositiveIntOutputs;
/** Class for building a User Dictionary. This class allows for custom segmentation of phrases. */
public final class UserDictionary implements Dictionary {
+ private static final Pattern LINE_COMMENT = Pattern.compile("^#.*$");
+ private static final Pattern WHITESPACE = Pattern.compile("\\s");
+ private static final Pattern SPACES = Pattern.compile(" +");
+
// phrase text -> phrase ID
private final TokenInfoFST fst;
@@ -54,16 +57,16 @@ public final class UserDictionary implements Dictionary {
public static UserDictionary open(Reader reader) throws IOException {
BufferedReader br = new BufferedReader(reader);
- String line = null;
+ String line;
List<String[]> featureEntries = new ArrayList<>();
// text, segmentation, readings, POS
while ((line = br.readLine()) != null) {
// Remove comments
- line = line.replaceAll("^#.*$", "");
+ line = LINE_COMMENT.matcher(line).replaceAll("");
// Skip empty lines or comment lines
- if (line.trim().length() == 0) {
+ if (line.trim().isEmpty()) {
continue;
}
String[] values = CSVUtil.parse(line);
@@ -102,10 +105,10 @@ public final class UserDictionary implements Dictionary {
long ord = 0;
for (String[] values : featureEntries) {
- String surface = values[0].replaceAll("\\s", "");
- String concatenatedSegment = values[1].replaceAll("\\s", "");
- String[] segmentation = values[1].replaceAll(" *", " ").split(" ");
- String[] readings = values[2].replaceAll(" *", " ").split(" ");
+ String surface = WHITESPACE.matcher(values[0]).replaceAll("");
+ String concatenatedSegment = WHITESPACE.matcher(values[1]).replaceAll("");
+ String[] segmentation = SPACES.split(values[1]);
+ String[] readings = SPACES.split(values[2]);
String pos = values[3];
if (segmentation.length != readings.length) {
@@ -144,7 +147,7 @@ public final class UserDictionary implements Dictionary {
scratch.growNoCopy(token.length());
scratch.setLength(token.length());
for (int i = 0; i < token.length(); i++) {
- scratch.setIntAt(i, (int) token.charAt(i));
+ scratch.setIntAt(i, token.charAt(i));
}
fstCompiler.add(scratch.get(), ord);
segmentations.add(wordIdAndLength);
@@ -153,8 +156,8 @@ public final class UserDictionary implements Dictionary {
this.fst =
new TokenInfoFST(
FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()), false);
- this.data = data.toArray(new String[data.size()]);
- this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
+ this.data = data.toArray(new String[0]);
+ this.segmentations = segmentations.toArray(new int[0][]);
}
/**
@@ -166,60 +169,57 @@ public final class UserDictionary implements Dictionary {
* @return array of {wordId, position, length}
*/
public int[][] lookup(char[] chars, int off, int len) throws IOException {
- // TODO: can we avoid this treemap/toIndexArray?
- TreeMap<Integer, int[]> result = new TreeMap<>(); // index, [length, length...]
- boolean found = false; // true if we found any results
-
+ List<Match> matches = null;
+ int numResults = 0;
final FST.BytesReader fstReader = fst.getBytesReader();
-
+ final int end = off + len;
FST.Arc<Long> arc = new FST.Arc<>();
- int end = off + len;
for (int startOffset = off; startOffset < end; startOffset++) {
+ int[] wordIdAndLength = null;
arc = fst.getFirstArc(arc);
int output = 0;
- int remaining = end - startOffset;
- for (int i = 0; i < remaining; i++) {
+ for (int i = 0, remaining = end - startOffset; i < remaining; i++) {
int ch = chars[startOffset + i];
if (fst.findTargetArc(ch, arc, arc, i == 0, fstReader) == null) {
break; // continue to next position
}
output += arc.output().intValue();
if (arc.isFinal()) {
- final int finalOutput = output + arc.nextFinalOutput().intValue();
- result.put(startOffset - off, segmentations[finalOutput]);
- found = true;
+ int finalOutput = output + arc.nextFinalOutput().intValue();
+ wordIdAndLength = segmentations[finalOutput];
+ }
+ }
+ if (wordIdAndLength != null) {
+ if (matches == null) {
+ matches = new ArrayList<>();
}
+ matches.add(new Match(startOffset - off, wordIdAndLength));
+ numResults += wordIdAndLength.length - 1;
}
}
-
- return found ? toIndexArray(result) : EMPTY_RESULT;
- }
-
- public TokenInfoFST getFST() {
- return fst;
- }
-
- private static final int[][] EMPTY_RESULT = new int[0][];
-
- /**
- * Convert Map of index and wordIdAndLength to array of {wordId, index, length}
- *
- * @return array of {wordId, index, length}
- */
- private int[][] toIndexArray(Map<Integer, int[]> input) {
- ArrayList<int[]> result = new ArrayList<>();
- for (Map.Entry<Integer, int[]> entry : input.entrySet()) {
- int[] wordIdAndLength = entry.getValue();
+ if (numResults == 0) {
+ return EMPTY_RESULT;
+ }
+ int[][] result = new int[numResults][];
+ int index = 0;
+ for (int i = 0; i < matches.size(); i++) {
+ Match match = matches.get(i);
+ int[] wordIdAndLength = match.wordIdAndLength;
int wordId = wordIdAndLength[0];
// convert length to index
- int current = entry.getKey();
+ int position = match.position;
for (int j = 1; j < wordIdAndLength.length; j++) { // first entry is wordId offset
- int[] token = {wordId + j - 1, current, wordIdAndLength[j]};
- result.add(token);
- current += wordIdAndLength[j];
+ // add a {wordId, index, length} token to the results
+ int[] token = {wordId + j - 1, position, wordIdAndLength[j]};
+ result[index++] = token;
+ position += wordIdAndLength[j];
}
}
- return result.toArray(new int[result.size()][]);
+ return result;
+ }
+
+ public TokenInfoFST getFST() {
+ return fst;
}
public int[] lookupSegmentation(int phraseID) {
@@ -299,4 +299,17 @@ public final class UserDictionary implements Dictionary {
}
return sb.deleteCharAt(sb.length() - 1).toString();
}
+
+ private static final int[][] EMPTY_RESULT = new int[0][];
+
+ private static class Match {
+
+ final int position;
+ final int[] wordIdAndLength;
+
+ Match(int position, int[] wordIdAndLength) {
+ this.position = position;
+ this.wordIdAndLength = wordIdAndLength;
+ }
+ }
}