You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/01/19 08:23:14 UTC
[lucene-solr] branch master updated: LUCENE-9677: simplify
Dictionary.affixData storage (#2218)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 426c902 LUCENE-9677: simplify Dictionary.affixData storage (#2218)
426c902 is described below
commit 426c902bc96c0d5023622daca20e09c9bfbab63b
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Tue Jan 19 09:22:33 2021 +0100
LUCENE-9677: simplify Dictionary.affixData storage (#2218)
Use char[] instead of byte[], get rid of unnecessary byte array readers/writers.
---
.../lucene/analysis/hunspell/Dictionary.java | 29 ++++++++++++++--------
.../apache/lucene/analysis/hunspell/Stemmer.java | 27 +++++++-------------
2 files changed, 27 insertions(+), 29 deletions(-)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index b932bf8..81b4bee 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -45,7 +45,6 @@ import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -118,10 +117,16 @@ public class Dictionary {
char[] stripData;
int[] stripOffsets;
- // 8 bytes per affix
- byte[] affixData = new byte[64];
+ // 4 chars per affix, each char representing an unsigned 2-byte integer
+ char[] affixData = new char[32];
private int currentAffix = 0;
+ // offsets in affixData
+ static final int AFFIX_FLAG = 0;
+ static final int AFFIX_STRIP_ORD = 1;
+ static final int AFFIX_CONDITION = 2;
+ static final int AFFIX_APPEND = 3;
+
// Default flag parsing strategy
private FlagParsingStrategy flagParsingStrategy = new SimpleFlagParsingStrategy();
@@ -480,12 +485,9 @@ public class Dictionary {
boolean isSuffix = conditionPattern.equals(SUFFIX_CONDITION_REGEX_PATTERN);
int numLines = Integer.parseInt(args[3]);
- affixData = ArrayUtil.grow(affixData, (currentAffix << 3) + (numLines << 3));
- ByteArrayDataOutput affixWriter =
- new ByteArrayDataOutput(affixData, currentAffix << 3, numLines << 3);
+ affixData = ArrayUtil.grow(affixData, currentAffix * 4 + numLines * 4);
for (int i = 0; i < numLines; i++) {
- assert affixWriter.getPosition() == currentAffix << 3;
String line = reader.readLine();
String[] ruleArgs = line.split("\\s+");
@@ -581,12 +583,13 @@ public class Dictionary {
"Too many unique append flags, please report this to dev@lucene.apache.org");
}
- affixWriter.writeShort((short) flag);
- affixWriter.writeShort((short) stripOrd.intValue());
+ int dataStart = currentAffix * 4;
+ affixData[dataStart + AFFIX_FLAG] = flag;
+ affixData[dataStart + AFFIX_STRIP_ORD] = (char) stripOrd.intValue();
// encode crossProduct into patternIndex
int patternOrd = patternIndex << 1 | (crossProduct ? 1 : 0);
- affixWriter.writeShort((short) patternOrd);
- affixWriter.writeShort((short) appendFlagsOrd);
+ affixData[dataStart + AFFIX_CONDITION] = (char) patternOrd;
+ affixData[dataStart + AFFIX_APPEND] = (char) appendFlagsOrd;
if (needsInputCleaning) {
CharSequence cleaned = cleanInput(affixArg, sb);
@@ -602,6 +605,10 @@ public class Dictionary {
}
}
+ char affixData(int affixIndex, int offset) {
+ return affixData[affixIndex * 4 + offset];
+ }
+
private FST<CharsRef> parseConversions(LineNumberReader reader, int num)
throws IOException, ParseException {
Map<String, String> mappings = new TreeMap<>();
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
index 0f3143a..7b50da2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
@@ -39,7 +38,6 @@ final class Stemmer {
private final Dictionary dictionary;
private final BytesRef scratch = new BytesRef();
private final StringBuilder segment = new StringBuilder();
- private final ByteArrayDataInput affixReader;
// used for normalization
private final StringBuilder scratchSegment = new StringBuilder();
@@ -56,7 +54,6 @@ final class Stemmer {
*/
public Stemmer(Dictionary dictionary) {
this.dictionary = dictionary;
- this.affixReader = new ByteArrayDataInput(dictionary.affixData);
for (int level = 0; level < 3; level++) {
if (dictionary.prefixes != null) {
prefixArcs[level] = new FST.Arc<>();
@@ -339,13 +336,10 @@ final class Stemmer {
if (prefix == previous) {
continue;
}
- affixReader.setPosition(8 * prefix);
- char flag = (char) (affixReader.readShort() & 0xffff);
- char stripOrd = (char) (affixReader.readShort() & 0xffff);
- int condition = (char) (affixReader.readShort() & 0xffff);
+ int condition = dictionary.affixData(prefix, Dictionary.AFFIX_CONDITION);
boolean crossProduct = (condition & 1) == 1;
condition >>>= 1;
- char append = (char) (affixReader.readShort() & 0xffff);
+ int append = dictionary.affixData(prefix, Dictionary.AFFIX_APPEND);
final boolean compatible;
if (recursionDepth == 0) {
@@ -373,6 +367,7 @@ final class Stemmer {
if (compatible) {
int deAffixedLength = length - i;
+ int stripOrd = dictionary.affixData(prefix, Dictionary.AFFIX_STRIP_ORD);
int stripStart = dictionary.stripOffsets[stripOrd];
int stripEnd = dictionary.stripOffsets[stripOrd + 1];
int stripLength = stripEnd - stripStart;
@@ -437,13 +432,10 @@ final class Stemmer {
if (suffix == previous) {
continue;
}
- affixReader.setPosition(8 * suffix);
- char flag = (char) (affixReader.readShort() & 0xffff);
- char stripOrd = (char) (affixReader.readShort() & 0xffff);
- int condition = (char) (affixReader.readShort() & 0xffff);
+ int condition = dictionary.affixData(suffix, Dictionary.AFFIX_CONDITION);
boolean crossProduct = (condition & 1) == 1;
condition >>>= 1;
- char append = (char) (affixReader.readShort() & 0xffff);
+ int append = dictionary.affixData(suffix, Dictionary.AFFIX_APPEND);
final boolean compatible;
if (recursionDepth == 0) {
@@ -473,6 +465,7 @@ final class Stemmer {
int appendLength = length - i;
int deAffixedLength = length - appendLength;
+ int stripOrd = dictionary.affixData(suffix, Dictionary.AFFIX_STRIP_ORD);
int stripStart = dictionary.stripOffsets[stripOrd];
int stripEnd = dictionary.stripOffsets[stripOrd + 1];
int stripLength = stripEnd - stripStart;
@@ -563,11 +556,9 @@ final class Stemmer {
boolean caseVariant)
throws IOException {
// TODO: just pass this in from before, no need to decode it twice
- affixReader.setPosition(8 * affix);
- char flag = (char) (affixReader.readShort() & 0xffff);
- affixReader.skipBytes(2); // strip
- boolean crossProduct = ((int) (char) (affixReader.readShort() & 0xffff) & 1) == 1;
- char append = (char) (affixReader.readShort() & 0xffff);
+ char flag = dictionary.affixData(affix, Dictionary.AFFIX_FLAG);
+ boolean crossProduct = (dictionary.affixData(affix, Dictionary.AFFIX_CONDITION) & 1) == 1;
+ char append = dictionary.affixData(affix, Dictionary.AFFIX_APPEND);
List<CharsRef> stems = new ArrayList<>();