You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/02/27 18:19:16 UTC
svn commit: r1572660 - in
/lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2:
Affix.java Dictionary.java Stemmer.java
Author: rmuir
Date: Thu Feb 27 17:19:15 2014
New Revision: 1572660
URL: http://svn.apache.org/r1572660
Log:
LUCENE-5468: encode affix data as 8 bytes per affix, before cutting over to FST
Removed:
lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Affix.java
Modified:
lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Dictionary.java
lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Stemmer.java
Modified: lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Dictionary.java?rev=1572660&r1=1572659&r2=1572660&view=diff
==============================================================================
--- lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Dictionary.java (original)
+++ lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Dictionary.java Thu Feb 27 17:19:15 2014
@@ -18,6 +18,8 @@ package org.apache.lucene.analysis.hunsp
*/
import org.apache.lucene.analysis.util.CharArrayMap;
+import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.IOUtils;
@@ -66,8 +68,8 @@ public class Dictionary {
private static final String PREFIX_CONDITION_REGEX_PATTERN = "%s.*";
private static final String SUFFIX_CONDITION_REGEX_PATTERN = ".*%s";
- public CharArrayMap<List<Affix>> prefixes;
- public CharArrayMap<List<Affix>> suffixes;
+ public CharArrayMap<List<Character>> prefixes;
+ public CharArrayMap<List<Character>> suffixes;
// all Patterns used by prefixes and suffixes. these are typically re-used across
// many affix stripping rules. so these are deduplicated, to save RAM.
@@ -84,6 +86,10 @@ public class Dictionary {
// the list of unique strip affixes.
public BytesRefHash stripLookup = new BytesRefHash();
+
+ // 8 bytes per affix
+ public byte[] affixData = new byte[64];
+ private int currentAffix = 0;
private FlagParsingStrategy flagParsingStrategy = new SimpleFlagParsingStrategy(); // Default flag parsing strategy
@@ -169,7 +175,7 @@ public class Dictionary {
* @param length Length from the offset that the String is
* @return List of HunspellAffix prefixes with an append that matches the String, or {@code null} if none are found
*/
- public List<Affix> lookupPrefix(char word[], int offset, int length) {
+ public List<Character> lookupPrefix(char word[], int offset, int length) {
return prefixes.get(word, offset, length);
}
@@ -181,7 +187,7 @@ public class Dictionary {
* @param length Length from the offset that the String is
* @return List of HunspellAffix suffixes with an append that matches the String, or {@code null} if none are found
*/
- List<Affix> lookupSuffix(char word[], int offset, int length) {
+ List<Character> lookupSuffix(char word[], int offset, int length) {
return suffixes.get(word, offset, length);
}
@@ -193,8 +199,8 @@ public class Dictionary {
* @throws IOException Can be thrown while reading from the InputStream
*/
private void readAffixFile(InputStream affixStream, CharsetDecoder decoder) throws IOException, ParseException {
- prefixes = new CharArrayMap<List<Affix>>(Version.LUCENE_CURRENT, 8, false);
- suffixes = new CharArrayMap<List<Affix>>(Version.LUCENE_CURRENT, 8, false);
+ prefixes = new CharArrayMap<List<Character>>(Version.LUCENE_CURRENT, 8, false);
+ suffixes = new CharArrayMap<List<Character>>(Version.LUCENE_CURRENT, 8, false);
Map<String,Integer> seenPatterns = new HashMap<>();
LineNumberReader reader = new LineNumberReader(new InputStreamReader(affixStream, decoder));
@@ -225,7 +231,7 @@ public class Dictionary {
* @param seenPatterns map from condition -> index of patterns, for deduplication.
* @throws IOException Can be thrown while reading the rule
*/
- private void parseAffix(CharArrayMap<List<Affix>> affixes,
+ private void parseAffix(CharArrayMap<List<Character>> affixes,
String header,
LineNumberReader reader,
String conditionPattern,
@@ -237,14 +243,20 @@ public class Dictionary {
boolean crossProduct = args[2].equals("Y");
int numLines = Integer.parseInt(args[3]);
+ affixData = ArrayUtil.grow(affixData, (currentAffix << 3) + (numLines << 3));
+ ByteArrayDataOutput affixWriter = new ByteArrayDataOutput(affixData, currentAffix << 3, numLines << 3);
+
for (int i = 0; i < numLines; i++) {
+ if (currentAffix > Short.MAX_VALUE) {
+ throw new UnsupportedOperationException("Too many affixes, please report this to dev@lucene.apache.org");
+ }
+ assert affixWriter.getPosition() == currentAffix << 3;
String line = reader.readLine();
String ruleArgs[] = line.split("\\s+");
if (ruleArgs.length < 5) {
throw new ParseException("The affix file contains a rule with less than five elements", reader.getLineNumber());
}
-
char flag = flagParsingStrategy.parseFlag(ruleArgs[1]);
String strip = ruleArgs[2].equals("0") ? "" : ruleArgs[2];
@@ -285,36 +297,42 @@ public class Dictionary {
patterns.add(pattern);
}
- Affix affix = new Affix();
scratch.copyChars(strip);
- int ord = stripLookup.add(scratch);
- if (ord < 0) {
+ int stripOrd = stripLookup.add(scratch);
+ if (stripOrd < 0) {
// already exists in our hash
- ord = (-ord)-1;
+ stripOrd = (-stripOrd)-1;
}
- affix.setStrip(ord);
- affix.setFlag(flag);
- affix.setCondition(patternIndex);
- affix.setCrossProduct(crossProduct);
+
if (appendFlags == null) {
appendFlags = NOFLAGS;
}
final int hashCode = encodeFlagsWithHash(scratch, appendFlags);
- ord = flagLookup.add(scratch, hashCode);
- if (ord < 0) {
+ int appendFlagsOrd = flagLookup.add(scratch, hashCode);
+ if (appendFlagsOrd < 0) {
// already exists in our hash
- ord = (-ord)-1;
+ appendFlagsOrd = (-appendFlagsOrd)-1;
+ } else if (appendFlagsOrd > Short.MAX_VALUE) {
+ // this limit is probably flexible, but its a good sanity check too
+ throw new UnsupportedOperationException("Too many unique flags, please report this to dev@lucene.apache.org");
}
- affix.setAppendFlags(ord);
- List<Affix> list = affixes.get(affixArg);
+ affixWriter.writeShort((short)flag);
+ affixWriter.writeShort((short)stripOrd);
+ // encode crossProduct into patternIndex
+ int patternOrd = patternIndex.intValue() << 1 | (crossProduct ? 1 : 0);
+ affixWriter.writeShort((short)patternOrd);
+ affixWriter.writeShort((short)appendFlagsOrd);
+
+ List<Character> list = affixes.get(affixArg);
if (list == null) {
- list = new ArrayList<Affix>();
+ list = new ArrayList<Character>();
affixes.put(affixArg, list);
}
- list.add(affix);
+ list.add((char)currentAffix);
+ currentAffix++;
}
}
Modified: lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Stemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Stemmer.java?rev=1572660&r1=1572659&r2=1572660&view=diff
==============================================================================
--- lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Stemmer.java (original)
+++ lucene/dev/branches/lucene5468/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell2/Stemmer.java Thu Feb 27 17:19:15 2014
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.regex.Pattern;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.Version;
@@ -37,6 +38,7 @@ final class Stemmer {
private final Dictionary dictionary;
private BytesRef scratch = new BytesRef();
private final StringBuilder segment = new StringBuilder();
+ private final ByteArrayDataInput affixReader;
/**
* Constructs a new Stemmer which will use the provided Dictionary to create its stems. Uses the
@@ -56,6 +58,7 @@ final class Stemmer {
*/
public Stemmer(Dictionary dictionary, int recursionCap) {
this.dictionary = dictionary;
+ this.affixReader = new ByteArrayDataInput(dictionary.affixData);
this.recursionCap = recursionCap;
}
@@ -122,17 +125,20 @@ final class Stemmer {
List<CharsRef> stems = new ArrayList<CharsRef>();
for (int i = 0; i < length; i++) {
- List<Affix> suffixes = dictionary.lookupSuffix(word, i, length - i);
+ List<Character> suffixes = dictionary.lookupSuffix(word, i, length - i);
if (suffixes == null) {
continue;
}
- for (Affix suffix : suffixes) {
- if (hasCrossCheckedFlag(suffix.getFlag(), flags)) {
+ for (Character suffix : suffixes) {
+ affixReader.setPosition(8 * suffix);
+ char flag = (char) (affixReader.readShort() & 0xffff);
+ if (hasCrossCheckedFlag(flag, flags)) {
int appendLength = length - i;
int deAffixedLength = length - appendLength;
// TODO: can we do this in-place?
- dictionary.stripLookup.get(suffix.getStrip(), scratch);
+ char stripOrd = (char) (affixReader.readShort() & 0xffff);
+ dictionary.stripLookup.get(stripOrd, scratch);
String strippedWord = new StringBuilder().append(word, 0, deAffixedLength).append(scratch.utf8ToString()).toString();
List<CharsRef> stemList = applyAffix(strippedWord.toCharArray(), strippedWord.length(), suffix, recursionDepth);
@@ -143,17 +149,20 @@ final class Stemmer {
}
for (int i = length - 1; i >= 0; i--) {
- List<Affix> prefixes = dictionary.lookupPrefix(word, 0, i);
+ List<Character> prefixes = dictionary.lookupPrefix(word, 0, i);
if (prefixes == null) {
continue;
}
- for (Affix prefix : prefixes) {
- if (hasCrossCheckedFlag(prefix.getFlag(), flags)) {
+ for (Character prefix : prefixes) {
+ affixReader.setPosition(8 * prefix);
+ char flag = (char) (affixReader.readShort() & 0xffff);
+ if (hasCrossCheckedFlag(flag, flags)) {
int deAffixedStart = i;
int deAffixedLength = length - deAffixedStart;
+ char stripOrd = (char) (affixReader.readShort() & 0xffff);
- dictionary.stripLookup.get(prefix.getStrip(), scratch);
+ dictionary.stripLookup.get(stripOrd, scratch);
String strippedWord = new StringBuilder().append(scratch.utf8ToString())
.append(word, deAffixedStart, deAffixedLength)
.toString();
@@ -176,11 +185,19 @@ final class Stemmer {
* @param recursionDepth Level of recursion this stemming step is at
* @return List of stems for the word, or an empty list if none are found
*/
- public List<CharsRef> applyAffix(char strippedWord[], int length, Affix affix, int recursionDepth) {
+ public List<CharsRef> applyAffix(char strippedWord[], int length, char affix, int recursionDepth) {
segment.setLength(0);
segment.append(strippedWord, 0, length);
- Pattern pattern = dictionary.patterns.get(affix.getCondition());
+ affixReader.setPosition(8 * affix);
+ char flag = (char) (affixReader.readShort() & 0xffff);
+ affixReader.skipBytes(2); // strip
+ int condition = (char) (affixReader.readShort() & 0xffff);
+ boolean crossProduct = (condition & 1) == 1;
+ condition >>>= 1;
+ char append = (char) (affixReader.readShort() & 0xffff);
+
+ Pattern pattern = dictionary.patterns.get(condition);
if (!pattern.matcher(segment).matches()) {
return Collections.emptyList();
}
@@ -188,12 +205,12 @@ final class Stemmer {
List<CharsRef> stems = new ArrayList<CharsRef>();
char wordFlags[] = dictionary.lookupWord(strippedWord, 0, length, scratch);
- if (wordFlags != null && Dictionary.hasFlag(wordFlags, affix.getFlag())) {
+ if (wordFlags != null && Dictionary.hasFlag(wordFlags, flag)) {
stems.add(new CharsRef(strippedWord, 0, length));
}
- if (affix.isCrossProduct() && recursionDepth < recursionCap) {
- dictionary.flagLookup.get(affix.getAppendFlags(), scratch);
+ if (crossProduct && recursionDepth < recursionCap) {
+ dictionary.flagLookup.get(append, scratch);
char appendFlags[] = Dictionary.decodeFlags(scratch);
stems.addAll(stem(strippedWord, length, appendFlags, ++recursionDepth));
}