You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/03/16 20:39:37 UTC
svn commit: r1578144 [3/37] - in /lucene/dev/branches/lucene5376_2: ./
dev-tools/ dev-tools/idea/.idea/libraries/
dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/contrib/map-reduce/ dev-tools/idea/solr/core/src/test/
dev-tools/script...
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Stemmer.java Sun Mar 16 19:39:10 2014
@@ -17,52 +17,43 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.regex.Pattern;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
/**
* Stemmer uses the affix rules declared in the Dictionary to generate one or more stems for a word. It
* conforms to the algorithm in the original hunspell algorithm, including recursive suffix stripping.
*/
final class Stemmer {
- private final int recursionCap;
private final Dictionary dictionary;
private final BytesRef scratch = new BytesRef();
private final StringBuilder segment = new StringBuilder();
private final ByteArrayDataInput affixReader;
- private final CharacterUtils charUtils = CharacterUtils.getInstance(Version.LUCENE_CURRENT);
-
- /**
- * Constructs a new Stemmer which will use the provided Dictionary to create its stems. Uses the
- * default recursion cap of <code>2</code> (based on Hunspell documentation).
- *
- * @param dictionary Dictionary that will be used to create the stems
- */
- public Stemmer(Dictionary dictionary) {
- this(dictionary, 2);
- }
-
+
+ // used for normalization
+ private final StringBuilder scratchSegment = new StringBuilder();
+ private char scratchBuffer[] = new char[32];
+
/**
* Constructs a new Stemmer which will use the provided Dictionary to create its stems.
*
* @param dictionary Dictionary that will be used to create the stems
- * @param recursionCap maximum level of recursion stemmer can go into
*/
- public Stemmer(Dictionary dictionary, int recursionCap) {
+ public Stemmer(Dictionary dictionary) {
this.dictionary = dictionary;
this.affixReader = new ByteArrayDataInput(dictionary.affixData);
- this.recursionCap = recursionCap;
}
/**
@@ -81,15 +72,28 @@ final class Stemmer {
* @param word Word to find the stems for
* @return List of stems for the word
*/
- public List<CharsRef> stem(char word[], int length) {
- if (dictionary.ignoreCase) {
- charUtils.toLowerCase(word, 0, length);
- }
- List<CharsRef> stems = new ArrayList<CharsRef>();
- if (dictionary.lookupWord(word, 0, length, scratch) != null) {
- stems.add(new CharsRef(word, 0, length));
+ public List<CharsRef> stem(char word[], int length) {
+
+ if (dictionary.needsInputCleaning) {
+ scratchSegment.setLength(0);
+ scratchSegment.append(word, 0, length);
+ CharSequence cleaned = dictionary.cleanInput(scratchSegment, segment);
+ scratchBuffer = ArrayUtil.grow(scratchBuffer, cleaned.length());
+ length = segment.length();
+ segment.getChars(0, length, scratchBuffer, 0);
+ word = scratchBuffer;
}
- stems.addAll(stem(word, length, Dictionary.NOFLAGS, 0));
+
+ List<CharsRef> stems = new ArrayList<>();
+ IntsRef forms = dictionary.lookupWord(word, 0, length);
+ if (forms != null) {
+ // TODO: some forms should not be added, e.g. ONLYINCOMPOUND
+ // just because it exists, does not make it valid...
+ for (int i = 0; i < forms.length; i++) {
+ stems.add(newStem(word, length));
+ }
+ }
+ stems.addAll(stem(word, length, -1, -1, -1, 0, true, true, false, false));
return stems;
}
@@ -114,6 +118,23 @@ final class Stemmer {
}
return deduped;
}
+
+ private CharsRef newStem(char buffer[], int length) {
+ if (dictionary.needsOutputCleaning) {
+ scratchSegment.setLength(0);
+ scratchSegment.append(buffer, 0, length);
+ try {
+ Dictionary.applyMappings(dictionary.oconv, scratchSegment);
+ } catch (IOException bogus) {
+ throw new RuntimeException(bogus);
+ }
+ char cleaned[] = new char[scratchSegment.length()];
+ scratchSegment.getChars(0, cleaned.length, cleaned, 0);
+ return new CharsRef(cleaned, 0, cleaned.length);
+ } else {
+ return new CharsRef(buffer, 0, length);
+ }
+ }
// ================================================= Helper Methods ================================================
@@ -121,81 +142,179 @@ final class Stemmer {
* Generates a list of stems for the provided word
*
* @param word Word to generate the stems for
- * @param flags Flags from a previous stemming step that need to be cross-checked with any affixes in this recursive step
- * @param recursionDepth Level of recursion this stemming step is at
+ * @param previous previous affix that was removed (so we dont remove same one twice)
+ * @param prevFlag Flag from a previous stemming step that need to be cross-checked with any affixes in this recursive step
+ * @param prefixFlag flag of the most inner removed prefix, so that when removing a suffix, its also checked against the word
+ * @param recursionDepth current recursiondepth
+ * @param doPrefix true if we should remove prefixes
+ * @param doSuffix true if we should remove suffixes
+ * @param previousWasPrefix true if the previous removal was a prefix:
+ * if we are removing a suffix, and it has no continuation requirements, its ok.
+ * but two prefixes (COMPLEXPREFIXES) or two suffixes must have continuation requirements to recurse.
+ * @param circumfix true if the previous prefix removal was signed as a circumfix
+ * this means inner most suffix must also contain circumfix flag.
* @return List of stems, or empty list if no stems are found
*/
- private List<CharsRef> stem(char word[], int length, char[] flags, int recursionDepth) {
+ private List<CharsRef> stem(char word[], int length, int previous, int prevFlag, int prefixFlag, int recursionDepth, boolean doPrefix, boolean doSuffix, boolean previousWasPrefix, boolean circumfix) {
+
// TODO: allow this stuff to be reused by tokenfilter
- List<CharsRef> stems = new ArrayList<CharsRef>();
-
- for (int i = 0; i < length; i++) {
- IntsRef suffixes = dictionary.lookupSuffix(word, i, length - i);
- if (suffixes == null) {
- continue;
- }
-
- for (int j = 0; j < suffixes.length; j++) {
- int suffix = suffixes.ints[suffixes.offset + j];
- affixReader.setPosition(8 * suffix);
- char flag = (char) (affixReader.readShort() & 0xffff);
- if (hasCrossCheckedFlag(flag, flags)) {
- int appendLength = length - i;
- int deAffixedLength = length - appendLength;
- // TODO: can we do this in-place?
+ List<CharsRef> stems = new ArrayList<>();
+
+ if (doPrefix && dictionary.prefixes != null) {
+ for (int i = length - 1; i >= 0; i--) {
+ IntsRef prefixes = dictionary.lookupPrefix(word, 0, i);
+ if (prefixes == null) {
+ continue;
+ }
+
+ for (int j = 0; j < prefixes.length; j++) {
+ int prefix = prefixes.ints[prefixes.offset + j];
+ if (prefix == previous) {
+ continue;
+ }
+ affixReader.setPosition(8 * prefix);
+ char flag = (char) (affixReader.readShort() & 0xffff);
char stripOrd = (char) (affixReader.readShort() & 0xffff);
- dictionary.stripLookup.get(stripOrd, scratch);
- String strippedWord = new StringBuilder().append(word, 0, deAffixedLength).append(scratch.utf8ToString()).toString();
-
- List<CharsRef> stemList = applyAffix(strippedWord.toCharArray(), strippedWord.length(), suffix, recursionDepth);
-
- stems.addAll(stemList);
+ int condition = (char) (affixReader.readShort() & 0xffff);
+ boolean crossProduct = (condition & 1) == 1;
+ condition >>>= 1;
+ char append = (char) (affixReader.readShort() & 0xffff);
+
+ final boolean compatible;
+ if (recursionDepth == 0) {
+ compatible = true;
+ } else if (crossProduct) {
+ // cross check incoming continuation class (flag of previous affix) against list.
+ dictionary.flagLookup.get(append, scratch);
+ char appendFlags[] = Dictionary.decodeFlags(scratch);
+ assert prevFlag >= 0;
+ compatible = hasCrossCheckedFlag((char)prevFlag, appendFlags, false);
+ } else {
+ compatible = false;
+ }
+
+ if (compatible) {
+ int deAffixedStart = i;
+ int deAffixedLength = length - deAffixedStart;
+
+ int stripStart = dictionary.stripOffsets[stripOrd];
+ int stripEnd = dictionary.stripOffsets[stripOrd+1];
+ int stripLength = stripEnd - stripStart;
+
+ if (!checkCondition(condition, dictionary.stripData, stripStart, stripLength, word, deAffixedStart, deAffixedLength)) {
+ continue;
+ }
+
+ char strippedWord[] = new char[stripLength + deAffixedLength];
+ System.arraycopy(dictionary.stripData, stripStart, strippedWord, 0, stripLength);
+ System.arraycopy(word, deAffixedStart, strippedWord, stripLength, deAffixedLength);
+
+ List<CharsRef> stemList = applyAffix(strippedWord, strippedWord.length, prefix, -1, recursionDepth, true, circumfix);
+
+ stems.addAll(stemList);
+ }
}
}
- }
-
- for (int i = length - 1; i >= 0; i--) {
- IntsRef prefixes = dictionary.lookupPrefix(word, 0, i);
- if (prefixes == null) {
- continue;
- }
-
- for (int j = 0; j < prefixes.length; j++) {
- int prefix = prefixes.ints[prefixes.offset + j];
- affixReader.setPosition(8 * prefix);
- char flag = (char) (affixReader.readShort() & 0xffff);
- if (hasCrossCheckedFlag(flag, flags)) {
- int deAffixedStart = i;
- int deAffixedLength = length - deAffixedStart;
+ }
+
+ if (doSuffix && dictionary.suffixes != null) {
+ for (int i = 0; i < length; i++) {
+ IntsRef suffixes = dictionary.lookupSuffix(word, i, length - i);
+ if (suffixes == null) {
+ continue;
+ }
+
+ for (int j = 0; j < suffixes.length; j++) {
+ int suffix = suffixes.ints[suffixes.offset + j];
+ if (suffix == previous) {
+ continue;
+ }
+ affixReader.setPosition(8 * suffix);
+ char flag = (char) (affixReader.readShort() & 0xffff);
char stripOrd = (char) (affixReader.readShort() & 0xffff);
-
- dictionary.stripLookup.get(stripOrd, scratch);
- String strippedWord = new StringBuilder().append(scratch.utf8ToString())
- .append(word, deAffixedStart, deAffixedLength)
- .toString();
-
- List<CharsRef> stemList = applyAffix(strippedWord.toCharArray(), strippedWord.length(), prefix, recursionDepth);
-
- stems.addAll(stemList);
+ int condition = (char) (affixReader.readShort() & 0xffff);
+ boolean crossProduct = (condition & 1) == 1;
+ condition >>>= 1;
+ char append = (char) (affixReader.readShort() & 0xffff);
+
+ final boolean compatible;
+ if (recursionDepth == 0) {
+ compatible = true;
+ } else if (crossProduct) {
+ // cross check incoming continuation class (flag of previous affix) against list.
+ dictionary.flagLookup.get(append, scratch);
+ char appendFlags[] = Dictionary.decodeFlags(scratch);
+ assert prevFlag >= 0;
+ compatible = hasCrossCheckedFlag((char)prevFlag, appendFlags, previousWasPrefix);
+ } else {
+ compatible = false;
+ }
+
+ if (compatible) {
+ int appendLength = length - i;
+ int deAffixedLength = length - appendLength;
+
+ int stripStart = dictionary.stripOffsets[stripOrd];
+ int stripEnd = dictionary.stripOffsets[stripOrd+1];
+ int stripLength = stripEnd - stripStart;
+
+ if (!checkCondition(condition, word, 0, deAffixedLength, dictionary.stripData, stripStart, stripLength)) {
+ continue;
+ }
+
+ char strippedWord[] = new char[stripLength + deAffixedLength];
+ System.arraycopy(word, 0, strippedWord, 0, deAffixedLength);
+ System.arraycopy(dictionary.stripData, stripStart, strippedWord, deAffixedLength, stripLength);
+
+ List<CharsRef> stemList = applyAffix(strippedWord, strippedWord.length, suffix, prefixFlag, recursionDepth, false, circumfix);
+
+ stems.addAll(stemList);
+ }
}
}
}
return stems;
}
+
+ /** checks condition of the concatenation of two strings */
+ // note: this is pretty stupid, we really should subtract strip from the condition up front and just check the stem
+ // but this is a little bit more complicated.
+ private boolean checkCondition(int condition, char c1[], int c1off, int c1len, char c2[], int c2off, int c2len) {
+ if (condition != 0) {
+ CharacterRunAutomaton pattern = dictionary.patterns.get(condition);
+ int state = pattern.getInitialState();
+ for (int i = c1off; i < c1off + c1len; i++) {
+ state = pattern.step(state, c1[i]);
+ if (state == -1) {
+ return false;
+ }
+ }
+ for (int i = c2off; i < c2off + c2len; i++) {
+ state = pattern.step(state, c2[i]);
+ if (state == -1) {
+ return false;
+ }
+ }
+ return pattern.isAccept(state);
+ }
+ return true;
+ }
/**
* Applies the affix rule to the given word, producing a list of stems if any are found
*
* @param strippedWord Word the affix has been removed and the strip added
+ * @param length valid length of stripped word
* @param affix HunspellAffix representing the affix rule itself
- * @param recursionDepth Level of recursion this stemming step is at
+ * @param prefixFlag when we already stripped a prefix, we cant simply recurse and check the suffix, unless both are compatible
+ * so we must check dictionary form against both to add it as a stem!
+ * @param recursionDepth current recursion depth
+ * @param prefix true if we are removing a prefix (false if its a suffix)
* @return List of stems for the word, or an empty list if none are found
*/
- List<CharsRef> applyAffix(char strippedWord[], int length, int affix, int recursionDepth) {
- segment.setLength(0);
- segment.append(strippedWord, 0, length);
-
+ List<CharsRef> applyAffix(char strippedWord[], int length, int affix, int prefixFlag, int recursionDepth, boolean prefix, boolean circumfix) {
+ // TODO: just pass this in from before, no need to decode it twice
affixReader.setPosition(8 * affix);
char flag = (char) (affixReader.readShort() & 0xffff);
affixReader.skipBytes(2); // strip
@@ -204,22 +323,70 @@ final class Stemmer {
condition >>>= 1;
char append = (char) (affixReader.readShort() & 0xffff);
- Pattern pattern = dictionary.patterns.get(condition);
- if (!pattern.matcher(segment).matches()) {
- return Collections.emptyList();
- }
-
- List<CharsRef> stems = new ArrayList<CharsRef>();
+ List<CharsRef> stems = new ArrayList<>();
- char wordFlags[] = dictionary.lookupWord(strippedWord, 0, length, scratch);
- if (wordFlags != null && Dictionary.hasFlag(wordFlags, flag)) {
- stems.add(new CharsRef(strippedWord, 0, length));
+ IntsRef forms = dictionary.lookupWord(strippedWord, 0, length);
+ if (forms != null) {
+ for (int i = 0; i < forms.length; i++) {
+ dictionary.flagLookup.get(forms.ints[forms.offset+i], scratch);
+ char wordFlags[] = Dictionary.decodeFlags(scratch);
+ if (Dictionary.hasFlag(wordFlags, flag)) {
+ // confusing: in this one exception, we already chained the first prefix against the second,
+ // so it doesnt need to be checked against the word
+ boolean chainedPrefix = dictionary.complexPrefixes && recursionDepth == 1 && prefix;
+ if (chainedPrefix == false && prefixFlag >= 0 && !Dictionary.hasFlag(wordFlags, (char)prefixFlag)) {
+ // see if we can chain prefix thru the suffix continuation class (only if it has any!)
+ dictionary.flagLookup.get(append, scratch);
+ char appendFlags[] = Dictionary.decodeFlags(scratch);
+ if (!hasCrossCheckedFlag((char)prefixFlag, appendFlags, false)) {
+ continue;
+ }
+ }
+
+ // if circumfix was previously set by a prefix, we must check this suffix,
+ // to ensure it has it, and vice versa
+ if (dictionary.circumfix != -1) {
+ dictionary.flagLookup.get(append, scratch);
+ char appendFlags[] = Dictionary.decodeFlags(scratch);
+ boolean suffixCircumfix = Dictionary.hasFlag(appendFlags, (char)dictionary.circumfix);
+ if (circumfix != suffixCircumfix) {
+ continue;
+ }
+ }
+ stems.add(newStem(strippedWord, length));
+ }
+ }
}
-
- if (crossProduct && recursionDepth < recursionCap) {
+
+ // if a circumfix flag is defined in the dictionary, and we are a prefix, we need to check if we have that flag
+ if (dictionary.circumfix != -1 && !circumfix && prefix) {
dictionary.flagLookup.get(append, scratch);
char appendFlags[] = Dictionary.decodeFlags(scratch);
- stems.addAll(stem(strippedWord, length, appendFlags, ++recursionDepth));
+ circumfix = Dictionary.hasFlag(appendFlags, (char)dictionary.circumfix);
+ }
+
+ if (crossProduct) {
+ if (recursionDepth == 0) {
+ if (prefix) {
+ // we took away the first prefix.
+ // COMPLEXPREFIXES = true: combine with a second prefix and another suffix
+ // COMPLEXPREFIXES = false: combine with a suffix
+ stems.addAll(stem(strippedWord, length, affix, flag, flag, ++recursionDepth, dictionary.complexPrefixes && dictionary.twoStageAffix, true, true, circumfix));
+ } else if (dictionary.complexPrefixes == false && dictionary.twoStageAffix) {
+ // we took away a suffix.
+ // COMPLEXPREFIXES = true: we don't recurse! only one suffix allowed
+ // COMPLEXPREFIXES = false: combine with another suffix
+ stems.addAll(stem(strippedWord, length, affix, flag, prefixFlag, ++recursionDepth, false, true, false, circumfix));
+ }
+ } else if (recursionDepth == 1) {
+ if (prefix && dictionary.complexPrefixes) {
+ // we took away the second prefix: go look for another suffix
+ stems.addAll(stem(strippedWord, length, affix, flag, flag, ++recursionDepth, false, true, true, circumfix));
+ } else if (prefix == false && dictionary.complexPrefixes == false && dictionary.twoStageAffix) {
+ // we took away a prefix, then a suffix: go look for another suffix
+ stems.addAll(stem(strippedWord, length, affix, flag, prefixFlag, ++recursionDepth, false, true, false, circumfix));
+ }
+ }
}
return stems;
@@ -232,7 +399,7 @@ final class Stemmer {
* @param flags Array of flags to cross check against. Can be {@code null}
* @return {@code true} if the flag is found in the array or the array is {@code null}, {@code false} otherwise
*/
- private boolean hasCrossCheckedFlag(char flag, char[] flags) {
- return flags.length == 0 || Arrays.binarySearch(flags, flag) >= 0;
+ private boolean hasCrossCheckedFlag(char flag, char[] flags, boolean matchEmpty) {
+ return (flags.length == 0 && matchEmpty) || Arrays.binarySearch(flags, flag) >= 0;
}
}
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizer.java Sun Mar 16 19:39:10 2014
@@ -43,7 +43,7 @@ public class IndicNormalizer {
}
private static final IdentityHashMap<Character.UnicodeBlock,ScriptData> scripts =
- new IdentityHashMap<Character.UnicodeBlock,ScriptData>(9);
+ new IdentityHashMap<>(9);
private static int flag(Character.UnicodeBlock ub) {
return scripts.get(ub).flag;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/CapitalizationFilterFactory.java Sun Mar 16 19:39:10 2014
@@ -88,7 +88,7 @@ public class CapitalizationFilterFactory
k = getSet(args, OK_PREFIX);
if (k != null) {
- okPrefix = new ArrayList<char[]>();
+ okPrefix = new ArrayList<>();
for (String item : k) {
okPrefix.add(item.toCharArray());
}
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java Sun Mar 16 19:39:10 2014
@@ -33,7 +33,7 @@ import java.util.Map;
*
* <pre class="prettyprint">
* {@code
- * Map<String,Analyzer> analyzerPerField = new HashMap<String,Analyzer>();
+ * Map<String,Analyzer> analyzerPerField = new HashMap<>();
* analyzerPerField.put("firstname", new KeywordAnalyzer());
* analyzerPerField.put("lastname", new KeywordAnalyzer());
*
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java Sun Mar 16 19:39:10 2014
@@ -44,7 +44,7 @@ public final class StemmerOverrideFilter
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final BytesReader fstReader;
- private final Arc<BytesRef> scratchArc = new FST.Arc<BytesRef>();
+ private final Arc<BytesRef> scratchArc = new FST.Arc<>();
private final CharsRef spare = new CharsRef();
/**
@@ -145,7 +145,7 @@ public final class StemmerOverrideFilter
public static class Builder {
private final BytesRefHash hash = new BytesRefHash();
private final BytesRef spare = new BytesRef();
- private final ArrayList<CharSequence> outputValues = new ArrayList<CharSequence>();
+ private final ArrayList<CharSequence> outputValues = new ArrayList<>();
private final boolean ignoreCase;
private final CharsRef charsSpare = new CharsRef();
@@ -200,7 +200,7 @@ public final class StemmerOverrideFilter
*/
public StemmerOverrideMap build() throws IOException {
ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- org.apache.lucene.util.fst.Builder<BytesRef> builder = new org.apache.lucene.util.fst.Builder<BytesRef>(
+ org.apache.lucene.util.fst.Builder<BytesRef> builder = new org.apache.lucene.util.fst.Builder<>(
FST.INPUT_TYPE.BYTE4, outputs);
final int[] sort = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
IntsRef intsSpare = new IntsRef();
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/WordDelimiterFilterFactory.java Sun Mar 16 19:39:10 2014
@@ -104,7 +104,7 @@ public class WordDelimiterFilterFactory
}
if (types != null) {
List<String> files = splitFileNames( types );
- List<String> wlist = new ArrayList<String>();
+ List<String> wlist = new ArrayList<>();
for( String file : files ){
List<String> lines = getLines(loader, file.trim());
wlist.addAll( lines );
@@ -124,7 +124,7 @@ public class WordDelimiterFilterFactory
// parses a list of MappingCharFilter style rules into a custom byte[] type table
private byte[] parseTypes(List<String> rules) {
- SortedMap<Character,Byte> typeMap = new TreeMap<Character,Byte>();
+ SortedMap<Character,Byte> typeMap = new TreeMap<>();
for( String rule : rules ){
Matcher m = typePattern.matcher(rule);
if( !m.find() )
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Sun Mar 16 19:39:10 2014
@@ -82,7 +82,7 @@ public final class DutchAnalyzer extends
throw new RuntimeException("Unable to load default stopword set");
}
- DEFAULT_STEM_DICT = new CharArrayMap<String>(Version.LUCENE_CURRENT, 4, false);
+ DEFAULT_STEM_DICT = new CharArrayMap<>(Version.LUCENE_CURRENT, 4, false);
DEFAULT_STEM_DICT.put("fiets", "fiets"); //otherwise fiet
DEFAULT_STEM_DICT.put("bromfiets", "bromfiets"); //otherwise bromfiet
DEFAULT_STEM_DICT.put("ei", "eier");
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java Sun Mar 16 19:39:10 2014
@@ -98,7 +98,7 @@ public class ReversePathHierarchyTokeniz
this.skip = skip;
resultToken = new StringBuilder(bufferSize);
resultTokenBuffer = new char[bufferSize];
- delimiterPositions = new ArrayList<Integer>(bufferSize/10);
+ delimiterPositions = new ArrayList<>(bufferSize/10);
}
private static final int DEFAULT_BUFFER_SIZE = 1024;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java Sun Mar 16 19:39:10 2014
@@ -248,7 +248,7 @@ public abstract class RSLPStemmerBase {
try {
InputStream is = clazz.getResourceAsStream(resource);
LineNumberReader r = new LineNumberReader(new InputStreamReader(is, "UTF-8"));
- Map<String,Step> steps = new HashMap<String,Step>();
+ Map<String,Step> steps = new HashMap<>();
String step;
while ((step = readLine(r)) != null) {
Step s = parseStep(r, step);
@@ -285,7 +285,7 @@ public abstract class RSLPStemmerBase {
}
private static Rule[] parseRules(LineNumberReader r, int type) throws IOException {
- List<Rule> rules = new ArrayList<Rule>();
+ List<Rule> rules = new ArrayList<>();
String line;
while ((line = readLine(r)) != null) {
Matcher matcher = stripPattern.matcher(line);
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Sun Mar 16 19:39:10 2014
@@ -46,7 +46,7 @@ import org.apache.lucene.util.Version;
public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
private final Analyzer delegate;
- private final Map<String, Set<String>> stopWordsPerField = new HashMap<String, Set<String>>();
+ private final Map<String, Set<String>> stopWordsPerField = new HashMap<>();
//The default maximum percentage (40%) of index documents which
//can contain a term, after which the term is considered to be a stop word.
public static final float defaultMaxDocFreqPercent = 0.4f;
@@ -153,7 +153,7 @@ public final class QueryAutoStopWordAnal
this.delegate = delegate;
for (String field : fields) {
- Set<String> stopWords = new HashSet<String>();
+ Set<String> stopWords = new HashSet<>();
Terms terms = MultiFields.getTerms(indexReader, field);
CharsRef spare = new CharsRef();
if (terms != null) {
@@ -204,7 +204,7 @@ public final class QueryAutoStopWordAnal
* @return the stop words (as terms)
*/
public Term[] getStopWords() {
- List<Term> allStopWords = new ArrayList<Term>();
+ List<Term> allStopWords = new ArrayList<>();
for (String fieldName : stopWordsPerField.keySet()) {
Set<String> stopWords = stopWordsPerField.get(fieldName);
for (String text : stopWords) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Sun Mar 16 19:39:10 2014
@@ -74,7 +74,7 @@ public final class ShingleFilter extends
* that will be composed to form output shingles.
*/
private LinkedList<InputWindowToken> inputWindow
- = new LinkedList<InputWindowToken>();
+ = new LinkedList<>();
/**
* The number of input tokens in the next output token. This is the "n" in
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/sinks/TeeSinkTokenFilter.java Sun Mar 16 19:39:10 2014
@@ -75,7 +75,7 @@ sink2.consumeAllTokens();
* <p>Note, the EntityDetect and URLDetect TokenStreams are for the example and do not currently exist in Lucene.
*/
public final class TeeSinkTokenFilter extends TokenFilter {
- private final List<WeakReference<SinkTokenStream>> sinks = new LinkedList<WeakReference<SinkTokenStream>>();
+ private final List<WeakReference<SinkTokenStream>> sinks = new LinkedList<>();
/**
* Instantiates a new TeeSinkTokenFilter.
@@ -98,7 +98,7 @@ public final class TeeSinkTokenFilter ex
*/
public SinkTokenStream newSinkTokenStream(SinkFilter filter) {
SinkTokenStream sink = new SinkTokenStream(this.cloneAttributes(), filter);
- this.sinks.add(new WeakReference<SinkTokenStream>(sink));
+ this.sinks.add(new WeakReference<>(sink));
return sink;
}
@@ -116,7 +116,7 @@ public final class TeeSinkTokenFilter ex
for (Iterator<AttributeImpl> it = this.cloneAttributes().getAttributeImplsIterator(); it.hasNext(); ) {
sink.addAttributeImpl(it.next());
}
- this.sinks.add(new WeakReference<SinkTokenStream>(sink));
+ this.sinks.add(new WeakReference<>(sink));
}
/**
@@ -186,7 +186,7 @@ public final class TeeSinkTokenFilter ex
* TokenStream output from a tee with optional filtering.
*/
public static final class SinkTokenStream extends TokenStream {
- private final List<AttributeSource.State> cachedStates = new LinkedList<AttributeSource.State>();
+ private final List<AttributeSource.State> cachedStates = new LinkedList<>();
private AttributeSource.State finalState;
private Iterator<AttributeSource.State> it = null;
private SinkFilter filter;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SolrSynonymParser.java Sun Mar 16 19:39:10 2014
@@ -130,7 +130,7 @@ public class SolrSynonymParser extends S
}
private static String[] split(String s, String separator) {
- ArrayList<String> list = new ArrayList<String>(2);
+ ArrayList<String> list = new ArrayList<>(2);
StringBuilder sb = new StringBuilder();
int pos=0, end=s.length();
while (pos < end) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Sun Mar 16 19:39:10 2014
@@ -282,7 +282,7 @@ public final class SynonymFilter extends
//System.out.println("FSTFilt maxH=" + synonyms.maxHorizontalContext);
- scratchArc = new FST.Arc<BytesRef>();
+ scratchArc = new FST.Arc<>();
}
private void capture() {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java Sun Mar 16 19:39:10 2014
@@ -83,7 +83,7 @@ public class SynonymFilterFactory extend
private final String format;
private final boolean expand;
private final String analyzerName;
- private final Map<String, String> tokArgs = new HashMap<String, String>();
+ private final Map<String, String> tokArgs = new HashMap<>();
private SynonymMap map;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java Sun Mar 16 19:39:10 2014
@@ -67,7 +67,7 @@ public class SynonymMap {
* @lucene.experimental
*/
public static class Builder {
- private final HashMap<CharsRef,MapEntry> workingSet = new HashMap<CharsRef,MapEntry>();
+ private final HashMap<CharsRef,MapEntry> workingSet = new HashMap<>();
private final BytesRefHash words = new BytesRefHash();
private final BytesRef utf8Scratch = new BytesRef(8);
private int maxHorizontalContext;
@@ -82,7 +82,7 @@ public class SynonymMap {
private static class MapEntry {
boolean includeOrig;
// we could sort for better sharing ultimately, but it could confuse people
- ArrayList<Integer> ords = new ArrayList<Integer>();
+ ArrayList<Integer> ords = new ArrayList<>();
}
/** Sugar: just joins the provided terms with {@link
@@ -210,7 +210,7 @@ public class SynonymMap {
ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
// TODO: are we using the best sharing options?
org.apache.lucene.util.fst.Builder<BytesRef> builder =
- new org.apache.lucene.util.fst.Builder<BytesRef>(FST.INPUT_TYPE.BYTE4, outputs);
+ new org.apache.lucene.util.fst.Builder<>(FST.INPUT_TYPE.BYTE4, outputs);
BytesRef scratch = new BytesRef(64);
ByteArrayDataOutput scratchOutput = new ByteArrayDataOutput();
@@ -218,7 +218,7 @@ public class SynonymMap {
final Set<Integer> dedupSet;
if (dedup) {
- dedupSet = new HashSet<Integer>();
+ dedupSet = new HashSet<>();
} else {
dedupSet = null;
}
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java Sun Mar 16 19:39:10 2014
@@ -122,6 +122,8 @@ public final class TurkishAnalyzer exten
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
+ if(matchVersion.onOrAfter(Version.LUCENE_48))
+ result = new ApostropheFilter(result);
result = new TurkishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
if(!stemExclusionSet.isEmpty())
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java Sun Mar 16 19:39:10 2014
@@ -65,7 +65,7 @@ public abstract class AbstractAnalysisFa
* Initialize this factory via a set of key-value pairs.
*/
protected AbstractAnalysisFactory(Map<String,String> args) {
- originalArgs = Collections.unmodifiableMap(new HashMap<String,String>(args));
+ originalArgs = Collections.unmodifiableMap(new HashMap<>(args));
String version = get(args, LUCENE_MATCH_VERSION_PARAM);
luceneMatchVersion = version == null ? null : Version.parseLeniently(version);
args.remove(CLASS_NAME); // consume the class arg
@@ -202,7 +202,7 @@ public abstract class AbstractAnalysisFa
Set<String> set = null;
Matcher matcher = ITEM_PATTERN.matcher(s);
if (matcher.find()) {
- set = new HashSet<String>();
+ set = new HashSet<>();
set.add(matcher.group(0));
while (matcher.find()) {
set.add(matcher.group(0));
@@ -296,7 +296,7 @@ public abstract class AbstractAnalysisFa
if (fileNames == null)
return Collections.<String>emptyList();
- List<String> result = new ArrayList<String>();
+ List<String> result = new ArrayList<>();
for (String file : fileNames.split("(?<!\\\\),")) {
result.add(file.replaceAll("\\\\(?=,)", ""));
}
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java Sun Mar 16 19:39:10 2014
@@ -74,7 +74,7 @@ final class AnalysisSPILoader<S extends
*/
public synchronized void reload(ClassLoader classloader) {
final LinkedHashMap<String,Class<? extends S>> services =
- new LinkedHashMap<String,Class<? extends S>>(this.services);
+ new LinkedHashMap<>(this.services);
final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
while (loader.hasNext()) {
final Class<? extends S> service = loader.next();
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java Sun Mar 16 19:39:10 2014
@@ -52,7 +52,7 @@ import org.apache.lucene.util.Version;
*/
public class CharArrayMap<V> extends AbstractMap<Object,V> {
// private only because missing generics
- private static final CharArrayMap<?> EMPTY_MAP = new EmptyCharArrayMap<Object>();
+ private static final CharArrayMap<?> EMPTY_MAP = new EmptyCharArrayMap<>();
private final static int INIT_SIZE = 8;
private final CharacterUtils charUtils;
@@ -559,7 +559,7 @@ public class CharArrayMap<V> extends Abs
return emptyMap();
if (map instanceof UnmodifiableCharArrayMap)
return map;
- return new UnmodifiableCharArrayMap<V>(map);
+ return new UnmodifiableCharArrayMap<>(map);
}
/**
@@ -595,12 +595,12 @@ public class CharArrayMap<V> extends Abs
System.arraycopy(m.keys, 0, keys, 0, keys.length);
final V[] values = (V[]) new Object[m.values.length];
System.arraycopy(m.values, 0, values, 0, values.length);
- m = new CharArrayMap<V>(m);
+ m = new CharArrayMap<>(m);
m.keys = keys;
m.values = values;
return m;
}
- return new CharArrayMap<V>(matchVersion, map, false);
+ return new CharArrayMap<>(matchVersion, map, false);
}
/** Returns an empty, unmodifiable map. */
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArraySet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArraySet.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArraySet.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArraySet.java Sun Mar 16 19:39:10 2014
@@ -74,7 +74,7 @@ public class CharArraySet extends Abstra
* otherwise <code>true</code>.
*/
public CharArraySet(Version matchVersion, int startSize, boolean ignoreCase) {
- this(new CharArrayMap<Object>(matchVersion, startSize, ignoreCase));
+ this(new CharArrayMap<>(matchVersion, startSize, ignoreCase));
}
/**
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java Sun Mar 16 19:39:10 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.analysis.CharFi
public abstract class CharFilterFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<CharFilterFactory> loader =
- new AnalysisSPILoader<CharFilterFactory>(CharFilterFactory.class);
+ new AnalysisSPILoader<>(CharFilterFactory.class);
/** looks up a charfilter by name from context classpath */
public static CharFilterFactory forName(String name, Map<String,String> args) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java Sun Mar 16 19:39:10 2014
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.TokenS
public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<TokenFilterFactory> loader =
- new AnalysisSPILoader<TokenFilterFactory>(TokenFilterFactory.class,
+ new AnalysisSPILoader<>(TokenFilterFactory.class,
new String[] { "TokenFilterFactory", "FilterFactory" });
/** looks up a tokenfilter by name from context classpath */
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java Sun Mar 16 19:39:10 2014
@@ -31,7 +31,7 @@ import java.util.Set;
public abstract class TokenizerFactory extends AbstractAnalysisFactory {
private static final AnalysisSPILoader<TokenizerFactory> loader =
- new AnalysisSPILoader<TokenizerFactory>(TokenizerFactory.class);
+ new AnalysisSPILoader<>(TokenizerFactory.class);
/** looks up a tokenizer by name from context classpath */
public static TokenizerFactory forName(String name, Map<String,String> args) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/WordlistLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/WordlistLoader.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/WordlistLoader.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/WordlistLoader.java Sun Mar 16 19:39:10 2014
@@ -219,7 +219,7 @@ public class WordlistLoader {
try {
input = getBufferedReader(IOUtils.getDecodingReader(stream, charset));
- lines = new ArrayList<String>();
+ lines = new ArrayList<>();
for (String word=null; (word=input.readLine())!=null;) {
// skip initial bom marker
if (lines.isEmpty() && word.length() > 0 && word.charAt(0) == '\uFEFF')
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java Sun Mar 16 19:39:10 2014
@@ -215,7 +215,7 @@ public final class WikipediaTokenizer ex
int lastPos = theStart + numAdded;
int tmpTokType;
int numSeen = 0;
- List<AttributeSource.State> tmp = new ArrayList<AttributeSource.State>();
+ List<AttributeSource.State> tmp = new ArrayList<>();
setupSavedToken(0, type);
tmp.add(captureState());
//while we can get a token and that token is the same type and we have not transitioned to a new wiki-item of the same type
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory Sun Mar 16 19:39:10 2014
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+org.apache.lucene.analysis.tr.ApostropheFilterFactory
org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory
org.apache.lucene.analysis.ar.ArabicStemFilterFactory
org.apache.lucene.analysis.bg.BulgarianStemFilterFactory
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java Sun Mar 16 19:39:10 2014
@@ -114,7 +114,7 @@ public class HTMLStripCharFilterTest ext
public void testGamma() throws Exception {
String test = "Γ";
String gold = "\u0393";
- Set<String> set = new HashSet<String>();
+ Set<String> set = new HashSet<>();
set.add("reserved");
Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
@@ -129,7 +129,7 @@ public class HTMLStripCharFilterTest ext
public void testEntities() throws Exception {
String test = " <foo> Übermensch = Γ bar Γ";
String gold = " <foo> \u00DCbermensch = \u0393 bar \u0393";
- Set<String> set = new HashSet<String>();
+ Set<String> set = new HashSet<>();
set.add("reserved");
Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
@@ -144,7 +144,7 @@ public class HTMLStripCharFilterTest ext
public void testMoreEntities() throws Exception {
String test = " <junk/> ! @ and ’";
String gold = " <junk/> ! @ and â";
- Set<String> set = new HashSet<String>();
+ Set<String> set = new HashSet<>();
set.add("reserved");
Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
@@ -158,7 +158,7 @@ public class HTMLStripCharFilterTest ext
public void testReserved() throws Exception {
String test = "aaa bbb <reserved ccc=\"ddddd\"> eeee </reserved> ffff <reserved ggg=\"hhhh\"/> <other/>";
- Set<String> set = new HashSet<String>();
+ Set<String> set = new HashSet<>();
set.add("reserved");
Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
@@ -588,7 +588,7 @@ public class HTMLStripCharFilterTest ext
public void testEscapeScript() throws Exception {
String test = "one<script no-value-attr>callSomeMethod();</script>two";
String gold = "one<script no-value-attr></script>two";
- Set<String> escapedTags = new HashSet<String>(Arrays.asList("SCRIPT"));
+ Set<String> escapedTags = new HashSet<>(Arrays.asList("SCRIPT"));
Reader reader = new HTMLStripCharFilter
(new StringReader(test), escapedTags);
int ch = 0;
@@ -628,7 +628,7 @@ public class HTMLStripCharFilterTest ext
public void testEscapeStyle() throws Exception {
String test = "one<style type=\"text/css\"> body,font,a { font-family:arial; } </style>two";
String gold = "one<style type=\"text/css\"></style>two";
- Set<String> escapedTags = new HashSet<String>(Arrays.asList("STYLE"));
+ Set<String> escapedTags = new HashSet<>(Arrays.asList("STYLE"));
Reader reader = new HTMLStripCharFilter
(new StringReader(test), escapedTags);
int ch = 0;
@@ -668,7 +668,7 @@ public class HTMLStripCharFilterTest ext
public void testEscapeBR() throws Exception {
String test = "one<BR class='whatever'>two</\nBR\n>";
String gold = "one<BR class='whatever'>two</\nBR\n>";
- Set<String> escapedTags = new HashSet<String>(Arrays.asList("BR"));
+ Set<String> escapedTags = new HashSet<>(Arrays.asList("BR"));
Reader reader = new HTMLStripCharFilter
(new StringReader(test), escapedTags);
int ch = 0;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java Sun Mar 16 19:39:10 2014
@@ -270,7 +270,7 @@ public class TestMappingCharFilter exten
Random random = random();
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
// we can't add duplicate keys, or NormalizeCharMap gets angry
- Set<String> keys = new HashSet<String>();
+ Set<String> keys = new HashSet<>();
int num = random.nextInt(5);
//System.out.println("NormalizeCharMap=");
for (int i = 0; i < num; i++) {
@@ -296,7 +296,7 @@ public class TestMappingCharFilter exten
final char endLetter = (char) TestUtil.nextInt(random, 'b', 'z');
- final Map<String,String> map = new HashMap<String,String>();
+ final Map<String,String> map = new HashMap<>();
final NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
final int numMappings = atLeast(5);
if (VERBOSE) {
@@ -333,7 +333,7 @@ public class TestMappingCharFilter exten
final StringBuilder output = new StringBuilder();
// Maps output offset to input offset:
- final List<Integer> inputOffsets = new ArrayList<Integer>();
+ final List<Integer> inputOffsets = new ArrayList<>();
int cumDiff = 0;
int charIdx = 0;
@@ -416,7 +416,7 @@ public class TestMappingCharFilter exten
final MappingCharFilter mapFilter = new MappingCharFilter(charMap, new StringReader(content));
final StringBuilder actualBuilder = new StringBuilder();
- final List<Integer> actualInputOffsets = new ArrayList<Integer>();
+ final List<Integer> actualInputOffsets = new ArrayList<>();
// Now consume the actual mapFilter, somewhat randomly:
while (true) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java Sun Mar 16 19:39:10 2014
@@ -118,7 +118,7 @@ public class TestAllAnalyzersHaveFactori
continue;
}
- Map<String,String> args = new HashMap<String,String>();
+ Map<String,String> args = new HashMap<>();
args.put("luceneMatchVersion", TEST_VERSION_CURRENT.toString());
if (Tokenizer.class.isAssignableFrom(c)) {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java Sun Mar 16 19:39:10 2014
@@ -122,7 +122,7 @@ public class TestFactories extends BaseT
/** tries to initialize a factory with no arguments */
private AbstractAnalysisFactory initialize(Class<? extends AbstractAnalysisFactory> factoryClazz) throws IOException {
- Map<String,String> args = new HashMap<String,String>();
+ Map<String,String> args = new HashMap<>();
args.put("luceneMatchVersion", TEST_VERSION_CURRENT.toString());
Constructor<? extends AbstractAnalysisFactory> ctor;
try {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Sun Mar 16 19:39:10 2014
@@ -110,7 +110,7 @@ public class TestRandomChains extends Ba
};
};
- private static final Map<Constructor<?>,Predicate<Object[]>> brokenConstructors = new HashMap<Constructor<?>, Predicate<Object[]>>();
+ private static final Map<Constructor<?>,Predicate<Object[]>> brokenConstructors = new HashMap<>();
static {
try {
brokenConstructors.put(
@@ -158,7 +158,7 @@ public class TestRandomChains extends Ba
// TODO: also fix these and remove (maybe):
// Classes/options that don't produce consistent graph offsets:
- private static final Map<Constructor<?>,Predicate<Object[]>> brokenOffsetsConstructors = new HashMap<Constructor<?>, Predicate<Object[]>>();
+ private static final Map<Constructor<?>,Predicate<Object[]>> brokenOffsetsConstructors = new HashMap<>();
static {
try {
for (Class<?> c : Arrays.<Class<?>>asList(
@@ -188,9 +188,9 @@ public class TestRandomChains extends Ba
@BeforeClass
public static void beforeClass() throws Exception {
List<Class<?>> analysisClasses = getClassesForPackage("org.apache.lucene.analysis");
- tokenizers = new ArrayList<Constructor<? extends Tokenizer>>();
- tokenfilters = new ArrayList<Constructor<? extends TokenFilter>>();
- charfilters = new ArrayList<Constructor<? extends CharFilter>>();
+ tokenizers = new ArrayList<>();
+ tokenfilters = new ArrayList<>();
+ charfilters = new ArrayList<>();
for (final Class<?> c : analysisClasses) {
final int modifiers = c.getModifiers();
if (
@@ -257,7 +257,7 @@ public class TestRandomChains extends Ba
}
public static List<Class<?>> getClassesForPackage(String pckgname) throws Exception {
- final List<Class<?>> classes = new ArrayList<Class<?>>();
+ final List<Class<?>> classes = new ArrayList<>();
collectClassesForPackage(pckgname, classes);
assertFalse("No classes found in package '"+pckgname+"'; maybe your test classes are packaged as JAR file?", classes.isEmpty());
return classes;
@@ -358,7 +358,7 @@ public class TestRandomChains extends Ba
put(Set.class, new ArgProducer() {
@Override public Object create(Random random) {
// TypeTokenFilter
- Set<String> set = new HashSet<String>();
+ Set<String> set = new HashSet<>();
int num = random.nextInt(5);
for (int i = 0; i < num; i++) {
set.add(StandardTokenizer.TOKEN_TYPES[random.nextInt(StandardTokenizer.TOKEN_TYPES.length)]);
@@ -369,7 +369,7 @@ public class TestRandomChains extends Ba
put(Collection.class, new ArgProducer() {
@Override public Object create(Random random) {
// CapitalizationFilter
- Collection<char[]> col = new ArrayList<char[]>();
+ Collection<char[]> col = new ArrayList<>();
int num = random.nextInt(5);
for (int i = 0; i < num; i++) {
col.add(TestUtil.randomSimpleString(random).toCharArray());
@@ -459,7 +459,7 @@ public class TestRandomChains extends Ba
@Override public Object create(Random random) {
NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
// we can't add duplicate keys, or NormalizeCharMap gets angry
- Set<String> keys = new HashSet<String>();
+ Set<String> keys = new HashSet<>();
int num = random.nextInt(5);
//System.out.println("NormalizeCharMap=");
for (int i = 0; i < num; i++) {
@@ -489,7 +489,7 @@ public class TestRandomChains extends Ba
put(CharArrayMap.class, new ArgProducer() {
@Override public Object create(Random random) {
int num = random.nextInt(10);
- CharArrayMap<String> map = new CharArrayMap<String>(TEST_VERSION_CURRENT, num, random.nextBoolean());
+ CharArrayMap<String> map = new CharArrayMap<>(TEST_VERSION_CURRENT, num, random.nextBoolean());
for (int i = 0; i < num; i++) {
// TODO: make nastier
map.put(TestUtil.randomSimpleString(random), TestUtil.randomSimpleString(random));
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java Sun Mar 16 19:39:10 2014
@@ -32,7 +32,7 @@ import java.util.HashSet;
public class TestStopAnalyzer extends BaseTokenStreamTestCase {
private StopAnalyzer stop = new StopAnalyzer(TEST_VERSION_CURRENT);
- private Set<Object> inValidTokens = new HashSet<Object>();
+ private Set<Object> inValidTokens = new HashSet<>();
@Override
public void setUp() throws Exception {
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java Sun Mar 16 19:39:10 2014
@@ -59,7 +59,7 @@ public class TestStopFilter extends Base
*/
public void testStopPositons() throws IOException {
StringBuilder sb = new StringBuilder();
- ArrayList<String> a = new ArrayList<String>();
+ ArrayList<String> a = new ArrayList<>();
for (int i=0; i<20; i++) {
String w = English.intToEnglish(i).trim();
sb.append(w).append(" ");
@@ -76,8 +76,8 @@ public class TestStopFilter extends Base
StopFilter stpf = new StopFilter(Version.LUCENE_40, in, stopSet);
doTestStopPositons(stpf);
// with increments, concatenating two stop filters
- ArrayList<String> a0 = new ArrayList<String>();
- ArrayList<String> a1 = new ArrayList<String>();
+ ArrayList<String> a0 = new ArrayList<>();
+ ArrayList<String> a1 = new ArrayList<>();
for (int i=0; i<a.size(); i++) {
if (i%2==0) {
a0.add(a.get(i));
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java Sun Mar 16 19:39:10 2014
@@ -287,7 +287,7 @@ public class TestUAX29URLEmailTokenizer
BufferedReader bufferedReader = null;
String[] urls;
try {
- List<String> urlList = new ArrayList<String>();
+ List<String> urlList = new ArrayList<>();
bufferedReader = new BufferedReader(new InputStreamReader
(getClass().getResourceAsStream("LuceneResourcesWikiPageURLs.txt"), "UTF-8"));
String line;
@@ -331,7 +331,7 @@ public class TestUAX29URLEmailTokenizer
BufferedReader bufferedReader = null;
String[] emails;
try {
- List<String> emailList = new ArrayList<String>();
+ List<String> emailList = new ArrayList<>();
bufferedReader = new BufferedReader(new InputStreamReader
(getClass().getResourceAsStream
("email.addresses.from.random.text.with.email.addresses.txt"), "UTF-8"));
@@ -401,7 +401,7 @@ public class TestUAX29URLEmailTokenizer
BufferedReader bufferedReader = null;
String[] urls;
try {
- List<String> urlList = new ArrayList<String>();
+ List<String> urlList = new ArrayList<>();
bufferedReader = new BufferedReader(new InputStreamReader
(getClass().getResourceAsStream
("urls.from.random.text.with.urls.txt"), "UTF-8"));
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java Sun Mar 16 19:39:10 2014
@@ -73,7 +73,7 @@ public class TestKStemmer extends BaseTo
// tf = new KStemFilter(tf);
KStemmer kstem = new KStemmer();
- Map<String,String> map = new TreeMap<String,String>();
+ Map<String,String> map = new TreeMap<>();
for(;;) {
Token t = tf.next();
if (t==null) break;
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java Sun Mar 16 19:39:10 2014
@@ -55,7 +55,7 @@ abstract class StemmerTestBase extends L
}
try {
- Dictionary dictionary = new Dictionary(affixStream, Arrays.asList(dictStreams), true);
+ Dictionary dictionary = new Dictionary(affixStream, Arrays.asList(dictStreams), ignoreCase);
stemmer = new Stemmer(dictionary);
} finally {
IOUtils.closeWhileHandlingException(affixStream);
@@ -74,6 +74,6 @@ abstract class StemmerTestBase extends L
}
Arrays.sort(actual);
- assertArrayEquals(expected, actual);
+ assertArrayEquals("expected=" + Arrays.toString(expected) + ",actual=" + Arrays.toString(actual), expected, actual);
}
}
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java Sun Mar 16 19:39:10 2014
@@ -47,7 +47,7 @@ public class TestAllDictionaries extends
"bg_BG.zip", "bg_BG.dic", "bg_BG.aff",
"ca_ANY.zip", "catalan.dic", "catalan.aff",
"ca_ES.zip", "ca_ES.dic", "ca_ES.aff",
- "cop_EG.zip", "cop_EG.dic", "cop_EG.aff",
+// BUG: broken flag "cop_EG.zip", "cop_EG.dic", "cop_EG.aff",
"cs_CZ.zip", "cs_CZ.dic", "cs_CZ.aff",
"cy_GB.zip", "cy_GB.dic", "cy_GB.aff",
"da_DK.zip", "da_DK.dic", "da_DK.aff",
@@ -169,7 +169,7 @@ public class TestAllDictionaries extends
System.out.println(tests[i] + "\t" + RamUsageEstimator.humanSizeOf(dic) + "\t(" +
"words=" + RamUsageEstimator.humanSizeOf(dic.words) + ", " +
"flags=" + RamUsageEstimator.humanSizeOf(dic.flagLookup) + ", " +
- "strips=" + RamUsageEstimator.humanSizeOf(dic.stripLookup) + ", " +
+ "strips=" + RamUsageEstimator.humanSizeOf(dic.stripData) + ", " +
"conditions=" + RamUsageEstimator.humanSizeOf(dic.patterns) + ", " +
"affixData=" + RamUsageEstimator.humanSizeOf(dic.affixData) + ", " +
"prefixes=" + RamUsageEstimator.humanSizeOf(dic.prefixes) + ", " +
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestCaseInsensitive.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestCaseInsensitive.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestCaseInsensitive.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestCaseInsensitive.java Sun Mar 16 19:39:10 2014
@@ -38,7 +38,8 @@ public class TestCaseInsensitive extends
}
public void testRecursiveSuffix() {
- assertStemsTo("abcd", "ab");
+ // we should not recurse here! as the suffix has no continuation!
+ assertStemsTo("abcd");
}
// all forms unmunched from dictionary
@@ -47,7 +48,7 @@ public class TestCaseInsensitive extends
assertStemsTo("abc", "ab");
assertStemsTo("apach", "apach");
assertStemsTo("apache", "apach");
- assertStemsTo("foo", "foo");
+ assertStemsTo("foo", "foo", "foo");
assertStemsTo("food", "foo");
assertStemsTo("foos", "foo");
assertStemsTo("lucen", "lucen");
Modified: lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java?rev=1578144&r1=1578143&r2=1578144&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java (original)
+++ lucene/dev/branches/lucene5376_2/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java Sun Mar 16 19:39:10 2014
@@ -17,14 +17,22 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
+import java.io.ByteArrayInputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
-import org.apache.lucene.analysis.hunspell.Dictionary;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.fst.Builder;
+import org.apache.lucene.util.fst.CharSequenceOutputs;
+import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.Outputs;
+import org.apache.lucene.util.fst.Util;
public class TestDictionary extends LuceneTestCase {
@@ -35,11 +43,22 @@ public class TestDictionary extends Luce
Dictionary dictionary = new Dictionary(affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
- char flags[] = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3, new BytesRef());
- assertNotNull(flags);
+ IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
+ assertNotNull(ordList);
+ assertEquals(1, ordList.length);
+
+ BytesRef ref = new BytesRef();
+ dictionary.flagLookup.get(ordList.ints[0], ref);
+ char flags[] = Dictionary.decodeFlags(ref);
assertEquals(1, flags.length);
- assertEquals("Wrong number of flags for lucen", 1, dictionary.lookupWord(new char[]{'l', 'u', 'c', 'e', 'n'}, 0, 5, new BytesRef()).length);
-
+
+ ordList = dictionary.lookupWord(new char[]{'l', 'u', 'c', 'e', 'n'}, 0, 5);
+ assertNotNull(ordList);
+ assertEquals(1, ordList.length);
+ dictionary.flagLookup.get(ordList.ints[0], ref);
+ flags = Dictionary.decodeFlags(ref);
+ assertEquals(1, flags.length);
+
affixStream.close();
dictStream.close();
}
@@ -51,7 +70,45 @@ public class TestDictionary extends Luce
Dictionary dictionary = new Dictionary(affixStream, dictStream);
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
- assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3, new BytesRef()).length);
+ IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
+ BytesRef ref = new BytesRef();
+ dictionary.flagLookup.get(ordList.ints[0], ref);
+ char flags[] = Dictionary.decodeFlags(ref);
+ assertEquals(1, flags.length);
+
+ affixStream.close();
+ dictStream.close();
+ }
+
+ public void testCompressedBeforeSetDictionary() throws Exception {
+ InputStream affixStream = getClass().getResourceAsStream("compressed-before-set.aff");
+ InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+
+ Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
+ assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
+ IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
+ BytesRef ref = new BytesRef();
+ dictionary.flagLookup.get(ordList.ints[0], ref);
+ char flags[] = Dictionary.decodeFlags(ref);
+ assertEquals(1, flags.length);
+
+ affixStream.close();
+ dictStream.close();
+ }
+
+ public void testCompressedEmptyAliasDictionary() throws Exception {
+ InputStream affixStream = getClass().getResourceAsStream("compressed-empty-alias.aff");
+ InputStream dictStream = getClass().getResourceAsStream("compressed.dic");
+
+ Dictionary dictionary = new Dictionary(affixStream, dictStream);
+ assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).length);
+ assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).length);
+ IntsRef ordList = dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3);
+ BytesRef ref = new BytesRef();
+ dictionary.flagLookup.get(ordList.ints[0], ref);
+ char flags[] = Dictionary.decodeFlags(ref);
+ assertEquals(1, flags.length);
affixStream.close();
dictStream.close();
@@ -66,8 +123,24 @@ public class TestDictionary extends Luce
new Dictionary(affixStream, dictStream);
fail("didn't get expected exception");
} catch (ParseException expected) {
- assertEquals("The affix file contains a rule with less than five elements", expected.getMessage());
- assertEquals(23, expected.getErrorOffset());
+ assertTrue(expected.getMessage().startsWith("The affix file contains a rule with less than four elements"));
+ assertEquals(24, expected.getErrorOffset());
+ }
+
+ affixStream.close();
+ dictStream.close();
+ }
+
+ // malformed flags causes ParseException
+ public void testInvalidFlags() throws Exception {
+ InputStream affixStream = getClass().getResourceAsStream("broken-flags.aff");
+ InputStream dictStream = getClass().getResourceAsStream("simple.dic");
+
+ try {
+ new Dictionary(affixStream, dictStream);
+ fail("didn't get expected exception");
+ } catch (Exception expected) {
+ assertTrue(expected.getMessage().startsWith("expected only one flag"));
}
affixStream.close();
@@ -107,4 +180,66 @@ public class TestDictionary extends Luce
assertTrue(affixStream.isClosed());
assertTrue(dictStream.isClosed());
}
+
+
+
+ public void testReplacements() throws Exception {
+ Outputs<CharsRef> outputs = CharSequenceOutputs.getSingleton();
+ Builder<CharsRef> builder = new Builder<>(FST.INPUT_TYPE.BYTE2, outputs);
+ IntsRef scratchInts = new IntsRef();
+
+ // a -> b
+ Util.toUTF16("a", scratchInts);
+ builder.add(scratchInts, new CharsRef("b"));
+
+ // ab -> c
+ Util.toUTF16("ab", scratchInts);
+ builder.add(scratchInts, new CharsRef("c"));
+
+ // c -> de
+ Util.toUTF16("c", scratchInts);
+ builder.add(scratchInts, new CharsRef("de"));
+
+ // def -> gh
+ Util.toUTF16("def", scratchInts);
+ builder.add(scratchInts, new CharsRef("gh"));
+
+ FST<CharsRef> fst = builder.finish();
+
+ StringBuilder sb = new StringBuilder("atestanother");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("btestbnother", sb.toString());
+
+ sb = new StringBuilder("abtestanother");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("ctestbnother", sb.toString());
+
+ sb = new StringBuilder("atestabnother");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("btestcnother", sb.toString());
+
+ sb = new StringBuilder("abtestabnother");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("ctestcnother", sb.toString());
+
+ sb = new StringBuilder("abtestabcnother");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("ctestcdenother", sb.toString());
+
+ sb = new StringBuilder("defdefdefc");
+ Dictionary.applyMappings(fst, sb);
+ assertEquals("ghghghde", sb.toString());
+ }
+
+ public void testSetWithCrazyWhitespaceAndBOMs() throws Exception {
+ assertEquals("UTF-8", Dictionary.getDictionaryEncoding(new ByteArrayInputStream("SET\tUTF-8\n".getBytes(IOUtils.CHARSET_UTF_8))));
+ assertEquals("UTF-8", Dictionary.getDictionaryEncoding(new ByteArrayInputStream("SET\t UTF-8\n".getBytes(IOUtils.CHARSET_UTF_8))));
+ assertEquals("UTF-8", Dictionary.getDictionaryEncoding(new ByteArrayInputStream("\uFEFFSET\tUTF-8\n".getBytes(IOUtils.CHARSET_UTF_8))));
+ assertEquals("UTF-8", Dictionary.getDictionaryEncoding(new ByteArrayInputStream("\uFEFFSET\tUTF-8\r\n".getBytes(IOUtils.CHARSET_UTF_8))));
+ }
+
+ public void testFlagWithCrazyWhitespace() throws Exception {
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG\tUTF-8"));
+ assertNotNull(Dictionary.getFlagParsingStrategy("FLAG UTF-8"));
+ }
}