You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/03/27 10:17:13 UTC
svn commit: r1669533 - in /lucene/dev/trunk/lucene/analysis/common/src:
java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
Author: mikemccand
Date: Fri Mar 27 09:17:12 2015
New Revision: 1669533
URL: http://svn.apache.org/r1669533
Log:
fix test lowercase bug
Modified:
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java?rev=1669533&r1=1669532&r2=1669533&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java Fri Mar 27 09:17:12 2015
@@ -181,9 +181,9 @@ public final class StemmerOverrideFilter
charsSpare.grow(length);
final char[] buffer = charsSpare.chars();
for (int i = 0; i < length; ) {
- i += Character.toChars(
- Character.toLowerCase(
- Character.codePointAt(input, i)), buffer, i);
+ i += Character.toChars(
+ Character.toLowerCase(
+ Character.codePointAt(input, i)), buffer, i);
}
spare.copyChars(buffer, 0, length);
} else {
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java?rev=1669533&r1=1669532&r2=1669533&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java Fri Mar 27 09:17:12 2015
@@ -19,9 +19,10 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Map.Entry;
+import java.util.Map;
import java.util.Set;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -31,6 +32,7 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilter.StemmerOverrideMap;
+import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.TestUtil;
/**
@@ -79,7 +81,12 @@ public class TestStemmerOverrideFilter e
public void testRandomRealisticWhiteSpace() throws IOException {
Map<String,String> map = new HashMap<>();
+ Set<String> seen = new HashSet<>();
int numTerms = atLeast(50);
+ boolean ignoreCase = random().nextBoolean();
+
+ CharacterUtils charUtils = CharacterUtils.getInstance();
+
for (int i = 0; i < numTerms; i++) {
String randomRealisticUnicodeString = TestUtil
.randomRealisticUnicodeString(random());
@@ -93,16 +100,31 @@ public class TestStemmerOverrideFilter e
j += Character.charCount(cp);
}
if (builder.length() > 0) {
- String value = TestUtil.randomSimpleString(random());
- map.put(builder.toString(),
- value.isEmpty() ? "a" : value);
-
+ String inputValue = builder.toString();
+
+ // Make sure we don't try to add two inputs that vary only by case:
+ String seenInputValue;
+ if (ignoreCase) {
+ // TODO: can we simply use inputValue.toLowerCase(Locale.ROOT)???
+ char[] buffer = inputValue.toCharArray();
+ charUtils.toLowerCase(buffer, 0, buffer.length);
+ seenInputValue = buffer.toString();
+ } else {
+ seenInputValue = inputValue;
+ }
+
+ if (seen.contains(seenInputValue) == false) {
+ seen.add(seenInputValue);
+ String value = TestUtil.randomSimpleString(random());
+ map.put(inputValue,
+ value.isEmpty() ? "a" : value);
+ }
}
}
if (map.isEmpty()) {
map.put("booked", "books");
}
- StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(random().nextBoolean());
+ StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(ignoreCase);
Set<Entry<String,String>> entrySet = map.entrySet();
StringBuilder input = new StringBuilder();
List<String> output = new ArrayList<>();