You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [4/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-tool...
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Sun Aug 11 12:19:13 2013
@@ -19,6 +19,9 @@ package org.apache.lucene.analysis.wikip
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+/**
+ * JFlex-generated tokenizer that is aware of Wikipedia syntax.
+ */
%%
%class WikipediaTokenizerImpl
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenFilterFactory Sun Aug 11 12:19:13 2013
@@ -66,6 +66,8 @@ org.apache.lucene.analysis.miscellaneous
org.apache.lucene.analysis.miscellaneous.StemmerOverrideFilterFactory
org.apache.lucene.analysis.miscellaneous.TrimFilterFactory
org.apache.lucene.analysis.miscellaneous.WordDelimiterFilterFactory
+org.apache.lucene.analysis.miscellaneous.ScandinavianFoldingFilterFactory
+org.apache.lucene.analysis.miscellaneous.ScandinavianNormalizationFilterFactory
org.apache.lucene.analysis.ngram.EdgeNGramFilterFactory
org.apache.lucene.analysis.ngram.NGramFilterFactory
org.apache.lucene.analysis.no.NorwegianLightStemFilterFactory
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Sun Aug 11 12:19:13 2013
@@ -151,12 +151,12 @@ public class TestCompoundWordTokenFilter
"fiol", "fodral", "Basfiolsfodralmakaregesäll", "Bas", "fiol",
"fodral", "makare", "gesäll", "Skomakare", "Sko", "makare",
"Vindrutetorkare", "Vind", "rute", "torkare", "Vindrutetorkarblad",
- "Vind", "rute", "blad", "abba" }, new int[] { 0, 0, 3, 8, 8, 11, 17,
- 17, 20, 24, 24, 28, 33, 33, 39, 44, 44, 49, 54, 54, 58, 62, 69, 69, 72,
- 77, 84, 84, 87, 92, 98, 104, 111, 111, 114, 121, 121, 125, 129, 137,
- 137, 141, 151, 156 }, new int[] { 7, 3, 7, 16, 11, 16, 23, 20, 23, 32,
- 28, 32, 43, 39, 43, 53, 49, 53, 68, 58, 62, 68, 83, 72, 76, 83, 110,
- 87, 91, 98, 104, 110, 120, 114, 120, 136, 125, 129, 136, 155, 141, 145,
+ "Vind", "rute", "blad", "abba" }, new int[] { 0, 0, 0, 8, 8, 8, 17,
+ 17, 17, 24, 24, 24, 33, 33, 33, 44, 44, 44, 54, 54, 54, 54, 69, 69, 69,
+ 69, 84, 84, 84, 84, 84, 84, 111, 111, 111, 121, 121, 121, 121, 137,
+ 137, 137, 137, 156 }, new int[] { 7, 7, 7, 16, 16, 16, 23, 23, 23, 32,
+ 32, 32, 43, 43, 43, 53, 53, 53, 68, 68, 68, 68, 83, 83, 83, 83, 110,
+ 110, 110, 110, 110, 110, 120, 120, 120, 136, 136, 136, 136, 155, 155, 155,
155, 160 }, new int[] { 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1,
0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1,
0, 0, 0, 1 });
@@ -174,8 +174,8 @@ public class TestCompoundWordTokenFilter
CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, true);
assertTokenStreamContents(tf, new String[] { "Basfiolsfodralmakaregesäll", "Bas",
- "fiolsfodral", "fodral", "makare", "gesäll" }, new int[] { 0, 0, 3, 8,
- 14, 20 }, new int[] { 26, 3, 14, 14, 20, 26 }, new int[] { 1, 0, 0, 0,
+ "fiolsfodral", "fodral", "makare", "gesäll" }, new int[] { 0, 0, 0, 0,
+ 0, 0 }, new int[] { 26, 26, 26, 26, 26, 26 }, new int[] { 1, 0, 0, 0,
0, 0 });
}
@@ -194,8 +194,8 @@ public class TestCompoundWordTokenFilter
assertTokenStreamContents(tf,
new String[] { "abcdef", "ab", "cd", "ef" },
- new int[] { 0, 0, 2, 4},
- new int[] { 6, 2, 4, 6},
+ new int[] { 0, 0, 0, 0},
+ new int[] { 6, 6, 6, 6},
new int[] { 1, 0, 0, 0}
);
}
@@ -216,8 +216,8 @@ public class TestCompoundWordTokenFilter
// since "d" is shorter than the minimum subword size, it should not be added to the token stream
assertTokenStreamContents(tf,
new String[] { "abcdefg", "abc", "efg" },
- new int[] { 0, 0, 4},
- new int[] { 7, 3, 7},
+ new int[] { 0, 0, 0},
+ new int[] { 7, 7, 7},
new int[] { 1, 0, 0}
);
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Sun Aug 11 12:19:13 2013
@@ -170,8 +170,6 @@ public class TestAnalyzers extends BaseT
char[] termBuffer = filter.getAttribute(CharTermAttribute.class).buffer();
int length = highSurEndingLower.length();
assertEquals('\ud801', termBuffer[length - 1]);
- assertEquals('\udc3e', termBuffer[length]);
-
}
public void testLowerCaseTokenizer() throws IOException {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Sun Aug 11 12:19:13 2013
@@ -59,7 +59,6 @@ import org.apache.lucene.analysis.charfi
import org.apache.lucene.analysis.cjk.CJKBigramFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsFilter;
import org.apache.lucene.analysis.commongrams.CommonGramsQueryFilter;
-import org.apache.lucene.analysis.compound.DictionaryCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.HyphenationCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.TestCompoundWordTokenFilter;
import org.apache.lucene.analysis.compound.hyphenation.HyphenationTree;
@@ -165,8 +164,6 @@ public class TestRandomChains extends Ba
for (Class<?> c : Arrays.<Class<?>>asList(
ReversePathHierarchyTokenizer.class,
PathHierarchyTokenizer.class,
- HyphenationCompoundWordTokenFilter.class,
- DictionaryCompoundWordTokenFilter.class,
// TODO: it seems to mess up offsets!?
WikipediaTokenizer.class,
// TODO: doesn't handle graph inputs
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopAnalyzer.java Sun Aug 11 12:19:13 2013
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.util.Version;
-import java.io.StringReader;
import java.io.IOException;
import java.util.Iterator;
import java.util.Set;
@@ -47,8 +46,7 @@ public class TestStopAnalyzer extends Ba
public void testDefaults() throws IOException {
assertTrue(stop != null);
- StringReader reader = new StringReader("This is a test of the english stop analyzer");
- TokenStream stream = stop.tokenStream("test", reader);
+ TokenStream stream = stop.tokenStream("test", "This is a test of the english stop analyzer");
assertTrue(stream != null);
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
stream.reset();
@@ -61,8 +59,7 @@ public class TestStopAnalyzer extends Ba
public void testStopList() throws IOException {
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("good", "test", "analyzer"), false);
StopAnalyzer newStop = new StopAnalyzer(Version.LUCENE_40, stopWordsSet);
- StringReader reader = new StringReader("This is a good test of the english stop analyzer");
- TokenStream stream = newStop.tokenStream("test", reader);
+ TokenStream stream = newStop.tokenStream("test", "This is a good test of the english stop analyzer");
assertNotNull(stream);
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
@@ -76,9 +73,9 @@ public class TestStopAnalyzer extends Ba
public void testStopListPositions() throws IOException {
CharArraySet stopWordsSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("good", "test", "analyzer"), false);
StopAnalyzer newStop = new StopAnalyzer(TEST_VERSION_CURRENT, stopWordsSet);
- StringReader reader = new StringReader("This is a good test of the english stop analyzer with positions");
- int expectedIncr[] = { 1, 1, 1, 3, 1, 1, 1, 2, 1};
- TokenStream stream = newStop.tokenStream("test", reader);
+ String s = "This is a good test of the english stop analyzer with positions";
+ int expectedIncr[] = { 1, 1, 1, 3, 1, 1, 1, 2, 1};
+ TokenStream stream = newStop.tokenStream("test", s);
assertNotNull(stream);
int i = 0;
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java Sun Aug 11 12:19:13 2013
@@ -20,7 +20,9 @@ package org.apache.lucene.analysis.hunsp
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Assert;
@@ -131,12 +133,30 @@ public class HunspellDictionaryTest exte
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
-
+ assertEquals("Wrong number of flags for lucen", 1, dictionary.lookupWord(new char[]{'l', 'u', 'c', 'e', 'n'}, 0, 5).get(0).getFlags().length);
+
affixStream.close();
dictStream.close();
}
@Test
+ public void testHunspellDictionary_multipleDictWithOverride() throws IOException, ParseException {
+ InputStream affixStream = getClass().getResourceAsStream("test.aff");
+ List<InputStream> dictStreams = new ArrayList<InputStream>();
+ dictStreams.add(getClass().getResourceAsStream("test.dic"));
+ dictStreams.add(getClass().getResourceAsStream("testOverride.dic"));
+
+ HunspellDictionary dictionary = new HunspellDictionary(affixStream, dictStreams, TEST_VERSION_CURRENT, false);
+ assertEquals("Wrong number of flags for lucen", 3, dictionary.lookupWord(new char[]{'l', 'u', 'c', 'e', 'n'}, 0, 5).get(0).getFlags().length);
+ assertEquals("Wrong number of flags for bar", 1, dictionary.lookupWord(new char[]{'b', 'a', 'r'}, 0, 3).get(0).getFlags().length);
+
+ affixStream.close();
+ for(InputStream dstream : dictStreams) {
+ dstream.close();
+ }
+ }
+
+ @Test
public void testCompressedHunspellDictionary_loadDicAff() throws IOException, ParseException {
InputStream affixStream = getClass().getResourceAsStream("testCompressed.aff");
InputStream dictStream = getClass().getResourceAsStream("testCompressed.dic");
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Sun Aug 11 12:19:13 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -57,13 +58,13 @@ public class HunspellStemFilterTest ext
public void testKeywordAttribute() throws IOException {
MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
tokenizer.setEnableChecks(true);
- HunspellStemFilter filter = new HunspellStemFilter(tokenizer, DICTIONARY);
+ HunspellStemFilter filter = new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
// assert with keywork marker
tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
- filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY);
+ filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
}
@@ -74,7 +75,7 @@ public class HunspellStemFilterTest ext
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY));
+ return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
@@ -85,7 +86,7 @@ public class HunspellStemFilterTest ext
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KeywordTokenizer(reader);
- return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY));
+ return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
checkOneTermReuse(a, "", "");
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemmerTest.java Sun Aug 11 12:19:13 2013
@@ -94,17 +94,21 @@ public class HunspellStemmerTest extends
assertEquals(1, stems.size());
assertEquals("foo", stems.get(0).getStemString());
- stems = stemmer.stem("food");
+ stems = stemmer.stem("mood");
assertEquals(1, stems.size());
- assertEquals("foo", stems.get(0).getStemString());
+ assertEquals("moo", stems.get(0).getStemString());
stems = stemmer.stem("Foos");
assertEquals(1, stems.size());
assertEquals("foo", stems.get(0).getStemString());
-
+
+ // The "Foo" rule gets overridden by the "foo" rule, and we don't merge
stems = stemmer.stem("Food");
+ assertEquals(0, stems.size());
+
+ stems = stemmer.stem("Mood");
assertEquals(1, stems.size());
- assertEquals("foo", stems.get(0).getStemString());
+ assertEquals("moo", stems.get(0).getStemString());
}
@Test
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/test.dic
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/test.dic?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/test.dic (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/test.dic Sun Aug 11 12:19:13 2013
@@ -1,9 +1,10 @@
-6
+9
lucen/A
lucene
mahout/A
olr/B
ab/C
Apach/A
+Foo/E
foo/D
-Foo/E
\ No newline at end of file
+Moo/E
\ No newline at end of file
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Sun Aug 11 12:19:13 2013
@@ -18,13 +18,10 @@ package org.apache.lucene.analysis.misce
*/
import java.io.IOException;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -47,14 +44,14 @@ public class TestLimitTokenCountAnalyzer
Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
// dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
// less than the limit, ensure we behave correctly
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
// equal to limit
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 "), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
}
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java Sun Aug 11 12:19:13 2013
@@ -43,17 +43,17 @@ public class TestLimitTokenPositionFilte
};
// dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"),
new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")),
new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
// less than the limit, ensure we behave correctly
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 "),
new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
// equal to limit
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 "),
new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
}
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Sun Aug 11 12:19:13 2013
@@ -1,7 +1,6 @@
package org.apache.lucene.analysis.miscellaneous;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -38,8 +37,7 @@ public class TestPerFieldAnalyzerWrapper
PerFieldAnalyzerWrapper analyzer =
new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
- TokenStream tokenStream = analyzer.tokenStream("field",
- new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream("field", text);
CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
tokenStream.reset();
@@ -48,8 +46,7 @@ public class TestPerFieldAnalyzerWrapper
"Qwerty",
termAtt.toString());
- tokenStream = analyzer.tokenStream("special",
- new StringReader(text));
+ tokenStream = analyzer.tokenStream("special", text);
termAtt = tokenStream.getAttribute(CharTermAttribute.class);
tokenStream.reset();
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java Sun Aug 11 12:19:13 2013
@@ -78,7 +78,7 @@ public class TestStemmerOverrideFilter e
char[] charArray = randomRealisticUnicodeString.toCharArray();
StringBuilder builder = new StringBuilder();
for (int j = 0; j < charArray.length;) {
- int cp = Character.codePointAt(charArray, j);
+ int cp = Character.codePointAt(charArray, j, charArray.length);
if (!Character.isWhitespace(cp)) {
builder.appendCodePoint(cp);
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Sun Aug 11 12:19:13 2013
@@ -32,8 +32,10 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.shingle.ShingleFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
/**
* Tests {@link EdgeNGramTokenFilter} for correctness.
@@ -192,9 +194,9 @@ public class EdgeNGramTokenFilterTest ex
}
public void testGraphs() throws IOException {
- TokenStream tk = new LetterTokenizer(Version.LUCENE_44, new StringReader("abc d efgh ij klmno p q"));
+ TokenStream tk = new LetterTokenizer(TEST_VERSION_CURRENT, new StringReader("abc d efgh ij klmno p q"));
tk = new ShingleFilter(tk);
- tk = new EdgeNGramTokenFilter(Version.LUCENE_44, tk, 7, 10);
+ tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, 7, 10);
tk.reset();
assertTokenStreamContents(tk,
new String[] { "efgh ij", "ij klmn", "ij klmno", "klmno p" },
@@ -205,4 +207,25 @@ public class EdgeNGramTokenFilterTest ex
23
);
}
+
+ public void testSupplementaryCharacters() throws IOException {
+ final String s = _TestUtil.randomUnicodeString(random(), 10);
+ final int codePointCount = s.codePointCount(0, s.length());
+ final int minGram = _TestUtil.nextInt(random(), 1, 3);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 10);
+ TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
+ final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
+ final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
+ tk.reset();
+ for (int i = minGram; i <= Math.min(codePointCount, maxGram); ++i) {
+ assertTrue(tk.incrementToken());
+ assertEquals(0, offsetAtt.startOffset());
+ assertEquals(s.length(), offsetAtt.endOffset());
+ final int end = Character.offsetByCodePoints(s, 0, i);
+ assertEquals(s.substring(0, end), termAtt.toString());
+ }
+ assertFalse(tk.incrementToken());
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Sun Aug 11 12:19:13 2013
@@ -21,15 +21,15 @@ package org.apache.lucene.analysis.ngram
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util._TestUtil;
+import com.carrotsearch.randomizedtesting.generators.RandomStrings;
+
/**
* Tests {@link EdgeNGramTokenizer} for correctness.
*/
@@ -120,25 +120,60 @@ public class EdgeNGramTokenizerTest exte
false);
}
+ private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
+ final String s = RandomStrings.randomAsciiOfLength(random(), length);
+ testNGrams(minGram, maxGram, s, nonTokenChars);
+ }
+
+ private static void testNGrams(int minGram, int maxGram, String s, String nonTokenChars) throws IOException {
+ NGramTokenizerTest.testNGrams(minGram, maxGram, s, nonTokenChars, true);
+ }
+
public void testLargeInput() throws IOException {
- final String input = _TestUtil.randomSimpleString(random(), 1024 * 5);
- final int minGram = _TestUtil.nextInt(random(), 1, 1024);
- final int maxGram = _TestUtil.nextInt(random(), minGram, 5 * 1024);
- EdgeNGramTokenizer tk = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, new StringReader(input), minGram, maxGram);
- final CharTermAttribute charTermAtt = tk.addAttribute(CharTermAttribute.class);
- final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
- final PositionIncrementAttribute posIncAtt = tk.addAttribute(PositionIncrementAttribute.class);
- tk.reset();
- for (int i = minGram; i <= maxGram && i <= input.length(); ++i) {
- assertTrue(tk.incrementToken());
- assertEquals(0, offsetAtt.startOffset());
- assertEquals(i, offsetAtt.endOffset());
- assertEquals(1, posIncAtt.getPositionIncrement());
- assertEquals(input.substring(0, i), charTermAtt.toString());
+ // test sliding
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+ public void testLargeMaxGram() throws IOException {
+ // test sliding with maxGram > 1024
+ final int minGram = _TestUtil.nextInt(random(), 1290, 1300);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 1300);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+ public void testPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "a");
+ }
+
+ public void testHeavyPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "abcdef");
+ }
+
+ public void testFewTokenChars() throws IOException {
+ final char[] chrs = new char[_TestUtil.nextInt(random(), 4000, 5000)];
+ Arrays.fill(chrs, ' ');
+ for (int i = 0; i < chrs.length; ++i) {
+ if (random().nextFloat() < 0.1) {
+ chrs[i] = 'a';
+ }
}
- assertFalse(tk.incrementToken());
- tk.end();
- assertEquals(input.length(), offsetAtt.startOffset());
+ final int minGram = _TestUtil.nextInt(random(), 1, 2);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 2);
+ testNGrams(minGram, maxGram, new String(chrs), " ");
+ }
+
+ public void testFullUTF8Range() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ final String s = _TestUtil.randomUnicodeString(random(), 4 * 1024);
+ testNGrams(minGram, maxGram, s, "");
+ testNGrams(minGram, maxGram, s, "abcdef");
}
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Sun Aug 11 12:19:13 2013
@@ -26,7 +26,10 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
import java.io.IOException;
import java.io.Reader;
@@ -177,4 +180,27 @@ public class NGramTokenFilterTest extend
);
}
+ public void testSupplementaryCharacters() throws IOException {
+ final String s = _TestUtil.randomUnicodeString(random(), 10);
+ final int codePointCount = s.codePointCount(0, s.length());
+ final int minGram = _TestUtil.nextInt(random(), 1, 3);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 10);
+ TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ tk = new NGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
+ final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
+ final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
+ tk.reset();
+ for (int start = 0; start < codePointCount; ++start) {
+ for (int end = start + minGram; end <= Math.min(codePointCount, start + maxGram); ++end) {
+ assertTrue(tk.incrementToken());
+ assertEquals(0, offsetAtt.startOffset());
+ assertEquals(s.length(), offsetAtt.endOffset());
+ final int startIndex = Character.offsetByCodePoints(s, 0, start);
+ final int endIndex = Character.offsetByCodePoints(s, 0, end);
+ assertEquals(s.substring(startIndex, endIndex), termAtt.toString());
+ }
+ }
+ assertFalse(tk.incrementToken());
+ }
+
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Sun Aug 11 12:19:13 2013
@@ -18,9 +18,12 @@ package org.apache.lucene.analysis.ngram
*/
+import static org.apache.lucene.analysis.ngram.NGramTokenizerTest.isTokenChar;
+
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -115,23 +118,74 @@ public class NGramTokenizerTest extends
checkRandomData(random(), a, 50*RANDOM_MULTIPLIER, 1027, false, false);
}
- private void testNGrams(int minGram, int maxGram, int length) throws IOException {
+ private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
final String s = RandomStrings.randomAsciiOfLength(random(), length);
- final TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram);
+ testNGrams(minGram, maxGram, s, nonTokenChars);
+ }
+
+ private static void testNGrams(int minGram, int maxGram, String s, String nonTokenChars) throws IOException {
+ testNGrams(minGram, maxGram, s, nonTokenChars, false);
+ }
+
+ static int[] toCodePoints(CharSequence s) {
+ final int[] codePoints = new int[Character.codePointCount(s, 0, s.length())];
+ for (int i = 0, j = 0; i < s.length(); ++j) {
+ codePoints[j] = Character.codePointAt(s, i);
+ i += Character.charCount(codePoints[j]);
+ }
+ return codePoints;
+ }
+
+ static boolean isTokenChar(String nonTokenChars, int codePoint) {
+ for (int i = 0; i < nonTokenChars.length(); ) {
+ final int cp = nonTokenChars.codePointAt(i);
+ if (cp == codePoint) {
+ return false;
+ }
+ i += Character.charCount(cp);
+ }
+ return true;
+ }
+
+ static void testNGrams(int minGram, int maxGram, String s, final String nonTokenChars, boolean edgesOnly) throws IOException {
+ // convert the string to code points
+ final int[] codePoints = toCodePoints(s);
+ final int[] offsets = new int[codePoints.length + 1];
+ for (int i = 0; i < codePoints.length; ++i) {
+ offsets[i+1] = offsets[i] + Character.charCount(codePoints[i]);
+ }
+ final TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram, edgesOnly) {
+ @Override
+ protected boolean isTokenChar(int chr) {
+ return nonTokenChars.indexOf(chr) < 0;
+ }
+ };
final CharTermAttribute termAtt = grams.addAttribute(CharTermAttribute.class);
final PositionIncrementAttribute posIncAtt = grams.addAttribute(PositionIncrementAttribute.class);
final PositionLengthAttribute posLenAtt = grams.addAttribute(PositionLengthAttribute.class);
final OffsetAttribute offsetAtt = grams.addAttribute(OffsetAttribute.class);
grams.reset();
- for (int start = 0; start < s.length(); ++start) {
- for (int end = start + minGram; end <= start + maxGram && end <= s.length(); ++end) {
+ for (int start = 0; start < codePoints.length; ++start) {
+ nextGram:
+ for (int end = start + minGram; end <= start + maxGram && end <= codePoints.length; ++end) {
+ if (edgesOnly && start > 0 && isTokenChar(nonTokenChars, codePoints[start - 1])) {
+ // not on an edge
+ continue nextGram;
+ }
+ for (int j = start; j < end; ++j) {
+ if (!isTokenChar(nonTokenChars, codePoints[j])) {
+ continue nextGram;
+ }
+ }
assertTrue(grams.incrementToken());
- assertEquals(s.substring(start, end), termAtt.toString());
+ assertArrayEquals(Arrays.copyOfRange(codePoints, start, end), toCodePoints(termAtt));
assertEquals(1, posIncAtt.getPositionIncrement());
- assertEquals(start, offsetAtt.startOffset());
- assertEquals(end, offsetAtt.endOffset());
+ assertEquals(1, posLenAtt.getPositionLength());
+ assertEquals(offsets[start], offsetAtt.startOffset());
+ assertEquals(offsets[end], offsetAtt.endOffset());
}
}
+ assertFalse(grams.incrementToken());
grams.end();
assertEquals(s.length(), offsetAtt.startOffset());
assertEquals(s.length(), offsetAtt.endOffset());
@@ -141,14 +195,47 @@ public class NGramTokenizerTest extends
// test sliding
final int minGram = _TestUtil.nextInt(random(), 1, 100);
final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
- testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024));
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
}
public void testLargeMaxGram() throws IOException {
// test sliding with maxGram > 1024
- final int minGram = _TestUtil.nextInt(random(), 1200, 1300);
+ final int minGram = _TestUtil.nextInt(random(), 1290, 1300);
final int maxGram = _TestUtil.nextInt(random(), minGram, 1300);
- testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024));
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+ public void testPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "a");
+ }
+
+ public void testHeavyPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "abcdef");
+ }
+
+ public void testFewTokenChars() throws IOException {
+ final char[] chrs = new char[_TestUtil.nextInt(random(), 4000, 5000)];
+ Arrays.fill(chrs, ' ');
+ for (int i = 0; i < chrs.length; ++i) {
+ if (random().nextFloat() < 0.1) {
+ chrs[i] = 'a';
+ }
+ }
+ final int minGram = _TestUtil.nextInt(random(), 1, 2);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 2);
+ testNGrams(minGram, maxGram, new String(chrs), " ");
+ }
+
+ public void testFullUTF8Range() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ final String s = _TestUtil.randomUnicodeString(random(), 4 * 1024);
+ testNGrams(minGram, maxGram, s, "");
+ testNGrams(minGram, maxGram, s, "abcdef");
}
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java Sun Aug 11 12:19:13 2013
@@ -32,6 +32,9 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK;
+
/**
* Simple tests for {@link NorwegianLightStemFilter}
@@ -42,7 +45,7 @@ public class TestNorwegianLightStemFilte
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new NorwegianLightStemFilter(source));
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL));
}
};
@@ -51,6 +54,18 @@ public class TestNorwegianLightStemFilte
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_light.txt")));
}
+ /** Test against a Nynorsk vocabulary file */
+ public void testNynorskVocabulary() throws IOException {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NYNORSK));
+ }
+ };
+ assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_light.txt")));
+ }
+
public void testKeyword() throws IOException {
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java Sun Aug 11 12:19:13 2013
@@ -34,7 +34,23 @@ public class TestNorwegianLightStemFilte
stream = tokenFilterFactory("NorwegianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl" });
}
-
+
+ /** Test stemming with variant set explicitly to Bokmål */
+ public void testBokmaalStemming() throws Exception {
+ Reader reader = new StringReader("epler eple");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nb").create(stream);
+ assertTokenStreamContents(stream, new String[] { "epl", "epl" });
+ }
+
+ /** Test stemming with variant set explicitly to Nynorsk */
+ public void testNynorskStemming() throws Exception {
+ Reader reader = new StringReader("gutar gutane");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nn").create(stream);
+ assertTokenStreamContents(stream, new String[] { "gut", "gut" });
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java Sun Aug 11 12:19:13 2013
@@ -32,6 +32,8 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK;
/**
* Simple tests for {@link NorwegianMinimalStemFilter}
@@ -42,15 +44,27 @@ public class TestNorwegianMinimalStemFil
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source));
+ return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, BOKMAAL));
}
};
- /** Test against a vocabulary file */
+ /** Test against a Bokmål vocabulary file */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_minimal.txt")));
}
+ /** Test against a Nynorsk vocabulary file */
+ public void testNynorskVocabulary() throws IOException {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, NYNORSK));
+ }
+ };
+ assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_minimal.txt")));
+ }
+
public void testKeyword() throws IOException {
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java Sun Aug 11 12:19:13 2013
@@ -35,6 +35,22 @@ public class TestNorwegianMinimalStemFil
assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
}
+ /** Test stemming with variant set explicitly to Bokmål */
+ public void testBokmaalStemming() throws Exception {
+ Reader reader = new StringReader("eple eplet epler eplene eplets eplenes");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nb").create(stream);
+ assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
+ }
+
+ /** Test stemming with variant set explicitly to Nynorsk */
+ public void testNynorskStemming() throws Exception {
+ Reader reader = new StringReader("gut guten gutar gutane gutens gutanes");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nn").create(stream);
+ assertTokenStreamContents(stream, new String[] { "gut", "gut", "gut", "gut", "gut", "gut" });
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Sun Aug 11 12:19:13 2013
@@ -26,7 +26,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.RAMDirectory;
-import java.io.StringReader;
import java.util.Arrays;
import java.util.Collections;
@@ -66,44 +65,44 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoStopwords() throws Exception {
// Note: an empty list of fields passed in
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Collections.<String>emptyList(), 1);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("quick"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("variedField", "quick");
assertTokenStreamContents(protectedTokenStream, new String[]{"quick"});
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
public void testDefaultStopwordsAllFields() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
assertTokenStreamContents(protectedTokenStream, new String[0]); // Default stop word filtering will remove boring
}
public void testStopwordsAllFieldsMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on terms in > one half of docs remove boring
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring");
// A filter on terms in > half of docs should not remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[]{"vaguelyboring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 4f);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring");
// A filter on terms in > quarter of docs should remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
public void testStopwordsPerFieldMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField"), 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on one Field should not affect queries on another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField", "repetitiveField"), 1f / 2f);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on the right Field should affect queries on it
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
@@ -121,11 +120,11 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoFieldNamePollution() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("repetitiveField"), 10);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// Check filter set up OK
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("variedField", "boring");
// Filter should not prevent stopwords in one field being used in another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
@@ -134,7 +133,7 @@ public class QueryAutoStopWordAnalyzerTe
QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(
TEST_VERSION_CURRENT,
new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), reader, 10);
- TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
+ TokenStream ts = a.tokenStream("repetitiveField", "this boring");
assertTokenStreamContents(ts, new String[] { "this" });
}
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Sun Aug 11 12:19:13 2013
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.shing
* limitations under the License.
*/
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -97,7 +95,7 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperPhraseQuery() throws Exception {
PhraseQuery q = new PhraseQuery();
- TokenStream ts = analyzer.tokenStream("content", new StringReader("this sentence"));
+ TokenStream ts = analyzer.tokenStream("content", "this sentence");
int j = -1;
PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
@@ -123,7 +121,7 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperBooleanQuery() throws Exception {
BooleanQuery q = new BooleanQuery();
- TokenStream ts = analyzer.tokenStream("content", new StringReader("test sentence"));
+ TokenStream ts = analyzer.tokenStream("content", "test sentence");
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Sun Aug 11 12:19:13 2013
@@ -91,7 +91,7 @@ public class TestTeeSinkTokenFilter exte
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream tokenStream = analyzer.tokenStream("field", new StringReader("abcd "));
+ TokenStream tokenStream = analyzer.tokenStream("field", "abcd ");
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(tokenStream);
TokenStream sink = tee.newSinkTokenStream();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java Sun Aug 11 12:19:13 2013
@@ -29,6 +29,7 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
+import org.apache.lucene.analysis.cjk.CJKAnalyzer;
public class TestSynonymFilterFactory extends BaseTokenStreamFactoryTestCase {
/** test that we can parse and use the solr syn file */
@@ -64,6 +65,28 @@ public class TestSynonymFilterFactory ex
}
}
+ /** Test that analyzer and tokenizerFactory is both specified */
+ public void testAnalyzer() throws Exception {
+ final String analyzer = CJKAnalyzer.class.getName();
+ final String tokenizerFactory = PatternTokenizerFactory.class.getName();
+ TokenFilterFactory factory = null;
+
+ factory = tokenFilterFactory("Synonym",
+ "synonyms", "synonyms2.txt",
+ "analyzer", analyzer);
+ assertNotNull(factory);
+
+ try {
+ tokenFilterFactory("Synonym",
+ "synonyms", "synonyms.txt",
+ "analyzer", analyzer,
+ "tokenizerFactory", tokenizerFactory);
+ fail();
+ } catch (IllegalArgumentException expected) {
+ assertTrue(expected.getMessage().contains("Analyzer and TokenizerFactory can't be specified both"));
+ }
+ }
+
static final String TOK_SYN_ARG_VAL = "argument";
static final String TOK_FOO_ARG_VAL = "foofoofoo";
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Sun Aug 11 12:19:13 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.th;
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -30,7 +29,6 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.util.Version;
/**
* Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer
@@ -122,10 +120,10 @@ public class TestThaiAnalyzer extends Ba
public void testAttributeReuse() throws Exception {
ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
// just consume
- TokenStream ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ TokenStream ts = analyzer.tokenStream("dummy", "ภาษาà¹à¸à¸¢");
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
// this consumer adds flagsAtt, which this analyzer does not use.
- ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ ts = analyzer.tokenStream("dummy", "ภาษาà¹à¸à¸¢");
ts.addAttribute(FlagsAttribute.class);
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Sun Aug 11 12:19:13 2013
@@ -123,7 +123,7 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
ts.reset();
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
while (ts.incrementToken()) {
@@ -161,7 +161,7 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
ts.reset();
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java Sun Aug 11 12:19:13 2013
@@ -20,10 +20,11 @@ package org.apache.lucene.analysis.util;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/**
@@ -32,32 +33,6 @@ import org.junit.Test;
public class TestCharacterUtils extends LuceneTestCase {
@Test
- public void testCodePointAtCharArrayInt() {
- CharacterUtils java4 = CharacterUtils.getJava4Instance();
- char[] cpAt3 = "Abc\ud801\udc1c".toCharArray();
- char[] highSurrogateAt3 = "Abc\ud801".toCharArray();
- assertEquals((int) 'A', java4.codePointAt(cpAt3, 0));
- assertEquals((int) '\ud801', java4.codePointAt(cpAt3, 3));
- assertEquals((int) '\ud801', java4.codePointAt(highSurrogateAt3, 3));
- try {
- java4.codePointAt(highSurrogateAt3, 4);
- fail("array index out of bounds");
- } catch (IndexOutOfBoundsException e) {
- }
-
- CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
- assertEquals((int) 'A', java5.codePointAt(cpAt3, 0));
- assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
- cpAt3, 3));
- assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3));
- try {
- java5.codePointAt(highSurrogateAt3, 4);
- fail("array index out of bounds");
- } catch (IndexOutOfBoundsException e) {
- }
- }
-
- @Test
public void testCodePointAtCharSequenceInt() {
CharacterUtils java4 = CharacterUtils.getJava4Instance();
String cpAt3 = "Abc\ud801\udc1c";
@@ -98,7 +73,68 @@ public class TestCharacterUtils extends
assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
cpAt3, 3, 5));
assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3, 4));
+ }
+
+ @Test
+ public void testCodePointCount() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ final String s = _TestUtil.randomUnicodeString(random());
+ assertEquals(s.length(), java4.codePointCount(s));
+ assertEquals(Character.codePointCount(s, 0, s.length()), java5.codePointCount(s));
+ }
+
+ @Test
+ public void testOffsetByCodePoint() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ for (int i = 0; i < 10; ++i) {
+ final char[] s = _TestUtil.randomUnicodeString(random()).toCharArray();
+ final int index = _TestUtil.nextInt(random(), 0, s.length);
+ final int offset = random().nextInt(7) - 3;
+ try {
+ final int o = java4.offsetByCodePoints(s, 0, s.length, index, offset);
+ assertEquals(o, index + offset);
+ } catch (IndexOutOfBoundsException e) {
+ assertTrue((index + offset) < 0 || (index + offset) > s.length);
+ }
+
+ int o;
+ try {
+ o = java5.offsetByCodePoints(s, 0, s.length, index, offset);
+ } catch (IndexOutOfBoundsException e) {
+ try {
+ Character.offsetByCodePoints(s, 0, s.length, index, offset);
+ fail();
+ } catch (IndexOutOfBoundsException e2) {
+ // OK
+ }
+ o = -1;
+ }
+ if (o >= 0) {
+ assertEquals(Character.offsetByCodePoints(s, 0, s.length, index, offset), o);
+ }
+ }
+ }
+
+ public void testConversions() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ testConversions(java4);
+ testConversions(java5);
+ }
+ private void testConversions(CharacterUtils charUtils) {
+ final char[] orig = _TestUtil.randomUnicodeString(random(), 100).toCharArray();
+ final int[] buf = new int[orig.length];
+ final char[] restored = new char[buf.length];
+ final int o1 = _TestUtil.nextInt(random(), 0, Math.min(5, orig.length));
+ final int o2 = _TestUtil.nextInt(random(), 0, o1);
+ final int o3 = _TestUtil.nextInt(random(), 0, o1);
+ final int codePointCount = charUtils.toCodePoints(orig, o1, orig.length - o1, buf, o2);
+ final int charCount = charUtils.toChars(buf, o2, codePointCount, restored, o3);
+ assertEquals(orig.length - o1, charCount);
+ assertArrayEquals(Arrays.copyOfRange(orig, o1, o1 + charCount), Arrays.copyOfRange(restored, o3, o3 + charCount));
}
@Test
@@ -132,7 +168,7 @@ public class TestCharacterUtils extends
assertEquals(0, buffer.getOffset());
assertEquals(6, buffer.getLength());
assertEquals("hellow", new String(buffer.getBuffer()));
- assertTrue(instance.fill(buffer,reader));
+ assertFalse(instance.fill(buffer,reader));
assertEquals(4, buffer.getLength());
assertEquals(0, buffer.getOffset());
@@ -159,15 +195,12 @@ public class TestCharacterUtils extends
assertEquals(4, buffer.getLength());
assertEquals("123\ud801", new String(buffer.getBuffer(),
buffer.getOffset(), buffer.getLength()));
- assertTrue(instance.fill(buffer, reader));
- assertEquals(2, buffer.getLength());
- assertEquals("\ud801\udc1c", new String(buffer.getBuffer(), buffer
- .getOffset(), buffer.getLength()));
- assertTrue(instance.fill(buffer, reader));
- assertEquals(1, buffer.getLength());
- assertEquals("\ud801", new String(buffer.getBuffer(), buffer
+ assertFalse(instance.fill(buffer, reader));
+ assertEquals(3, buffer.getLength());
+ assertEquals("\ud801\udc1c\ud801", new String(buffer.getBuffer(), buffer
.getOffset(), buffer.getLength()));
assertFalse(instance.fill(buffer, reader));
+ assertEquals(0, buffer.getLength());
}
@Test
Modified: lucene/dev/branches/lucene4956/lucene/analysis/icu/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/icu/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/icu/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/icu/build.xml Sun Aug 11 12:19:13 2013
@@ -23,6 +23,8 @@
Analysis integration with ICU (International Components for Unicode).
</description>
+ <property name="rat.additional-includes" value="src/tools/**"/>
+
<import file="../analysis-module-build.xml"/>
<path id="icujar">
@@ -59,7 +61,7 @@
<property name="gennorm2.src.files"
value="nfc.txt nfkc.txt nfkc_cf.txt BasicFoldings.txt DiacriticFolding.txt DingbatFolding.txt HanRadicalFolding.txt NativeDigitFolding.txt"/>
<property name="gennorm2.tmp" value="${build.dir}/gennorm2/utr30.tmp"/>
- <property name="gennorm2.dst" value="src/resources/org/apache/lucene/analysis/icu/utr30.nrm"/>
+ <property name="gennorm2.dst" value="${resources.dir}/org/apache/lucene/analysis/icu/utr30.nrm"/>
<target name="gennorm2" depends="gen-utr30-data-files">
<echo>Note that the gennorm2 and icupkg tools must be on your PATH. These tools
are part of the ICU4C package. See http://site.icu-project.org/ </echo>
@@ -82,7 +84,7 @@ are part of the ICU4C package. See http:
</target>
<property name="rbbi.src.dir" location="src/data/uax29"/>
- <property name="rbbi.dst.dir" location="src/resources/org/apache/lucene/analysis/icu/segmentation"/>
+ <property name="rbbi.dst.dir" location="${resources.dir}/org/apache/lucene/analysis/icu/segmentation"/>
<target name="genrbbi" depends="compile-tools">
<mkdir dir="${rbbi.dst.dir}"/>
@@ -148,4 +150,6 @@ are part of the ICU4C package. See http:
</compile>
</target>
+ <target name="regenerate" depends="gen-html-strip-charfilter-supp-macros,gen-uax29-supp-macros,gen-utr30-data-files,gennorm2,genrbbi"/>
+
</project>
Modified: lucene/dev/branches/lucene4956/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Sun Aug 11 12:19:13 2013
@@ -249,7 +249,7 @@ public class TestICUTokenizer extends Ba
}
public void testTokenAttributes() throws Exception {
- TokenStream ts = a.tokenStream("dummy", new StringReader("This is a test"));
+ TokenStream ts = a.tokenStream("dummy", "This is a test");
ScriptAttribute scriptAtt = ts.addAttribute(ScriptAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateHTMLStripCharFilterSupplementaryMacros.java Sun Aug 11 12:19:13 2013
@@ -60,8 +60,7 @@ public class GenerateHTMLStripCharFilter
static void outputHeader() {
System.out.print(APACHE_LICENSE);
- System.out.print("// Generated using ICU4J " + VersionInfo.ICU_VERSION.toString() + " on ");
- System.out.println(DATE_FORMAT.format(new Date()));
+ System.out.println("// Generated using ICU4J " + VersionInfo.ICU_VERSION.toString());
System.out.println("// by " + GenerateHTMLStripCharFilterSupplementaryMacros.class.getName());
System.out.print(NL + NL);
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateJFlexSupplementaryMacros.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateJFlexSupplementaryMacros.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateJFlexSupplementaryMacros.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateJFlexSupplementaryMacros.java Sun Aug 11 12:19:13 2013
@@ -74,8 +74,7 @@ public class GenerateJFlexSupplementaryM
static void outputHeader() {
System.out.print(APACHE_LICENSE);
- System.out.print("// Generated using ICU4J " + VersionInfo.ICU_VERSION.toString() + " on ");
- System.out.println(DATE_FORMAT.format(new Date()));
+ System.out.println("// Generated using ICU4J " + VersionInfo.ICU_VERSION.toString());
System.out.println("// by " + GenerateJFlexSupplementaryMacros.class.getName());
System.out.print(NL + NL);
}
Modified: lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/build.xml Sun Aug 11 12:19:13 2013
@@ -26,6 +26,7 @@
<!-- currently whether rat detects this as binary or not
is platform dependent?! -->
<property name="rat.excludes" value="**/*.txt,**/bocchan.utf-8"/>
+ <property name="rat.additional-includes" value="src/tools/**"/>
<!-- we don't want to pull in ipadic/naist etc -->
<property name="ivy.default.configuration" value="default"/>
@@ -45,7 +46,7 @@
<property name="dict.encoding" value="euc-jp"/>
<property name="dict.format" value="ipadic"/>
<property name="dict.normalize" value="false"/>
- <property name="dict.target.dir" location="./src/resources"/>
+ <property name="dict.target.dir" location="${resources.dir}"/>
<available type="dir" file="${build.dir}/${ipadic.version}" property="dict.available"/>
@@ -83,7 +84,7 @@
<target name="build-dict" depends="compile-tools, download-dict">
<sequential>
<delete verbose="true">
- <fileset dir="src/resources/org/apache/lucene/analysis/ja/dict" includes="**/*"/>
+ <fileset dir="${resources.dir}/org/apache/lucene/analysis/ja/dict" includes="**/*"/>
</delete>
<!-- TODO: optimize the dictionary construction a bit so that you don't need 1G -->
<java fork="true" failonerror="true" maxmemory="1g" classname="org.apache.lucene.analysis.ja.util.DictionaryBuilder">
@@ -138,4 +139,6 @@
<!-- TODO: not until we properly make 'test-tools' work with clover etc
<target name="test" depends="module-build.test, test-tools"/> -->
+ <target name="regenerate" depends="build-dict"/>
+
</project>
Modified: lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java Sun Aug 11 12:19:13 2013
@@ -24,7 +24,7 @@ import org.apache.lucene.analysis.util.T
import java.util.Map;
/**
- * Factory for {@link JapaneseKatakanaStemFilterFactory}.
+ * Factory for {@link JapaneseKatakanaStemFilter}.
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
Modified: lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java Sun Aug 11 12:19:13 2013
@@ -44,7 +44,7 @@ public final class TokenInfoDictionary e
try {
is = getResource(FST_FILENAME_SUFFIX);
is = new BufferedInputStream(is);
- fst = new FST<Long>(new InputStreamDataInput(is), PositiveIntOutputs.getSingleton(true));
+ fst = new FST<Long>(new InputStreamDataInput(is), PositiveIntOutputs.getSingleton());
} catch (IOException ioe) {
priorE = ioe;
} finally {
Modified: lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java (original)
+++ lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java Sun Aug 11 12:19:13 2013
@@ -88,7 +88,7 @@ public final class UserDictionary implem
List<String> data = new ArrayList<String>(featureEntries.size());
List<int[]> segmentations = new ArrayList<int[]>(featureEntries.size());
- PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton(true);
+ PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton();
Builder<Long> fstBuilder = new Builder<Long>(FST.INPUT_TYPE.BYTE2, fstOutput);
IntsRef scratch = new IntsRef();
long ord = 0;
Modified: lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary$fst.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/kuromoji/src/resources/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary%24fst.dat?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
Binary files - no diff available.