You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2015/03/31 07:22:50 UTC
svn commit: r1670257 [5/39] - in /lucene/dev/branches/lucene6271: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous...
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java Tue Mar 31 05:22:40 2015
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.sv;
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -35,13 +34,25 @@ import static org.apache.lucene.analysis
* Simple tests for {@link SwedishLightStemFilter}
*/
public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new SwedishLightStemFilter(source));
- }
- };
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new SwedishLightStemFilter(source));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
@@ -59,6 +70,7 @@ public class TestSwedishLightStemFilter
}
};
checkOneTerm(a, "jaktkarlens", "jaktkarlens");
+ a.close();
}
/** blast some random strings through the analyzer */
@@ -75,5 +87,6 @@ public class TestSwedishLightStemFilter
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java Tue Mar 31 05:22:40 2015
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis.synon
* limitations under the License.
*/
-import java.io.Reader;
import java.io.StringReader;
import java.text.ParseException;
@@ -27,7 +26,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
-import org.junit.Test;
/**
* Tests parser for the Solr synonyms format
@@ -43,11 +41,13 @@ public class TestSolrSynonymParser exten
"foo => baz\n" +
"this test, that testing";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random()));
+ Analyzer analyzer = new MockAnalyzer(random());
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
parser.parse(new StringReader(testFile));
final SynonymMap map = parser.build();
+ analyzer.close();
- Analyzer analyzer = new Analyzer() {
+ analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
@@ -70,46 +70,77 @@ public class TestSolrSynonymParser exten
assertAnalyzesTo(analyzer, "this test",
new String[] { "this", "that", "test", "testing" },
new int[] { 1, 0, 1, 0 });
+ analyzer.close();
}
/** parse a syn file with bad syntax */
- @Test(expected=ParseException.class)
public void testInvalidDoubleMap() throws Exception {
- String testFile = "a => b => c";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random()));
- parser.parse(new StringReader(testFile));
+ String testFile = "a => b => c";
+ Analyzer analyzer = new MockAnalyzer(random());
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
+ try {
+ parser.parse(new StringReader(testFile));
+ fail("didn't get expected exception");
+ } catch (ParseException expected) {
+ // expected exc
+ }
+ analyzer.close();
}
/** parse a syn file with bad syntax */
- @Test(expected=ParseException.class)
public void testInvalidAnalyzesToNothingOutput() throws Exception {
String testFile = "a => 1";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.SIMPLE, false));
- parser.parse(new StringReader(testFile));
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, false);
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
+ try {
+ parser.parse(new StringReader(testFile));
+ fail("didn't get expected exception");
+ } catch (ParseException expected) {
+ // expected exc
+ }
+ analyzer.close();
}
/** parse a syn file with bad syntax */
- @Test(expected=ParseException.class)
public void testInvalidAnalyzesToNothingInput() throws Exception {
- String testFile = "1 => a";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.SIMPLE, false));
- parser.parse(new StringReader(testFile));
+ String testFile = "1 => a";
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, false);
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
+ try {
+ parser.parse(new StringReader(testFile));
+ fail("didn't get expected exception");
+ } catch (ParseException expected) {
+ // expected exc
+ }
+ analyzer.close();
}
/** parse a syn file with bad syntax */
- @Test(expected=ParseException.class)
public void testInvalidPositionsInput() throws Exception {
String testFile = "testola => the test";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new EnglishAnalyzer());
- parser.parse(new StringReader(testFile));
+ Analyzer analyzer = new EnglishAnalyzer();
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
+ try {
+ parser.parse(new StringReader(testFile));
+ fail("didn't get expected exception");
+ } catch (ParseException expected) {
+ // expected exc
+ }
+ analyzer.close();
}
/** parse a syn file with bad syntax */
- @Test(expected=ParseException.class)
public void testInvalidPositionsOutput() throws Exception {
String testFile = "the test => testola";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new EnglishAnalyzer());
- parser.parse(new StringReader(testFile));
+ Analyzer analyzer = new EnglishAnalyzer();
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
+ try {
+ parser.parse(new StringReader(testFile));
+ fail("didn't get expected exception");
+ } catch (ParseException expected) {
+ // expected exc
+ }
+ analyzer.close();
}
/** parse a syn file with some escaped syntax chars */
@@ -117,10 +148,12 @@ public class TestSolrSynonymParser exten
String testFile =
"a\\=>a => b\\=>b\n" +
"a\\,a => b\\,b";
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
+ Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer);
parser.parse(new StringReader(testFile));
final SynonymMap map = parser.build();
- Analyzer analyzer = new Analyzer() {
+ analyzer.close();
+ analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
@@ -139,5 +172,6 @@ public class TestSolrSynonymParser exten
assertAnalyzesTo(analyzer, "a,a",
new String[] { "b,b" },
new int[] { 1 });
+ analyzer.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Tue Mar 31 05:22:40 2015
@@ -37,7 +37,6 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.MockGraphTokenFilter;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
-import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.TestUtil;
@@ -166,6 +165,7 @@ public class TestSynonymMapFilter extend
new int[] {1, 1},
true);
checkAnalysisConsistency(random(), analyzer, false, "a b c");
+ analyzer.close();
}
public void testDoKeepOrig() throws Exception {
@@ -191,6 +191,7 @@ public class TestSynonymMapFilter extend
new int[] {1, 2, 1, 1},
true);
checkAnalysisConsistency(random(), analyzer, false, "a b c");
+ analyzer.close();
}
public void testBasic() throws Exception {
@@ -502,6 +503,7 @@ public class TestSynonymMapFilter extend
};
checkRandomData(random(), analyzer, 100);
+ analyzer.close();
}
}
@@ -560,6 +562,7 @@ public class TestSynonymMapFilter extend
};
checkRandomData(random, analyzer, 100);
+ analyzer.close();
}
}
@@ -584,6 +587,7 @@ public class TestSynonymMapFilter extend
};
checkAnalysisConsistency(random, analyzer, random.nextBoolean(), "");
+ analyzer.close();
}
}
@@ -613,6 +617,7 @@ public class TestSynonymMapFilter extend
};
checkRandomData(random, analyzer, 100, 1024);
+ analyzer.close();
}
}
@@ -621,10 +626,11 @@ public class TestSynonymMapFilter extend
String testFile =
"aaa => aaaa1 aaaa2 aaaa3\n" +
"bbb => bbbb1 bbbb2\n";
-
- SolrSynonymParser parser = new SolrSynonymParser(true, true, new MockAnalyzer(random()));
+ Analyzer synAnalyzer = new MockAnalyzer(random());
+ SolrSynonymParser parser = new SolrSynonymParser(true, true, synAnalyzer);
parser.parse(new StringReader(testFile));
final SynonymMap map = parser.build();
+ synAnalyzer.close();
Analyzer analyzer = new Analyzer() {
@Override
@@ -642,6 +648,7 @@ public class TestSynonymMapFilter extend
// xyzzy aaa pot of gold -> xyzzy aaaa1 aaaa2 aaaa3 gold
assertAnalyzesTo(analyzer, "xyzzy aaa pot of gold",
new String[] { "xyzzy", "aaaa1", "pot", "aaaa2", "of", "aaaa3", "gold" });
+ analyzer.close();
}
public void testBasic2() throws Exception {
@@ -716,6 +723,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "z x c $",
new String[] { "z", "xc", "$" },
new int[] { 1, 1, 1 });
+ a.close();
}
public void testRepeatsOff() throws Exception {
@@ -736,6 +744,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "a b",
new String[] { "ab" },
new int[] { 1 });
+ a.close();
}
public void testRepeatsOn() throws Exception {
@@ -756,6 +765,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "a b",
new String[] { "ab", "ab", "ab" },
new int[] { 1, 0, 0 });
+ a.close();
}
public void testRecursion() throws Exception {
@@ -774,6 +784,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "zoo zoo $ zoo",
new String[] { "zoo", "zoo", "$", "zoo" },
new int[] { 1, 1, 1, 1 });
+ a.close();
}
public void testRecursion2() throws Exception {
@@ -794,6 +805,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "zoo zoo $ zoo",
new String[] { "zoo", "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo", "zoo" },
new int[] { 1, 0, 1, 0, 0, 1, 0, 1, 0, 1 });
+ a.close();
}
public void testOutputHangsOffEnd() throws Exception {
@@ -869,6 +881,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "z x c $",
new String[] { "z", "x", "xc", "c", "$" },
new int[] { 1, 1, 0, 1, 1 });
+ a.close();
}
public void testRecursion3() throws Exception {
@@ -887,6 +900,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "zoo zoo $ zoo",
new String[] { "zoo", "zoo", "zoo", "$", "zoo" },
new int[] { 1, 0, 1, 1, 1 });
+ a.close();
}
public void testRecursion4() throws Exception {
@@ -906,6 +920,7 @@ public class TestSynonymMapFilter extend
assertAnalyzesTo(a, "zoo zoo $ zoo",
new String[] { "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" },
new int[] { 1, 0, 1, 1, 1, 0, 1 });
+ a.close();
}
public void testMultiwordOffsets() throws Exception {
@@ -926,6 +941,7 @@ public class TestSynonymMapFilter extend
new int[] { 0, 0, 9, 16 },
new int[] { 8, 22, 15, 22 },
new int[] { 1, 0, 1, 1 });
+ a.close();
}
public void testEmpty() throws Exception {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java Tue Mar 31 05:22:40 2015
@@ -17,7 +17,6 @@
package org.apache.lucene.analysis.synonym;
-import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
@@ -40,11 +39,13 @@ public class TestWordnetSynonymParser ex
"s(100000004,2,'king''s meany',n,1,1).\n";
public void testSynonyms() throws Exception {
- WordnetSynonymParser parser = new WordnetSynonymParser(true, true, new MockAnalyzer(random()));
+ Analyzer analyzer = new MockAnalyzer(random());
+ WordnetSynonymParser parser = new WordnetSynonymParser(true, true, analyzer);
parser.parse(new StringReader(synonymsFile));
final SynonymMap map = parser.build();
+ analyzer.close();
- Analyzer analyzer = new Analyzer() {
+ analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
@@ -66,5 +67,6 @@ public class TestWordnetSynonymParser ex
/* multi words */
assertAnalyzesTo(analyzer, "king's evil",
new String[] { "king's", "king's", "evil", "meany" });
+ analyzer.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -17,15 +17,9 @@ package org.apache.lucene.analysis.th;
* limitations under the License.
*/
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Random;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -46,18 +40,22 @@ public class TestThaiAnalyzer extends Ba
* testcase for offsets
*/
public void testOffsets() throws Exception {
- assertAnalyzesTo(new ThaiAnalyzer(CharArraySet.EMPTY_SET), "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET);
+ assertAnalyzesTo(analyzer, "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ" },
new int[] { 0, 3, 6, 9, 13, 17, 20, 23 },
new int[] { 3, 6, 9, 13, 17, 20, 23, 25 });
+ analyzer.close();
}
public void testStopWords() throws Exception {
- assertAnalyzesTo(new ThaiAnalyzer(), "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ Analyzer analyzer = new ThaiAnalyzer();
+ assertAnalyzesTo(analyzer, "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
new String[] { "à¹à¸ªà¸à¸", "à¸à¸²à¸", "à¸à¸µ" },
new int[] { 13, 20, 23 },
new int[] { 17, 23, 25 },
new int[] { 5, 2, 1 });
+ analyzer.close();
}
/*
@@ -78,32 +76,37 @@ public class TestThaiAnalyzer extends Ba
new int[] { 0, 3, 6, 9, 17, 21, 24, 27 },
new int[] { 3, 6, 9, 13, 21, 24, 27, 29 },
new int[] { 1, 1, 1, 1, 2, 1, 1, 1 });
+ analyzer.close();
}
public void testReusableTokenStream() throws Exception {
ThaiAnalyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET);
assertAnalyzesTo(analyzer, "", new String[] {});
-
- assertAnalyzesTo(
- analyzer,
- "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
- new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ"});
-
- assertAnalyzesTo(
- analyzer,
- "à¸à¸£à¸´à¸©à¸±à¸à¸à¸·à¹à¸ XY&Z - à¸à¸¸à¸¢à¸à¸±à¸ xyz@demo.com",
- new String[] { "à¸à¸£à¸´à¸©à¸±à¸", "à¸à¸·à¹à¸", "xy", "z", "à¸à¸¸à¸¢", "à¸à¸±à¸", "xyz", "demo.com" });
+
+ assertAnalyzesTo(
+ analyzer,
+ "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ"});
+
+ assertAnalyzesTo(
+ analyzer,
+ "à¸à¸£à¸´à¸©à¸±à¸à¸à¸·à¹à¸ XY&Z - à¸à¸¸à¸¢à¸à¸±à¸ xyz@demo.com",
+ new String[] { "à¸à¸£à¸´à¸©à¸±à¸", "à¸à¸·à¹à¸", "xy", "z", "à¸à¸¸à¸¢", "à¸à¸±à¸", "xyz", "demo.com" });
+ analyzer.close();
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new ThaiAnalyzer(), 1000*RANDOM_MULTIPLIER);
+ Analyzer analyzer = new ThaiAnalyzer();
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
+ analyzer.close();
}
/** blast some random large strings through the analyzer */
public void testRandomHugeStrings() throws Exception {
- Random random = random();
- checkRandomData(random, new ThaiAnalyzer(), 100*RANDOM_MULTIPLIER, 8192);
+ Analyzer analyzer = new ThaiAnalyzer();
+ checkRandomData(random(), analyzer, 100*RANDOM_MULTIPLIER, 8192);
+ analyzer.close();
}
// LUCENE-3044
@@ -116,12 +119,15 @@ public class TestThaiAnalyzer extends Ba
ts = analyzer.tokenStream("dummy", "ภาษาà¹à¸à¸¢");
ts.addAttribute(FlagsAttribute.class);
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
+ analyzer.close();
}
public void testTwoSentences() throws Exception {
- assertAnalyzesTo(new ThaiAnalyzer(CharArraySet.EMPTY_SET), "This is a test. à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
+ Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET);
+ assertAnalyzesTo(analyzer, "This is a test. à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
new String[] { "this", "is", "a", "test", "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ" },
new int[] { 0, 5, 8, 10, 16, 19, 22, 25, 29, 33, 36, 39 },
new int[] { 4, 7, 9, 14, 19, 22, 25, 29, 33, 36, 39, 41 });
+ analyzer.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -27,7 +27,7 @@ public class TestTurkishAnalyzer extends
/** This test fails with NPE when the
* stopwords file is missing in classpath */
public void testResourcesAvailable() {
- new TurkishAnalyzer();
+ new TurkishAnalyzer().close();
}
/** test stopwords and stemming */
@@ -41,6 +41,7 @@ public class TestTurkishAnalyzer extends
// apostrophes
checkOneTerm(a, "Kıbrıs'ta", "kıbrıs");
assertAnalyzesTo(a, "Van Gölü'ne", new String[]{"van", "göl"});
+ a.close();
}
/** test use of exclusion set */
@@ -49,10 +50,13 @@ public class TestTurkishAnalyzer extends
Analyzer a = new TurkishAnalyzer(TurkishAnalyzer.getDefaultStopSet(), exclusionSet);
checkOneTerm(a, "aÄacı", "aÄacı");
checkOneTerm(a, "aÄaç", "aÄaç");
+ a.close();
}
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
- checkRandomData(random(), new TurkishAnalyzer(), 1000*RANDOM_MULTIPLIER);
+ Analyzer analyzer = new TurkishAnalyzer();
+ checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
+ analyzer.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java Tue Mar 31 05:22:40 2015
@@ -18,12 +18,9 @@ package org.apache.lucene.analysis.tr;
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -80,5 +77,6 @@ public class TestTurkishLowerCaseFilter
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Tue Mar 31 05:22:40 2015
@@ -141,6 +141,7 @@ public class TestCharTokenizers extends
}
// just for fun
checkRandomData(random(), analyzer, num);
+ analyzer.close();
}
// LUCENE-3642: normalize BMP->SMP and check that offsets are correct
@@ -179,5 +180,6 @@ public class TestCharTokenizers extends
}
// just for fun
checkRandomData(random(), analyzer, num);
+ analyzer.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java Tue Mar 31 05:22:40 2015
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.util;
*/
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@@ -71,6 +70,7 @@ public class TestElision extends BaseTok
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java Tue Mar 31 05:22:40 2015
@@ -31,7 +31,7 @@ public class TestFilesystemResourceLoade
private void assertNotFound(ResourceLoader rl) throws Exception {
try {
- IOUtils.closeWhileHandlingException(rl.openResource("/this-directory-really-really-really-should-not-exist/foo/bar.txt"));
+ IOUtils.closeWhileHandlingException(rl.openResource("this-directory-really-really-really-should-not-exist/foo/bar.txt"));
fail("The resource does not exist, should fail!");
} catch (IOException ioe) {
// pass
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java Tue Mar 31 05:22:40 2015
@@ -27,22 +27,35 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.IOUtils;
/** Basic tests for {@link SegmentingTokenizerBase} */
public class TestSegmentingTokenizerBase extends BaseTokenStreamTestCase {
- private Analyzer sentence = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new WholeSentenceTokenizer());
- }
- };
+ private Analyzer sentence, sentenceAndWord;
- private Analyzer sentenceAndWord = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new SentenceAndWordTokenizer());
- }
- };
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ sentence = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new WholeSentenceTokenizer());
+ }
+ };
+ sentenceAndWord = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SentenceAndWordTokenizer());
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(sentence, sentenceAndWord);
+ super.tearDown();
+ }
+
/** Some simple examples, just outputting the whole sentence boundaries as "terms" */
public void testBasics() throws IOException {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java Tue Mar 31 05:22:40 2015
@@ -194,6 +194,7 @@ public class WikipediaTokenizerTest exte
};
// TODO: properly support positionLengthAttribute
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false);
+ a.close();
}
/** blast some random large strings through the analyzer */
@@ -209,5 +210,6 @@ public class WikipediaTokenizerTest exte
};
// TODO: properly support positionLengthAttribute
checkRandomData(random, a, 100*RANDOM_MULTIPLIER, 8192, false, false);
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java Tue Mar 31 05:22:40 2015
@@ -27,7 +27,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocValuesRangeQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
Modified: lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -17,7 +17,6 @@ package org.apache.lucene.collation;
* limitations under the License.
*/
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CollationTestBase;
import org.apache.lucene.util.BytesRef;
@@ -25,17 +24,24 @@ import org.apache.lucene.util.BytesRef;
import java.text.Collator;
import java.util.Locale;
-public class TestCollationKeyAnalyzer extends CollationTestBase {
- // the sort order of à versus U depends on the version of the rules being used
- // for the inherited root locale: Ã's order isnt specified in Locale.US since
- // it's not used in english.
- private boolean oStrokeFirst = Collator.getInstance(new Locale("")).compare("Ã", "U") < 0;
-
+public class TestCollationKeyAnalyzer extends CollationTestBase {
// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
// RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
// characters properly.
private Collator collator = Collator.getInstance(new Locale("ar"));
- private Analyzer analyzer = new CollationKeyAnalyzer(collator);
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new CollationKeyAnalyzer(collator);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
private BytesRef firstRangeBeginning = new BytesRef(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());
private BytesRef firstRangeEnd = new BytesRef(collator.getCollationKey(firstRangeEndOriginal).toByteArray());
@@ -65,7 +71,9 @@ public class TestCollationKeyAnalyzer ex
for (int i = 0; i < iters; i++) {
Collator collator = Collator.getInstance(Locale.GERMAN);
collator.setStrength(Collator.PRIMARY);
- assertThreadSafe(new CollationKeyAnalyzer(collator));
+ Analyzer analyzer = new CollationKeyAnalyzer(collator);
+ assertThreadSafe(analyzer);
+ analyzer.close();
}
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java Tue Mar 31 05:22:40 2015
@@ -18,22 +18,37 @@ package org.apache.lucene.analysis.icu;
*/
import java.io.IOException;
-import java.io.Reader;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
/**
* Tests ICUFoldingFilter
*/
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
- Analyzer a = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
- }
- };
+ Analyzer a;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ a = new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ a.close();
+ super.tearDown();
+ }
+
public void testDefaults() throws IOException {
// case folding
assertAnalyzesTo(a, "This is a test", new String[] { "this", "is", "a", "test" });
@@ -88,5 +103,6 @@ public class TestICUFoldingFilter extend
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2CharFilter.java Tue Mar 31 05:22:40 2015
@@ -129,6 +129,7 @@ public class TestICUNormalizer2CharFilte
}
checkOneTerm(a, input, normalized);
}
+ a.close();
}
public void testNFC() throws Exception {
@@ -187,6 +188,7 @@ public class TestICUNormalizer2CharFilte
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
// huge strings
checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192);
+ a.close();
// nfkd
a = new Analyzer() {
@@ -203,6 +205,7 @@ public class TestICUNormalizer2CharFilte
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
// huge strings
checkRandomData(random(), a, 25*RANDOM_MULTIPLIER, 8192);
+ a.close();
}
public void testCuriousString() throws Exception {
@@ -221,6 +224,7 @@ public class TestICUNormalizer2CharFilte
for (int i = 0; i < 1000; i++) {
checkAnalysisConsistency(random(), a, false, text);
}
+ a.close();
}
public void testCuriousMassiveString() throws Exception {
@@ -411,5 +415,6 @@ public class TestICUNormalizer2CharFilte
for (int i = 0; i < 25; i++) {
checkAnalysisConsistency(random(), a, false, text);
}
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java Tue Mar 31 05:22:40 2015
@@ -18,9 +18,11 @@ package org.apache.lucene.analysis.icu;
*/
import java.io.IOException;
-import java.io.Reader;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import com.ibm.icu.text.Normalizer2;
@@ -29,13 +31,25 @@ import com.ibm.icu.text.Normalizer2;
* Tests the ICUNormalizer2Filter
*/
public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
- Analyzer a = new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
- }
- };
+ Analyzer a;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ a = new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ a.close();
+ super.tearDown();
+ }
public void testDefaults() throws IOException {
// case folding
@@ -72,6 +86,7 @@ public class TestICUNormalizer2Filter ex
// decompose EAcute into E + combining Acute
assertAnalyzesTo(a, "\u00E9", new String[] { "\u0065\u0301" });
+ a.close();
}
/** blast some random strings through the analyzer */
@@ -88,5 +103,6 @@ public class TestICUNormalizer2Filter ex
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java Tue Mar 31 05:22:40 2015
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.icu;
*/
import java.io.IOException;
-import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
@@ -105,6 +104,7 @@ public class TestICUTransformFilter exte
}
};
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
+ a.close();
}
public void testEmptyTerm() throws IOException {
@@ -116,5 +116,6 @@ public class TestICUTransformFilter exte
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Tue Mar 31 05:22:40 2015
@@ -67,14 +67,26 @@ public class TestICUTokenizer extends Ba
assertTokenStreamContents(tokenizer, expected);
}
- private Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
- TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
- return new TokenStreamComponents(tokenizer, filter);
- }
- };
+ private Analyzer a;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
+ TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
+ return new TokenStreamComponents(tokenizer, filter);
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ a.close();
+ super.tearDown();
+ }
public void testArmenian() throws Exception {
assertAnalyzesTo(a, "ÕÕ«ÖÕ«ÕºÕ¥Õ¤Õ«Õ¡ÕµÕ« 13 Õ´Õ«Õ¬Õ«Õ¸Õ¶ Õ°Õ¸Õ¤Õ¾Õ¡Õ®Õ¶Õ¥ÖÕ¨ (4,600` Õ°Õ¡ÕµÕ¥ÖÕ¥Õ¶ Õ¾Õ«ÖÕ«ÕºÕ¥Õ¤Õ«Õ¡ÕµÕ¸ÖÕ´) Õ£ÖÕ¾Õ¥Õ¬ Õ¥Õ¶ Õ¯Õ¡Õ´Õ¡Õ¾Õ¸ÖÕ¶Õ¥ÖÕ« Õ¯Õ¸Õ²Õ´Õ«Ö Õ¸Ö Õ°Õ¡Õ´Õ¡ÖÕµÕ¡ Õ¢Õ¸Õ¬Õ¸Ö Õ°Õ¸Õ¤Õ¾Õ¡Õ®Õ¶Õ¥ÖÕ¨ Õ¯Õ¡ÖÕ¸Õ² Õ§ ÕÕ´Õ¢Õ¡Õ£ÖÕ¥Õ¬ ÖÕ¡Õ¶Õ¯Õ¡Ö Õ´Õ¡ÖÕ¤ Õ¸Õ¾ Õ¯Õ¡ÖÕ¸Õ² Õ§ Õ¢Õ¡ÖÕ¥Õ¬ ÕÕ«ÖÕ«ÕºÕ¥Õ¤Õ«Õ¡ÕµÕ« Õ¯Õ¡ÕµÖÕ¨Ö",
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java Tue Mar 31 05:22:40 2015
@@ -17,7 +17,6 @@ package org.apache.lucene.analysis.icu.s
* limitations under the License.
*/
-import java.io.Reader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -27,12 +26,24 @@ import org.apache.lucene.analysis.BaseTo
* test ICUTokenizer with dictionary-based CJ segmentation
*/
public class TestICUTokenizerCJK extends BaseTokenStreamTestCase {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true)));
- }
- };
+ Analyzer a;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(true)));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ a.close();
+ super.tearDown();
+ }
/**
* test stolen from smartcn
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java Tue Mar 31 05:22:40 2015
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.cjk.CJ
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.icu.ICUNormalizer2Filter;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.IOUtils;
/**
* Tests ICUTokenizer's ability to work with CJKBigramFilter.
@@ -34,35 +35,46 @@ import org.apache.lucene.analysis.util.C
*/
public class TestWithCJKBigramFilter extends BaseTokenStreamTestCase {
- /**
- * ICUTokenizer+CJKBigramFilter
- */
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
- TokenStream result = new CJKBigramFilter(source);
- return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
- }
- };
+ Analyzer analyzer, analyzer2;
- /**
- * ICUTokenizer+ICUNormalizer2Filter+CJKBigramFilter.
- *
- * ICUNormalizer2Filter uses nfkc_casefold by default, so this is a language-independent
- * superset of CJKWidthFilter's foldings.
- */
- private Analyzer analyzer2 = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
- // we put this before the CJKBigramFilter, because the normalization might combine
- // some halfwidth katakana forms, which will affect the bigramming.
- TokenStream result = new ICUNormalizer2Filter(source);
- result = new CJKBigramFilter(source);
- return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
- }
- };
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ /*
+ * ICUTokenizer+CJKBigramFilter
+ */
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
+ TokenStream result = new CJKBigramFilter(source);
+ return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
+ }
+ };
+ /*
+ * ICUTokenizer+ICUNormalizer2Filter+CJKBigramFilter.
+ *
+ * ICUNormalizer2Filter uses nfkc_casefold by default, so this is a language-independent
+ * superset of CJKWidthFilter's foldings.
+ */
+ analyzer2 = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new ICUTokenizer(newAttributeFactory(), new DefaultICUTokenizerConfig(false));
+ // we put this before the CJKBigramFilter, because the normalization might combine
+ // some halfwidth katakana forms, which will affect the bigramming.
+ TokenStream result = new ICUNormalizer2Filter(source);
+ result = new CJKBigramFilter(result);
+ return new TokenStreamComponents(source, new StopFilter(result, CharArraySet.EMPTY_SET));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(analyzer, analyzer2);
+ super.tearDown();
+ }
public void testJa1() throws IOException {
assertAnalyzesTo(analyzer, "ä¸äºä¸åäºå
ä¸å
«ä¹å",
Modified: lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -29,7 +29,19 @@ import java.util.Locale;
public class TestICUCollationKeyAnalyzer extends CollationTestBase {
private Collator collator = Collator.getInstance(new Locale("fa"));
- private Analyzer analyzer = new ICUCollationKeyAnalyzer(collator);
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new ICUCollationKeyAnalyzer(collator);
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
private BytesRef firstRangeBeginning = new BytesRef
(collator.getCollationKey(firstRangeBeginningOriginal).toByteArray());
@@ -62,7 +74,9 @@ public class TestICUCollationKeyAnalyzer
Locale locale = Locale.GERMAN;
Collator collator = Collator.getInstance(locale);
collator.setStrength(Collator.IDENTICAL);
- assertThreadSafe(new ICUCollationKeyAnalyzer(collator));
+ Analyzer a = new ICUCollationKeyAnalyzer(collator);
+ assertThreadSafe(a);
+ a.close();
}
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Tue Mar 31 05:22:40 2015
@@ -30,14 +30,25 @@ import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.UnicodeUtil;
public class TestExtendedMode extends BaseTokenStreamTestCase {
- private final Analyzer analyzer = new Analyzer() {
-
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.EXTENDED);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
/** simple test for supplementary characters */
public void testSurrogates() throws IOException {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -31,7 +31,7 @@ public class TestJapaneseAnalyzer extend
/** This test fails with NPE when the
* stopwords file is missing in classpath */
public void testResourcesAvailable() {
- new JapaneseAnalyzer();
+ new JapaneseAnalyzer().close();
}
/**
@@ -40,12 +40,14 @@ public class TestJapaneseAnalyzer extend
* and offsets are correct.
*/
public void testBasics() throws IOException {
- assertAnalyzesTo(new JapaneseAnalyzer(), "å¤ãã®å¦çã試é¨ã«è½ã¡ãã",
+ Analyzer a = new JapaneseAnalyzer();
+ assertAnalyzesTo(a, "å¤ãã®å¦çã試é¨ã«è½ã¡ãã",
new String[] { "å¤ã", "å¦ç", "試é¨", "è½ã¡ã" },
new int[] { 0, 3, 6, 9 },
new int[] { 2, 5, 8, 11 },
new int[] { 1, 2, 2, 2 }
);
+ a.close();
}
/**
@@ -53,7 +55,7 @@ public class TestJapaneseAnalyzer extend
*/
public void testDecomposition() throws IOException {
- final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH,
+ Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH,
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
@@ -108,7 +110,9 @@ public class TestJapaneseAnalyzer extend
);
// Kyoto University Baseball Club
- assertAnalyzesToPositions(new JapaneseAnalyzer(), "京é½å¤§å¦ç¡¬å¼éçé¨",
+ a.close();
+ a = new JapaneseAnalyzer();
+ assertAnalyzesToPositions(a, "京é½å¤§å¦ç¡¬å¼éçé¨",
new String[] { "京é½å¤§",
"å¦",
"硬å¼",
@@ -117,6 +121,7 @@ public class TestJapaneseAnalyzer extend
new int[] {1, 1, 1, 1, 1},
new int[] {1, 1, 1, 1, 1});
// toDotFile(a, "æç°ç©ºæ¸¯", "/mnt/scratch/out.dot");
+ a.close();
}
@@ -129,6 +134,7 @@ public class TestJapaneseAnalyzer extend
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
checkRandomData(random, a, atLeast(1000));
+ a.close();
}
/** blast some random large strings through the analyzer */
@@ -138,6 +144,7 @@ public class TestJapaneseAnalyzer extend
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
checkRandomData(random, a, 2*RANDOM_MULTIPLIER, 8192);
+ a.close();
}
// Copied from TestJapaneseTokenizer, to make sure passing
@@ -154,6 +161,7 @@ public class TestJapaneseAnalyzer extend
new int[] { 1, 2, 4 },
new Integer(4)
);
+ a.close();
}
// LUCENE-3897: this string (found by running all jawiki
@@ -165,6 +173,7 @@ public class TestJapaneseAnalyzer extend
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
checkAnalysisConsistency(random, a, random.nextBoolean(), s);
+ a.close();
}
// LUCENE-3897: this string (found by
@@ -176,6 +185,7 @@ public class TestJapaneseAnalyzer extend
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
checkAnalysisConsistency(random, a, random.nextBoolean(), s);
+ a.close();
}
// LUCENE-3897: this string (found by
@@ -187,6 +197,7 @@ public class TestJapaneseAnalyzer extend
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
checkAnalysisConsistency(random, a, random.nextBoolean(), s);
+ a.close();
}
public void test4thCuriousString() throws Exception {
@@ -194,8 +205,8 @@ public class TestJapaneseAnalyzer extend
final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH,
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
- Random random = random();
- checkAnalysisConsistency(random, a, true, s);
+ checkAnalysisConsistency(random(), a, true, s);
+ a.close();
}
public void test5thCuriousString() throws Exception {
@@ -203,7 +214,7 @@ public class TestJapaneseAnalyzer extend
final Analyzer a = new JapaneseAnalyzer(null, Mode.SEARCH,
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
- Random random = random();
- checkAnalysisConsistency(random, a, false, s);
+ checkAnalysisConsistency(random(), a, false, s);
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java Tue Mar 31 05:22:40 2015
@@ -28,13 +28,25 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.util.CharArraySet;
public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
- return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
- }
- };
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.DEFAULT_MODE);
+ return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
public void testBasics() throws IOException {
assertAnalyzesTo(analyzer, "ããã¯ã¾ã å®é¨æ®µéã«ããã¾ã",
@@ -55,6 +67,7 @@ public class TestJapaneseBaseFormFilter
assertAnalyzesTo(a, "ããã¯ã¾ã å®é¨æ®µéã«ããã¾ã",
new String[] { "ãã", "ã¯", "ã¾ã ", "å®é¨", "段é", "ã«", "ãã", "ã¾ã" }
);
+ a.close();
}
public void testEnglish() throws IOException {
@@ -75,5 +88,6 @@ public class TestJapaneseBaseFormFilter
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java Tue Mar 31 05:22:40 2015
@@ -22,38 +22,49 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.IOUtils;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
public class TestJapaneseIterationMarkCharFilter extends BaseTokenStreamTestCase {
-
- private Analyzer keywordAnalyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new JapaneseIterationMarkCharFilter(reader);
- }
- };
-
- private Analyzer japaneseAnalyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return new JapaneseIterationMarkCharFilter(reader);
- }
- };
+ private Analyzer keywordAnalyzer, japaneseAnalyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ keywordAnalyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new JapaneseIterationMarkCharFilter(reader);
+ }
+ };
+ japaneseAnalyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new JapaneseIterationMarkCharFilter(reader);
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(keywordAnalyzer, japaneseAnalyzer);
+ super.tearDown();
+ }
public void testKanji() throws IOException {
// Test single repetition
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java Tue Mar 31 05:22:40 2015
@@ -32,14 +32,26 @@ import java.io.IOException;
* Tests for {@link JapaneseKatakanaStemFilter}
*/
public class TestJapaneseKatakanaStemFilter extends BaseTokenStreamTestCase {
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- // Use a MockTokenizer here since this filter doesn't really depend on Kuromoji
- Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(source));
- }
- };
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ // Use a MockTokenizer here since this filter doesn't really depend on Kuromoji
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(source));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
/**
* Test a few common katakana spelling variations.
@@ -73,6 +85,7 @@ public class TestJapaneseKatakanaStemFil
}
};
checkOneTerm(a, "ã³ã¼ãã¼", "ã³ã¼ãã¼");
+ a.close();
}
public void testUnsupportedHalfWidthVariants() throws IOException {
@@ -93,5 +106,6 @@ public class TestJapaneseKatakanaStemFil
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java Tue Mar 31 05:22:40 2015
@@ -36,15 +36,26 @@ import org.junit.Ignore;
import org.junit.Test;
public class TestJapaneseNumberFilter extends BaseTokenStreamTestCase {
-
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, new JapaneseNumberFilter(tokenizer));
- }
- };
-
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, new JapaneseNumberFilter(tokenizer));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
+
@Test
public void testBasics() throws IOException {
@@ -188,6 +199,7 @@ public class TestJapaneseNumberFilter ex
new int[]{2, 4},
new int[]{1, 1}
);
+ keywordMarkingAnalyzer.close();
}
@Test
@@ -285,6 +297,7 @@ public class TestJapaneseNumberFilter ex
Files.newBufferedReader(input, StandardCharsets.UTF_8),
Files.newBufferedWriter(normalizedOutput, StandardCharsets.UTF_8)
);
+ plainAnalyzer.close();
}
public void analyze(Analyzer analyzer, Reader reader, Writer writer) throws IOException {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java Tue Mar 31 05:22:40 2015
@@ -23,31 +23,41 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.cjk.CJKWidthFilter;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.util.IOUtils;
import java.io.IOException;
-import java.io.Reader;
import java.util.Random;
/**
* Tests for {@link TestJapaneseReadingFormFilter}
*/
public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase {
- private Analyzer katakanaAnalyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false));
- }
- };
-
- private Analyzer romajiAnalyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true));
- }
- };
-
+ private Analyzer katakanaAnalyzer, romajiAnalyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ katakanaAnalyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false));
+ }
+ };
+ romajiAnalyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, JapaneseTokenizer.Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true));
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(katakanaAnalyzer, romajiAnalyzer);
+ super.tearDown();
+ }
public void testKatakanaReadings() throws IOException {
assertAnalyzesTo(katakanaAnalyzer, "ä»å¤ã¯ããã¼ãå
çã¨è©±ãã",
@@ -67,6 +77,7 @@ public class TestJapaneseReadingFormFilt
assertAnalyzesTo(a, "ä»å¤ã¯ï¾ï¾ï¾ï½°ï¾å
çã¨è©±ãã",
new String[] { "ã³ã³ã¤", "ã", "ããã¼ã", "ã»ã³ã»ã¤", "ã", "ããã·", "ã¿" }
);
+ a.close();
}
public void testRomajiReadings() throws IOException {
@@ -87,6 +98,7 @@ public class TestJapaneseReadingFormFilt
assertAnalyzesTo(a, "ä»å¤ã¯ï¾ï¾ï¾ï½°ï¾å
çã¨è©±ãã",
new String[] { "kon'ya", "ha", "robato", "sensei", "to", "hanashi", "ta" }
);
+ a.close();
}
public void testRandomData() throws IOException {
@@ -104,5 +116,6 @@ public class TestJapaneseReadingFormFilt
}
};
checkOneTerm(a, "", "");
+ a.close();
}
}
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java Tue Mar 31 05:22:40 2015
@@ -35,9 +35,9 @@ import org.apache.lucene.analysis.ja.dic
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.ja.tokenattributes.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.LuceneTestCase.Slow;
public class TestJapaneseTokenizer extends BaseTokenStreamTestCase {
@@ -57,38 +57,47 @@ public class TestJapaneseTokenizer exten
throw new RuntimeException(ioe);
}
}
-
- private Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
-
- private Analyzer analyzerNormal = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
-
- private Analyzer analyzerNoPunct = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
-
- private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
+
+ private Analyzer analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ analyzerNormal = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.NORMAL);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ analyzerNoPunct = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ extendedModeAnalyzerNoPunct = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), true, Mode.EXTENDED);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ IOUtils.close(analyzer, analyzerNormal, analyzerNoPunct, extendedModeAnalyzerNoPunct);
+ super.tearDown();
+ }
public void testNormalMode() throws Exception {
assertAnalyzesTo(analyzerNormal,
@@ -197,16 +206,16 @@ public class TestJapaneseTokenizer exten
public void testRandomHugeStringsMockGraphAfter() throws Exception {
// Randomly inject graph tokens after JapaneseTokenizer:
Random random = random();
- checkRandomData(random,
- new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
- TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
- return new TokenStreamComponents(tokenizer, graph);
- }
- },
- 20*RANDOM_MULTIPLIER, 8192);
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), readDict(), false, Mode.SEARCH);
+ TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
+ return new TokenStreamComponents(tokenizer, graph);
+ }
+ };
+ checkRandomData(random, analyzer, 20*RANDOM_MULTIPLIER, 8192);
+ analyzer.close();
}
public void testLargeDocReliability() throws Exception {
@@ -367,6 +376,7 @@ public class TestJapaneseTokenizer exten
surfaceForms);
assertTrue(gv2.finish().indexOf("22.0") != -1);
+ analyzer.close();
}
private void assertReadings(String input, String... readings) throws IOException {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java Tue Mar 31 05:22:40 2015
@@ -31,13 +31,25 @@ import org.apache.lucene.analysis.ja.Jap
public class TestSearchMode extends BaseTokenStreamTestCase {
private final static String SEGMENTATION_FILENAME = "search-segmentation-tests.txt";
- private final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH);
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
+ private Analyzer analyzer;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, true, Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ analyzer.close();
+ super.tearDown();
+ }
/** Test search mode segmentation */
public void testSearchSegmentation() throws IOException {
Modified: lucene/dev/branches/lucene6271/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java?rev=1670257&r1=1670256&r2=1670257&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java (original)
+++ lucene/dev/branches/lucene6271/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java Tue Mar 31 05:22:40 2015
@@ -46,6 +46,7 @@ public class TestMorfologikAnalyzer exte
assertAnalyzesTo(a, "liÅcie", new String[] { "liÅcie", "liÅÄ", "list", "lista" });
assertAnalyzesTo(a, "danych", new String[] { "dany", "dana", "dane", "daÄ" });
assertAnalyzesTo(a, "ÄóÄ
ÅÅżźÄÅ", new String[] { "ÄóÄ
ÅÅżźÄÅ" });
+ a.close();
}
/** Test stemming of multiple tokens and proper term metrics. */
@@ -66,11 +67,13 @@ public class TestMorfologikAnalyzer exte
new int[] { 0, 0, 3 },
new int[] { 1, 1, 13 },
new int[] { 1, 0, 1 });
+ a.close();
}
@SuppressWarnings("unused")
private void dumpTokens(String input) throws IOException {
- try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", input)) {
+ try (Analyzer a = getTestAnalyzer();
+ TokenStream ts = a.tokenStream("dummy", input)) {
ts.reset();
MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class);
@@ -100,6 +103,7 @@ public class TestMorfologikAnalyzer exte
assertEquals("second stream", "dany", termAtt_2.toString());
ts_2.end();
}
+ a.close();
}
/** Test stemming of mixed-case tokens. */
@@ -116,6 +120,7 @@ public class TestMorfologikAnalyzer exte
assertAnalyzesTo(a, "aarona", new String[] { "aarona" });
assertAnalyzesTo(a, "LiÅcie", new String[] { "liÅcie", "liÅÄ", "list", "lista" });
+ a.close();
}
private void assertPOSToken(TokenStream ts, String term, String... tags) throws IOException {
@@ -140,7 +145,8 @@ public class TestMorfologikAnalyzer exte
/** Test morphosyntactic annotations. */
public final void testPOSAttribute() throws IOException {
- try (TokenStream ts = getTestAnalyzer().tokenStream("dummy", "liÅcie")) {
+ try (Analyzer a = getTestAnalyzer();
+ TokenStream ts = a.tokenStream("dummy", "liÅcie")) {
ts.reset();
assertPOSToken(ts, "liÅcie",
"subst:sg:acc:n2",
@@ -187,10 +193,13 @@ public class TestMorfologikAnalyzer exte
new int[] { 0, 7, 7, 7, 7 },
new int[] { 6, 13, 13, 13, 13 },
new int[] { 1, 1, 0, 0, 0 });
+ a.close();
}
/** blast some random strings through the analyzer */
public void testRandom() throws Exception {
- checkRandomData(random(), getTestAnalyzer(), 1000 * RANDOM_MULTIPLIER);
+ Analyzer a = getTestAnalyzer();
+ checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
+ a.close();
}
}