You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/07/16 19:00:18 UTC
svn commit: r1503797 [3/19] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/suggest/
dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/ dev-tools/...
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -18,13 +18,10 @@ package org.apache.lucene.analysis.misce
*/
import java.io.IOException;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -47,14 +44,14 @@ public class TestLimitTokenCountAnalyzer
Analyzer a = new LimitTokenCountAnalyzer(mock, 2, consumeAll);
// dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
// less than the limit, ensure we behave correctly
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 "), new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
// equal to limit
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 "), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java Tue Jul 16 17:00:05 2013
@@ -43,17 +43,17 @@ public class TestLimitTokenPositionFilte
};
// dont use assertAnalyzesTo here, as the end offset is not the end of the string (unless consumeAll is true, in which case its correct)!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 3 4 5"),
new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 16 : null);
assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")),
new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, consumeAll ? 9 : null);
// less than the limit, ensure we behave correctly
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 ")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 "),
new String[] { "1" }, new int[] { 0 }, new int[] { 1 }, consumeAll ? 3 : null);
// equal to limit
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 ")),
+ assertTokenStreamContents(a.tokenStream("dummy", "1 2 "),
new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, consumeAll ? 6 : null);
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Tue Jul 16 17:00:05 2013
@@ -1,7 +1,6 @@
package org.apache.lucene.analysis.miscellaneous;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -38,8 +37,7 @@ public class TestPerFieldAnalyzerWrapper
PerFieldAnalyzerWrapper analyzer =
new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
- TokenStream tokenStream = analyzer.tokenStream("field",
- new StringReader(text));
+ TokenStream tokenStream = analyzer.tokenStream("field", text);
CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
tokenStream.reset();
@@ -48,8 +46,7 @@ public class TestPerFieldAnalyzerWrapper
"Qwerty",
termAtt.toString());
- tokenStream = analyzer.tokenStream("special",
- new StringReader(text));
+ tokenStream = analyzer.tokenStream("special", text);
termAtt = tokenStream.getAttribute(CharTermAttribute.class);
tokenStream.reset();
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java Tue Jul 16 17:00:05 2013
@@ -32,6 +32,9 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK;
+
/**
* Simple tests for {@link NorwegianLightStemFilter}
@@ -42,7 +45,7 @@ public class TestNorwegianLightStemFilte
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new NorwegianLightStemFilter(source));
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL));
}
};
@@ -51,6 +54,18 @@ public class TestNorwegianLightStemFilte
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_light.txt")));
}
+ /** Test against a Nynorsk vocabulary file */
+ public void testNynorskVocabulary() throws IOException {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NYNORSK));
+ }
+ };
+ assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_light.txt")));
+ }
+
public void testKeyword() throws IOException {
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java Tue Jul 16 17:00:05 2013
@@ -34,7 +34,23 @@ public class TestNorwegianLightStemFilte
stream = tokenFilterFactory("NorwegianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl" });
}
-
+
+ /** Test stemming with variant set explicitly to Bokmål */
+ public void testBokmaalStemming() throws Exception {
+ Reader reader = new StringReader("epler eple");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nb").create(stream);
+ assertTokenStreamContents(stream, new String[] { "epl", "epl" });
+ }
+
+ /** Test stemming with variant set explicitly to Nynorsk */
+ public void testNynorskStemming() throws Exception {
+ Reader reader = new StringReader("gutar gutane");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianLightStem", "variant", "nn").create(stream);
+ assertTokenStreamContents(stream, new String[] { "gut", "gut" });
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java Tue Jul 16 17:00:05 2013
@@ -32,6 +32,8 @@ import org.apache.lucene.analysis.miscel
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.BOKMAAL;
+import static org.apache.lucene.analysis.no.NorwegianLightStemmer.NYNORSK;
/**
* Simple tests for {@link NorwegianMinimalStemFilter}
@@ -42,15 +44,27 @@ public class TestNorwegianMinimalStemFil
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source));
+ return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, BOKMAAL));
}
};
- /** Test against a vocabulary file */
+ /** Test against a Bokmål vocabulary file */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_minimal.txt")));
}
+ /** Test against a Nynorsk vocabulary file */
+ public void testNynorskVocabulary() throws IOException {
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, NYNORSK));
+ }
+ };
+ assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_minimal.txt")));
+ }
+
public void testKeyword() throws IOException {
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java Tue Jul 16 17:00:05 2013
@@ -35,6 +35,22 @@ public class TestNorwegianMinimalStemFil
assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
}
+ /** Test stemming with variant set explicitly to Bokmål */
+ public void testBokmaalStemming() throws Exception {
+ Reader reader = new StringReader("eple eplet epler eplene eplets eplenes");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nb").create(stream);
+ assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
+ }
+
+ /** Test stemming with variant set explicitly to Nynorsk */
+ public void testNynorskStemming() throws Exception {
+ Reader reader = new StringReader("gut guten gutar gutane gutens gutanes");
+ TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nn").create(stream);
+ assertTokenStreamContents(stream, new String[] { "gut", "gut", "gut", "gut", "gut", "gut" });
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
try {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Tue Jul 16 17:00:05 2013
@@ -26,7 +26,6 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.RAMDirectory;
-import java.io.StringReader;
import java.util.Arrays;
import java.util.Collections;
@@ -66,44 +65,44 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoStopwords() throws Exception {
// Note: an empty list of fields passed in
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Collections.<String>emptyList(), 1);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("quick"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("variedField", "quick");
assertTokenStreamContents(protectedTokenStream, new String[]{"quick"});
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
public void testDefaultStopwordsAllFields() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
assertTokenStreamContents(protectedTokenStream, new String[0]); // Default stop word filtering will remove boring
}
public void testStopwordsAllFieldsMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on terms in > one half of docs remove boring
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring");
// A filter on terms in > half of docs should not remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[]{"vaguelyboring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, 1f / 4f);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("vaguelyboring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "vaguelyboring");
// A filter on terms in > quarter of docs should remove vaguelyBoring
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
public void testStopwordsPerFieldMaxPercentDocs() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField"), 1f / 2f);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on one Field should not affect queries on another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("variedField", "repetitiveField"), 1f / 2f);
- protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// A filter on the right Field should affect queries on it
assertTokenStreamContents(protectedTokenStream, new String[0]);
}
@@ -121,11 +120,11 @@ public class QueryAutoStopWordAnalyzerTe
public void testNoFieldNamePollution() throws Exception {
protectedAnalyzer = new QueryAutoStopWordAnalyzer(TEST_VERSION_CURRENT, appAnalyzer, reader, Arrays.asList("repetitiveField"), 10);
- TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", new StringReader("boring"));
+ TokenStream protectedTokenStream = protectedAnalyzer.tokenStream("repetitiveField", "boring");
// Check filter set up OK
assertTokenStreamContents(protectedTokenStream, new String[0]);
- protectedTokenStream = protectedAnalyzer.tokenStream("variedField", new StringReader("boring"));
+ protectedTokenStream = protectedAnalyzer.tokenStream("variedField", "boring");
// Filter should not prevent stopwords in one field being used in another
assertTokenStreamContents(protectedTokenStream, new String[]{"boring"});
}
@@ -134,7 +133,7 @@ public class QueryAutoStopWordAnalyzerTe
QueryAutoStopWordAnalyzer a = new QueryAutoStopWordAnalyzer(
TEST_VERSION_CURRENT,
new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false), reader, 10);
- TokenStream ts = a.tokenStream("repetitiveField", new StringReader("this boring"));
+ TokenStream ts = a.tokenStream("repetitiveField", "this boring");
assertTokenStreamContents(ts, new String[] { "this" });
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Tue Jul 16 17:00:05 2013
@@ -17,8 +17,6 @@ package org.apache.lucene.analysis.shing
* limitations under the License.
*/
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
@@ -97,7 +95,7 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperPhraseQuery() throws Exception {
PhraseQuery q = new PhraseQuery();
- TokenStream ts = analyzer.tokenStream("content", new StringReader("this sentence"));
+ TokenStream ts = analyzer.tokenStream("content", "this sentence");
int j = -1;
PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
@@ -123,7 +121,7 @@ public class ShingleAnalyzerWrapperTest
public void testShingleAnalyzerWrapperBooleanQuery() throws Exception {
BooleanQuery q = new BooleanQuery();
- TokenStream ts = analyzer.tokenStream("content", new StringReader("test sentence"));
+ TokenStream ts = analyzer.tokenStream("content", "test sentence");
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java Tue Jul 16 17:00:05 2013
@@ -91,7 +91,7 @@ public class TestTeeSinkTokenFilter exte
Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
- TokenStream tokenStream = analyzer.tokenStream("field", new StringReader("abcd "));
+ TokenStream tokenStream = analyzer.tokenStream("field", "abcd ");
TeeSinkTokenFilter tee = new TeeSinkTokenFilter(tokenStream);
TokenStream sink = tee.newSinkTokenStream();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.th;
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -30,7 +29,6 @@ import org.apache.lucene.analysis.core.K
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.util.Version;
/**
* Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer
@@ -122,10 +120,10 @@ public class TestThaiAnalyzer extends Ba
public void testAttributeReuse() throws Exception {
ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
// just consume
- TokenStream ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ TokenStream ts = analyzer.tokenStream("dummy", "ภาษาà¹à¸à¸¢");
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
// this consumer adds flagsAtt, which this analyzer does not use.
- ts = analyzer.tokenStream("dummy", new StringReader("ภาษาà¹à¸à¸¢"));
+ ts = analyzer.tokenStream("dummy", "ภาษาà¹à¸à¸¢");
ts.addAttribute(FlagsAttribute.class);
assertTokenStreamContents(ts, new String[] { "ภาษา", "à¹à¸à¸¢" });
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java Tue Jul 16 17:00:05 2013
@@ -123,7 +123,7 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
ts.reset();
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
while (ts.incrementToken()) {
@@ -161,7 +161,7 @@ public class TestCharTokenizers extends
int num = 1000 * RANDOM_MULTIPLIER;
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomUnicodeString(random());
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
ts.reset();
OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java Tue Jul 16 17:00:05 2013
@@ -249,7 +249,7 @@ public class TestICUTokenizer extends Ba
}
public void testTokenAttributes() throws Exception {
- TokenStream ts = a.tokenStream("dummy", new StringReader("This is a test"));
+ TokenStream ts = a.tokenStream("dummy", "This is a test");
ScriptAttribute scriptAtt = ts.addAttribute(ScriptAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilterFactory.java Tue Jul 16 17:00:05 2013
@@ -24,7 +24,7 @@ import org.apache.lucene.analysis.util.T
import java.util.Map;
/**
- * Factory for {@link JapaneseKatakanaStemFilterFactory}.
+ * Factory for {@link JapaneseKatakanaStemFilter}.
* <pre class="prettyprint">
* <fieldType name="text_ja" class="solr.TextField">
* <analyzer>
Modified: lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java Tue Jul 16 17:00:05 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.ja;
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -54,7 +53,7 @@ public class TestExtendedMode extends Ba
int numIterations = atLeast(1000);
for (int i = 0; i < numIterations; i++) {
String s = _TestUtil.randomUnicodeString(random(), 100);
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ja;
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -151,7 +150,7 @@ public class TestJapaneseAnalyzer extend
Mode.SEARCH,
JapaneseAnalyzer.getDefaultStopSet(),
JapaneseAnalyzer.getDefaultStopTags());
- assertTokenStreamContents(a.tokenStream("foo", new StringReader("abcd")),
+ assertTokenStreamContents(a.tokenStream("foo", "abcd"),
new String[] { "a", "b", "cd" },
new int[] { 0, 1, 2 },
new int[] { 1, 2, 4 },
Modified: lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java Tue Jul 16 17:00:05 2013
@@ -22,7 +22,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -142,7 +141,7 @@ public class TestJapaneseTokenizer exten
* ideally the test would actually fail instead of hanging...
*/
public void testDecomposition5() throws Exception {
- TokenStream ts = analyzer.tokenStream("bogus", new StringReader("ãããããããããããããããããããããããããããããããããããããããã"));
+ TokenStream ts = analyzer.tokenStream("bogus", "ãããããããããããããããããããããããããããããããããããããããã");
ts.reset();
while (ts.incrementToken()) {
@@ -166,8 +165,8 @@ public class TestJapaneseTokenizer exten
/** Tests that sentence offset is incorporated into the resulting offsets */
public void testTwoSentences() throws Exception {
/*
- //TokenStream ts = a.tokenStream("foo", new StringReader("妹ã®å²åã§ãã俺ã¨å¹´åã§ãä»åé¨çã§ãã"));
- TokenStream ts = analyzer.tokenStream("foo", new StringReader("�<!--\"<!--#<!--;?><!--#<!--#><!---->?>-->;"));
+ //TokenStream ts = a.tokenStream("foo", "妹ã®å²åã§ãã俺ã¨å¹´åã§ãä»åé¨çã§ãã");
+ TokenStream ts = analyzer.tokenStream("foo", "�<!--\"<!--#<!--;?><!--#<!--#><!---->?>-->;");
ts.reset();
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
while(ts.incrementToken()) {
@@ -214,7 +213,7 @@ public class TestJapaneseTokenizer exten
public void testLargeDocReliability() throws Exception {
for (int i = 0; i < 100; i++) {
String s = _TestUtil.randomUnicodeString(random(), 10000);
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
ts.reset();
while (ts.incrementToken()) {
}
@@ -235,7 +234,7 @@ public class TestJapaneseTokenizer exten
System.out.println("\nTEST: iter=" + i);
}
String s = _TestUtil.randomUnicodeString(random(), 100);
- TokenStream ts = analyzer.tokenStream("foo", new StringReader(s));
+ TokenStream ts = analyzer.tokenStream("foo", s);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
@@ -245,14 +244,14 @@ public class TestJapaneseTokenizer exten
}
public void testOnlyPunctuation() throws IOException {
- TokenStream ts = analyzerNoPunct.tokenStream("foo", new StringReader("ãããã"));
+ TokenStream ts = analyzerNoPunct.tokenStream("foo", "ãããã");
ts.reset();
assertFalse(ts.incrementToken());
ts.end();
}
public void testOnlyPunctuationExtended() throws IOException {
- TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", new StringReader("......"));
+ TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", "......");
ts.reset();
assertFalse(ts.incrementToken());
ts.end();
@@ -261,14 +260,14 @@ public class TestJapaneseTokenizer exten
// note: test is kinda silly since kuromoji emits punctuation tokens.
// but, when/if we filter these out it will be useful.
public void testEnd() throws Exception {
- assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã")),
+ assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", "ããã¯æ¬ã§ã¯ãªã"),
new String[] { "ãã", "ã¯", "æ¬", "ã§", "ã¯", "ãªã" },
new int[] { 0, 2, 3, 4, 5, 6 },
new int[] { 2, 3, 4, 5, 6, 8 },
new Integer(8)
);
- assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã ")),
+ assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", "ããã¯æ¬ã§ã¯ãªã "),
new String[] { "ãã", "ã¯", "æ¬", "ã§", "ã¯", "ãªã" },
new int[] { 0, 2, 3, 4, 5, 6, 8 },
new int[] { 2, 3, 4, 5, 6, 8, 9 },
@@ -279,7 +278,7 @@ public class TestJapaneseTokenizer exten
public void testUserDict() throws Exception {
// Not a great test because w/o userdict.txt the
// segmentation is the same:
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("é¢è¥¿å½é空港ã«è¡ã£ã")),
+ assertTokenStreamContents(analyzer.tokenStream("foo", "é¢è¥¿å½é空港ã«è¡ã£ã"),
new String[] { "é¢è¥¿", "å½é", "空港", "ã«", "è¡ã£", "ã" },
new int[] { 0, 2, 4, 6, 7, 9 },
new int[] { 2, 4, 6, 7, 9, 10 },
@@ -289,7 +288,7 @@ public class TestJapaneseTokenizer exten
public void testUserDict2() throws Exception {
// Better test: w/o userdict the segmentation is different:
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("æéé¾")),
+ assertTokenStreamContents(analyzer.tokenStream("foo", "æéé¾"),
new String[] { "æéé¾" },
new int[] { 0 },
new int[] { 3 },
@@ -299,7 +298,7 @@ public class TestJapaneseTokenizer exten
public void testUserDict3() throws Exception {
// Test entry that breaks into multiple tokens:
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("abcd")),
+ assertTokenStreamContents(analyzer.tokenStream("foo", "abcd"),
new String[] { "a", "b", "cd" },
new int[] { 0, 1, 2 },
new int[] { 1, 2, 4 },
@@ -315,7 +314,7 @@ public class TestJapaneseTokenizer exten
/*
public void testUserDict4() throws Exception {
// Test entry that has another entry as prefix
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("abcdefghij")),
+ assertTokenStreamContents(analyzer.tokenStream("foo", "abcdefghij"),
new String[] { "ab", "cd", "efg", "hij" },
new int[] { 0, 2, 4, 7 },
new int[] { 2, 4, 7, 10 },
@@ -366,7 +365,7 @@ public class TestJapaneseTokenizer exten
}
private void assertReadings(String input, String... readings) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
ts.reset();
for(String reading : readings) {
@@ -378,7 +377,7 @@ public class TestJapaneseTokenizer exten
}
private void assertPronunciations(String input, String... pronunciations) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
ts.reset();
for(String pronunciation : pronunciations) {
@@ -390,7 +389,7 @@ public class TestJapaneseTokenizer exten
}
private void assertBaseForms(String input, String... baseForms) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
BaseFormAttribute baseFormAtt = ts.addAttribute(BaseFormAttribute.class);
ts.reset();
for(String baseForm : baseForms) {
@@ -402,7 +401,7 @@ public class TestJapaneseTokenizer exten
}
private void assertInflectionTypes(String input, String... inflectionTypes) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
ts.reset();
for(String inflectionType : inflectionTypes) {
@@ -414,7 +413,7 @@ public class TestJapaneseTokenizer exten
}
private void assertInflectionForms(String input, String... inflectionForms) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
ts.reset();
for(String inflectionForm : inflectionForms) {
@@ -426,7 +425,7 @@ public class TestJapaneseTokenizer exten
}
private void assertPartsOfSpeech(String input, String... partsOfSpeech) throws IOException {
- TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ TokenStream ts = analyzer.tokenStream("ignored", input);
PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);
ts.reset();
for(String partOfSpeech : partsOfSpeech) {
@@ -619,7 +618,7 @@ public class TestJapaneseTokenizer exten
if (numIterations > 1) {
// warmup
for (int i = 0; i < numIterations; i++) {
- final TokenStream ts = analyzer.tokenStream("ignored", new StringReader(line));
+ final TokenStream ts = analyzer.tokenStream("ignored", line);
ts.reset();
while(ts.incrementToken());
}
@@ -628,7 +627,7 @@ public class TestJapaneseTokenizer exten
long totalStart = System.currentTimeMillis();
for (int i = 0; i < numIterations; i++) {
- final TokenStream ts = analyzer.tokenStream("ignored", new StringReader(line));
+ final TokenStream ts = analyzer.tokenStream("ignored", line);
ts.reset();
while(ts.incrementToken());
}
@@ -640,7 +639,7 @@ public class TestJapaneseTokenizer exten
totalStart = System.currentTimeMillis();
for (int i = 0; i < numIterations; i++) {
for (String sentence: sentences) {
- final TokenStream ts = analyzer.tokenStream("ignored", new StringReader(sentence));
+ final TokenStream ts = analyzer.tokenStream("ignored", sentence);
ts.reset();
while(ts.incrementToken());
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/ivy.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/ivy.xml (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/ivy.xml Tue Jul 16 17:00:05 2013
@@ -19,9 +19,9 @@
<ivy-module version="2.0">
<info organisation="org.apache.lucene" module="analyzers-morfologik"/>
<dependencies>
- <dependency org="org.carrot2" name="morfologik-polish" rev="1.5.5" transitive="false"/>
- <dependency org="org.carrot2" name="morfologik-fsa" rev="1.5.5" transitive="false"/>
- <dependency org="org.carrot2" name="morfologik-stemming" rev="1.5.5" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-polish" rev="1.6.0" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-fsa" rev="1.6.0" transitive="false"/>
+ <dependency org="org.carrot2" name="morfologik-stemming" rev="1.6.0" transitive="false"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
</ivy-module>
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -26,38 +26,21 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.Version;
-import morfologik.stemming.PolishStemmer.DICTIONARY;
-
/**
* {@link org.apache.lucene.analysis.Analyzer} using Morfologik library.
* @see <a href="http://morfologik.blogspot.com/">Morfologik project page</a>
*/
public class MorfologikAnalyzer extends Analyzer {
-
- private final DICTIONARY dictionary;
private final Version version;
/**
- * Builds an analyzer for a given PolishStemmer.DICTIONARY enum.
- *
- * @param vers
- * lucene compatibility version
- * @param dict
- * A constant specifying which dictionary to choose. See the
- * Morfologik documentation for details or use the default.
- */
- public MorfologikAnalyzer(final Version vers, final DICTIONARY dict) {
- this.version = vers;
- this.dictionary = dict;
- }
-
- /**
- * Builds an analyzer for an original MORFOLOGIK dictionary.
+ * Builds an analyzer with the default Morfologik's dictionary (polimorf).
*
- * @param vers lucene compatibility version
+ * @param version
+ * Lucene compatibility version
*/
- public MorfologikAnalyzer(final Version vers) {
- this(vers, DICTIONARY.MORFOLOGIK);
+ public MorfologikAnalyzer(final Version version) {
+ this.version = version;
}
/**
@@ -78,7 +61,7 @@ public class MorfologikAnalyzer extends
final Tokenizer src = new StandardTokenizer(this.version, reader);
return new TokenStreamComponents(
- src,
- new MorfologikFilter(new StandardFilter(this.version, src), this.dictionary, this.version));
+ src,
+ new MorfologikFilter(new StandardFilter(this.version, src), this.version));
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilter.java Tue Jul 16 17:00:05 2013
@@ -22,20 +22,21 @@ import java.io.IOException;
import java.util.*;
import morfologik.stemming.*;
-import morfologik.stemming.PolishStemmer.DICTIONARY;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.util.CharacterUtils;
import org.apache.lucene.util.*;
/**
- * {@link TokenFilter} using Morfologik library.
+ * {@link TokenFilter} using Morfologik library to transform input tokens into lemma and
+ * morphosyntactic (POS) tokens. Applies to Polish only.
*
- * MorfologikFilter contains a {@link MorphosyntacticTagsAttribute}, which provides morphosyntactic
- * annotations for produced lemmas. See the Morfologik documentation for details.
+ * <p>MorfologikFilter contains a {@link MorphosyntacticTagsAttribute}, which provides morphosyntactic
+ * annotations for produced lemmas. See the Morfologik documentation for details.</p>
*
* @see <a href="http://morfologik.blogspot.com/">Morfologik project page</a>
*/
@@ -44,6 +45,7 @@ public class MorfologikFilter extends To
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final MorphosyntacticTagsAttribute tagsAtt = addAttribute(MorphosyntacticTagsAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
private final CharsRef scratch = new CharsRef(0);
private final CharacterUtils charUtils;
@@ -58,13 +60,11 @@ public class MorfologikFilter extends To
private int lemmaListIndex;
/**
- * Builds a filter for given PolishStemmer.DICTIONARY enum.
- *
+ * Creates MorfologikFilter
* @param in input token stream
- * @param dict PolishStemmer.DICTIONARY enum
* @param version Lucene version compatibility for lowercasing.
*/
- public MorfologikFilter(final TokenStream in, final DICTIONARY dict, final Version version) {
+ public MorfologikFilter(final TokenStream in, final Version version) {
super(in);
this.input = in;
@@ -73,7 +73,7 @@ public class MorfologikFilter extends To
ClassLoader cl = me.getContextClassLoader();
try {
me.setContextClassLoader(PolishStemmer.class.getClassLoader());
- this.stemmer = new PolishStemmer(dict);
+ this.stemmer = new PolishStemmer();
this.charUtils = CharacterUtils.getInstance(version);
this.lemmaList = Collections.emptyList();
} finally {
@@ -81,29 +81,57 @@ public class MorfologikFilter extends To
}
}
+ /**
+ * The tag encoding format has been changing in Morfologik from version
+ * to version. Let's keep both variants and determine which one to run
+ * based on this flag.
+ */
+ private final static boolean multipleTagsPerLemma = true;
+
private void popNextLemma() {
- // Collect all tags for the next unique lemma.
- CharSequence currentStem;
- int tags = 0;
- do {
+ if (multipleTagsPerLemma) {
+ // One tag (concatenated) per lemma.
final WordData lemma = lemmaList.get(lemmaListIndex++);
- currentStem = lemma.getStem();
- final CharSequence tag = lemma.getTag();
+ termAtt.setEmpty().append(lemma.getStem());
+ CharSequence tag = lemma.getTag();
if (tag != null) {
- if (tagsList.size() <= tags) {
- tagsList.add(new StringBuilder());
+ String[] tags = tag.toString().split("\\+|\\|");
+ for (int i = 0; i < tags.length; i++) {
+ if (tagsList.size() <= i) {
+ tagsList.add(new StringBuilder());
+ }
+ StringBuilder buffer = tagsList.get(i);
+ buffer.setLength(0);
+ buffer.append(tags[i]);
}
-
- final StringBuilder buffer = tagsList.get(tags++);
- buffer.setLength(0);
- buffer.append(lemma.getTag());
+ tagsAtt.setTags(tagsList.subList(0, tags.length));
+ } else {
+ tagsAtt.setTags(Collections.<StringBuilder> emptyList());
}
- } while (lemmaListIndex < lemmaList.size() &&
- equalCharSequences(lemmaList.get(lemmaListIndex).getStem(), currentStem));
+ } else {
+ // One tag (concatenated) per stem (lemma repeated).
+ CharSequence currentStem;
+ int tags = 0;
+ do {
+ final WordData lemma = lemmaList.get(lemmaListIndex++);
+ currentStem = lemma.getStem();
+ final CharSequence tag = lemma.getTag();
+ if (tag != null) {
+ if (tagsList.size() <= tags) {
+ tagsList.add(new StringBuilder());
+ }
+
+ final StringBuilder buffer = tagsList.get(tags++);
+ buffer.setLength(0);
+ buffer.append(lemma.getTag());
+ }
+ } while (lemmaListIndex < lemmaList.size() &&
+ equalCharSequences(lemmaList.get(lemmaListIndex).getStem(), currentStem));
- // Set the lemma's base form and tags as attributes.
- termAtt.setEmpty().append(currentStem);
- tagsAtt.setTags(tagsList.subList(0, tags));
+ // Set the lemma's base form and tags as attributes.
+ termAtt.setEmpty().append(currentStem);
+ tagsAtt.setTags(tagsList.subList(0, tags));
+ }
}
/**
@@ -140,7 +168,8 @@ public class MorfologikFilter extends To
popNextLemma();
return true;
} else if (this.input.incrementToken()) {
- if (lookupSurfaceForm(termAtt) || lookupSurfaceForm(toLowercase(termAtt))) {
+ if (!keywordAttr.isKeyword() &&
+ (lookupSurfaceForm(termAtt) || lookupSurfaceForm(toLowercase(termAtt)))) {
current = captureState();
popNextLemma();
} else {
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikFilterFactory.java Tue Jul 16 17:00:05 2013
@@ -17,12 +17,8 @@ package org.apache.lucene.analysis.morfo
* limitations under the License.
*/
-import java.util.Arrays;
-import java.util.Locale;
import java.util.Map;
-import morfologik.stemming.PolishStemmer.DICTIONARY;
-
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -32,39 +28,28 @@ import org.apache.lucene.analysis.util.T
* <fieldType name="text_polish" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
* <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- * <filter class="solr.MorfologikFilterFactory" dictionary="MORFOLOGIK" />
+ * <filter class="solr.MorfologikFilterFactory" />
* </analyzer>
* </fieldType></pre>
*
- * <p>Any of Morfologik dictionaries can be used, these are at the moment:
- * <code>MORFOLOGIK</code> (Morfologik's original dictionary),
- * <code>MORFEUSZ</code> (Morfeusz-SIAT),
- * <code>COMBINED</code> (both of the dictionaries above, combined).
- *
* @see <a href="http://morfologik.blogspot.com/">Morfologik web site</a>
*/
public class MorfologikFilterFactory extends TokenFilterFactory {
- /** Dictionary. */
- private DICTIONARY dictionary = DICTIONARY.MORFOLOGIK;
-
/** Schema attribute. */
+ @Deprecated
public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary";
-
+
/** Creates a new MorfologikFilterFactory */
public MorfologikFilterFactory(Map<String,String> args) {
super(args);
+
+ // Be specific about no-longer-supported dictionary attribute.
String dictionaryName = get(args, DICTIONARY_SCHEMA_ATTRIBUTE);
if (dictionaryName != null && !dictionaryName.isEmpty()) {
- try {
- DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ROOT));
- assert dictionary != null;
- this.dictionary = dictionary;
- } catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute accepts the "
- + "following constants: " + Arrays.toString(DICTIONARY.values()) + ", this value is invalid: "
- + dictionaryName);
- }
+ throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute is no "
+ + "longer supported (Morfologik has one dictionary): " + dictionaryName);
}
+
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -72,6 +57,6 @@ public class MorfologikFilterFactory ext
@Override
public TokenStream create(TokenStream ts) {
- return new MorfologikFilter(ts, dictionary, luceneMatchVersion);
+ return new MorfologikFilter(ts, luceneMatchVersion);
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttribute.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttribute.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorphosyntacticTagsAttribute.java Tue Jul 16 17:00:05 2013
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.lucene.util.Attribute;
/**
- * Morfologik dictionaries provide morphosyntactic annotations for
+ * Morfologik provides morphosyntactic annotations for
* surface forms. For the exact format and description of these,
- * see the project's documentation (annotations vary by dictionary!).
+ * see the project's documentation.
*/
public interface MorphosyntacticTagsAttribute extends Attribute {
/**
@@ -36,7 +36,9 @@ public interface MorphosyntacticTagsAttr
public void setTags(List<StringBuilder> tags);
/**
- * Returns the POS tag of the term.
+ * Returns the POS tag of the term. A single word may have multiple POS tags,
+ * depending on the interpretation (context disambiguation is typically needed
+ * to determine which particular tag is appropriate).
*/
public List<StringBuilder> getTags();
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -18,11 +18,19 @@ package org.apache.lucene.analysis.morfo
*/
import java.io.IOException;
-import java.io.StringReader;
+import java.io.Reader;
import java.util.TreeSet;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.util.Version;
/**
* TODO: The tests below rely on the order of returned lemmas, which is probably not good.
@@ -56,16 +64,28 @@ public class TestMorfologikAnalyzer exte
assertAnalyzesToReuse(
a,
"T. Gl\u00FCcksberg",
- new String[] { "to", "tom", "tona", "Gl\u00FCcksberg" },
- new int[] { 0, 0, 0, 3 },
- new int[] { 1, 1, 1, 13 },
- new int[] { 1, 0, 0, 1 });
+ new String[] { "tom", "tona", "Gl\u00FCcksberg" },
+ new int[] { 0, 0, 3 },
+ new int[] { 1, 1, 13 },
+ new int[] { 1, 0, 1 });
+ }
+
+ @SuppressWarnings("unused")
+ private void dumpTokens(String input) throws IOException {
+ TokenStream ts = getTestAnalyzer().tokenStream("dummy", input);
+ ts.reset();
+
+ MorphosyntacticTagsAttribute attribute = ts.getAttribute(MorphosyntacticTagsAttribute.class);
+ CharTermAttribute charTerm = ts.getAttribute(CharTermAttribute.class);
+ while (ts.incrementToken()) {
+ System.out.println(charTerm.toString() + " => " + attribute.getTags());
+ }
}
/** Test reuse of MorfologikFilter with leftover stems. */
public final void testLeftoverStems() throws IOException {
Analyzer a = getTestAnalyzer();
- TokenStream ts_1 = a.tokenStream("dummy", new StringReader("liÅcie"));
+ TokenStream ts_1 = a.tokenStream("dummy", "liÅcie");
CharTermAttribute termAtt_1 = ts_1.getAttribute(CharTermAttribute.class);
ts_1.reset();
ts_1.incrementToken();
@@ -73,7 +93,7 @@ public class TestMorfologikAnalyzer exte
ts_1.end();
ts_1.close();
- TokenStream ts_2 = a.tokenStream("dummy", new StringReader("danych"));
+ TokenStream ts_2 = a.tokenStream("dummy", "danych");
CharTermAttribute termAtt_2 = ts_2.getAttribute(CharTermAttribute.class);
ts_2.reset();
ts_2.incrementToken();
@@ -120,7 +140,7 @@ public class TestMorfologikAnalyzer exte
/** Test morphosyntactic annotations. */
public final void testPOSAttribute() throws IOException {
- TokenStream ts = getTestAnalyzer().tokenStream("dummy", new StringReader("liÅcie"));
+ TokenStream ts = getTestAnalyzer().tokenStream("dummy", "liÅcie");
ts.reset();
assertPOSToken(ts, "liÅcie",
@@ -144,6 +164,34 @@ public class TestMorfologikAnalyzer exte
ts.close();
}
+ /** */
+ public final void testKeywordAttrTokens() throws IOException {
+ final Version version = TEST_VERSION_CURRENT;
+
+ Analyzer a = new MorfologikAnalyzer(version) {
+ @Override
+ protected TokenStreamComponents createComponents(String field, Reader reader) {
+ final CharArraySet keywords = new CharArraySet(version, 1, false);
+ keywords.add("liÅcie");
+
+ final Tokenizer src = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ TokenStream result = new StandardFilter(TEST_VERSION_CURRENT, src);
+ result = new SetKeywordMarkerFilter(result, keywords);
+ result = new MorfologikFilter(result, TEST_VERSION_CURRENT);
+
+ return new TokenStreamComponents(src, result);
+ }
+ };
+
+ assertAnalyzesToReuse(
+ a,
+ "liÅcie danych",
+ new String[] { "liÅcie", "dany", "dana", "dane", "daÄ" },
+ new int[] { 0, 7, 7, 7, 7 },
+ new int[] { 6, 13, 13, 13, 13 },
+ new int[] { 1, 1, 0, 0, 0 });
+ }
+
/** blast some random strings through the analyzer */
public void testRandom() throws Exception {
checkRandomData(random(), getTestAnalyzer(), 1000 * RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java Tue Jul 16 17:00:05 2013
@@ -18,8 +18,8 @@ package org.apache.lucene.analysis.morfo
*/
import java.io.StringReader;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
@@ -31,10 +31,7 @@ import org.apache.lucene.analysis.TokenS
public class TestMorfologikFilterFactory extends BaseTokenStreamTestCase {
public void testCreateDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety");
- Map<String,String> initParams = new HashMap<String,String>();
- initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE,
- "morfologik");
- MorfologikFilterFactory factory = new MorfologikFilterFactory(initParams);
+ MorfologikFilterFactory factory = new MorfologikFilterFactory(Collections.<String,String>emptyMap());
TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
Modified: lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java Tue Jul 16 17:00:05 2013
@@ -20,8 +20,11 @@ package org.apache.lucene.analysis.cn.sm
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Properties;
+import org.apache.lucene.util.IOUtils;
+
/**
* Manages analysis data configuration for SmartChineseAnalyzer
* <p>
@@ -77,13 +80,13 @@ public class AnalyzerProfile {
Properties prop = new Properties();
try {
FileInputStream input = new FileInputStream(propFile);
- prop.load(input);
+ prop.load(new InputStreamReader(input, IOUtils.CHARSET_UTF_8));
String dir = prop.getProperty("analysis.data.dir", "");
input.close();
return dir;
} catch (IOException e) {
+ return "";
}
- return "";
}
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java Tue Jul 16 17:00:05 2013
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.cn.sm
import java.io.IOException;
import java.io.Reader;
-import java.io.StringReader;
import java.util.Random;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -185,7 +184,7 @@ public class TestSmartChineseAnalyzer ex
sb.append("æè´ä¹°äºéå
·åæè£
ã");
}
Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT);
- TokenStream stream = analyzer.tokenStream("", new StringReader(sb.toString()));
+ TokenStream stream = analyzer.tokenStream("", sb.toString());
stream.reset();
while (stream.incrementToken()) {
}
@@ -198,7 +197,7 @@ public class TestSmartChineseAnalyzer ex
sb.append("æè´ä¹°äºéå
·åæè£
");
}
Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT);
- TokenStream stream = analyzer.tokenStream("", new StringReader(sb.toString()));
+ TokenStream stream = analyzer.tokenStream("", sb.toString());
stream.reset();
while (stream.incrementToken()) {
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMABaseAnalyzerTest.java Tue Jul 16 17:00:05 2013
@@ -35,7 +35,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
@@ -62,7 +61,7 @@ public class UIMABaseAnalyzerTest extend
@Test
public void baseUIMAAnalyzerStreamTest() throws Exception {
- TokenStream ts = analyzer.tokenStream("text", new StringReader("the big brown fox jumped on the wood"));
+ TokenStream ts = analyzer.tokenStream("text", "the big brown fox jumped on the wood");
assertTokenStreamContents(ts, new String[]{"the", "big", "brown", "fox", "jumped", "on", "the", "wood"});
}
Modified: lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java (original)
+++ lucene/dev/branches/lucene3069/lucene/analysis/uima/src/test/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzerTest.java Tue Jul 16 17:00:05 2013
@@ -23,8 +23,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.StringReader;
-
/**
* Testcase for {@link UIMATypeAwareAnalyzer}
*/
@@ -51,7 +49,7 @@ public class UIMATypeAwareAnalyzerTest e
public void baseUIMATypeAwareAnalyzerStreamTest() throws Exception {
// create a token stream
- TokenStream ts = analyzer.tokenStream("text", new StringReader("the big brown fox jumped on the wood"));
+ TokenStream ts = analyzer.tokenStream("text", "the big brown fox jumped on the wood");
// check that 'the big brown fox jumped on the wood' tokens have the expected PoS types
assertTokenStreamContents(ts,
Modified: lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Tue Jul 16 17:00:05 2013
@@ -119,14 +119,9 @@ public class CreateIndexTask extends Per
if (mergeScheduler.equals("org.apache.lucene.index.ConcurrentMergeScheduler")) {
ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) iwConf.getMergeScheduler();
- int v = config.get("concurrent.merge.scheduler.max.thread.count", -1);
- if (v != -1) {
- cms.setMaxThreadCount(v);
- }
- v = config.get("concurrent.merge.scheduler.max.merge.count", -1);
- if (v != -1) {
- cms.setMaxMergeCount(v);
- }
+ int maxThreadCount = config.get("concurrent.merge.scheduler.max.thread.count", ConcurrentMergeScheduler.DEFAULT_MAX_THREAD_COUNT);
+ int maxMergeCount = config.get("concurrent.merge.scheduler.max.merge.count", ConcurrentMergeScheduler.DEFAULT_MAX_MERGE_COUNT);
+ cms.setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
}
}
Modified: lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java (original)
+++ lucene/dev/branches/lucene3069/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java Tue Jul 16 17:00:05 2013
@@ -18,9 +18,9 @@ package org.apache.lucene.benchmark.byTa
*/
import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
+import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -80,8 +80,7 @@ public class Config {
}
// read props from string
this.props = new Properties();
- // props.load always assumes iso8859-1...
- props.load(new ByteArrayInputStream(sb.toString().getBytes("ISO-8859-1")));
+ props.load(new StringReader(sb.toString()));
// make sure work dir is set properly
if (props.get("work.dir") == null) {
Modified: lucene/dev/branches/lucene3069/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java (original)
+++ lucene/dev/branches/lucene3069/lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java Tue Jul 16 17:00:05 2013
@@ -21,7 +21,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
-import java.io.StringReader;
import java.text.Collator;
import java.util.List;
import java.util.Locale;
@@ -979,8 +978,8 @@ public class TestPerfTasksLogic extends
private void assertEqualCollation(Analyzer a1, Analyzer a2, String text)
throws Exception {
- TokenStream ts1 = a1.tokenStream("bogus", new StringReader(text));
- TokenStream ts2 = a2.tokenStream("bogus", new StringReader(text));
+ TokenStream ts1 = a1.tokenStream("bogus", text);
+ TokenStream ts2 = a2.tokenStream("bogus", text);
ts1.reset();
ts2.reset();
TermToBytesRefAttribute termAtt1 = ts1.addAttribute(TermToBytesRefAttribute.class);
@@ -1030,7 +1029,7 @@ public class TestPerfTasksLogic extends
Benchmark benchmark = execBenchmark(getAnalyzerFactoryConfig
("shingle-analyzer", "StandardTokenizer,ShingleFilter"));
benchmark.getRunData().getAnalyzer().tokenStream
- ("bogus", new StringReader(text)).close();
+ ("bogus", text).close();
BaseTokenStreamTestCase.assertAnalyzesTo(benchmark.getRunData().getAnalyzer(), text,
new String[] { "one", "one two", "two", "two three",
"three", "three four", "four", "four five",
Modified: lucene/dev/branches/lucene3069/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/build.xml?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/build.xml (original)
+++ lucene/dev/branches/lucene3069/lucene/build.xml Tue Jul 16 17:00:05 2013
@@ -184,8 +184,7 @@
<bundledSignatures name="jdk-unsafe-${javac.target}"/>
<bundledSignatures name="jdk-deprecated-${javac.target}"/>
<signaturesFileSet dir="${common.dir}/tools/forbiddenApis">
- <include name="executors.txt" />
- <include name="chars.txt" />
+ <include name="base.txt" />
</signaturesFileSet>
<fileset dir="${basedir}/build" includes="**/*.class" />
</forbidden-apis>
@@ -348,7 +347,7 @@
</target>
<!-- rat-sources-typedef is *not* a useless dependency. do not remove -->
- <target name="rat-sources" depends="rat-sources-typedef">
+ <target name="rat-sources" depends="rat-sources-typedef,common.rat-sources">
<subant target="rat-sources" failonerror="true" inheritall="false">
<propertyset refid="uptodate.and.compiled.properties"/>
<fileset dir="core" includes="build.xml"/>
Modified: lucene/dev/branches/lucene3069/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java?rev=1503797&r1=1503796&r2=1503797&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java (original)
+++ lucene/dev/branches/lucene3069/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java Tue Jul 16 17:00:05 2013
@@ -33,7 +33,6 @@ import org.apache.lucene.search.Wildcard
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.io.StringReader;
import java.util.Collection;
import java.util.LinkedList;
@@ -86,7 +85,7 @@ public class SimpleNaiveBayesClassifier
private String[] tokenizeDoc(String doc) throws IOException {
Collection<String> result = new LinkedList<String>();
- TokenStream tokenStream = analyzer.tokenStream(textFieldName, new StringReader(doc));
+ TokenStream tokenStream = analyzer.tokenStream(textFieldName, doc);
CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
tokenStream.reset();
while (tokenStream.incrementToken()) {