You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/20 17:27:23 UTC
svn commit: r1458924 [4/13] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/maven/
dev-tools/maven/solr/solrj/src/java/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/analysis/co...
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -62,7 +62,7 @@ public class TestGermanMinimalStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -68,7 +68,7 @@ public class TestGermanStemFilter extend
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java Wed Mar 20 16:27:13 2013
@@ -23,7 +23,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
@@ -57,7 +57,7 @@ public class TestPorterStemFilter extend
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("yourselves");
Tokenizer tokenizer = new MockTokenizer(new StringReader("yourselves yours"), MockTokenizer.WHITESPACE, false);
- TokenStream filter = new PorterStemFilter(new KeywordMarkerFilter(tokenizer, set));
+ TokenStream filter = new PorterStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
assertTokenStreamContents(filter, new String[] {"yourselves", "your"});
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestFinnishLightStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -185,7 +185,7 @@ public class TestFrenchLightStemFilter e
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -64,7 +64,7 @@ public class TestFrenchMinimalStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
/**
@@ -59,7 +59,7 @@ public class TestGalicianMinimalStemFilt
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestHungarianLightStemFilte
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -63,7 +63,7 @@ public class HunspellStemFilterTest ext
// assert with keywork marker
tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
- filter = new HunspellStemFilter(new KeywordMarkerFilter(tokenizer, set), DICTIONARY);
+ filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY);
assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java Wed Mar 20 16:27:13 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.analysis.misce
import java.io.IOException;
import java.io.StringReader;
import java.util.Locale;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
@@ -36,34 +37,68 @@ import org.junit.Test;
public class TestKeywordMarkerFilter extends BaseTokenStreamTestCase {
@Test
- public void testIncrementToken() throws IOException {
+ public void testSetFilterIncrementToken() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 5, true);
set.add("lucenefox");
String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new KeywordMarkerFilter(new MockTokenizer(new StringReader(
+ new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
"The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set)), output);
CharArraySet mixedCaseSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("LuceneFox"), false);
assertTokenStreamContents(new LowerCaseFilterMock(
- new KeywordMarkerFilter(new MockTokenizer(new StringReader(
+ new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
"The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), mixedCaseSet)), output);
CharArraySet set2 = set;
assertTokenStreamContents(new LowerCaseFilterMock(
- new KeywordMarkerFilter(new MockTokenizer(new StringReader(
+ new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
"The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set2)), output);
}
+
+ @Test
+ public void testPatternFilterIncrementToken() throws IOException {
+ String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
+ "jumps" };
+ assertTokenStreamContents(new LowerCaseFilterMock(
+ new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
+ "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[fF]ox"))), output);
+
+ output = new String[] { "the", "quick", "brown", "lucenefox",
+ "jumps" };
+
+ assertTokenStreamContents(new LowerCaseFilterMock(
+ new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
+ "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[f]ox"))), output);
+ }
// LUCENE-2901
public void testComposition() throws Exception {
TokenStream ts = new LowerCaseFilterMock(
- new KeywordMarkerFilter(
- new KeywordMarkerFilter(
+ new SetKeywordMarkerFilter(
+ new SetKeywordMarkerFilter(
new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Birds", "Houses"), false)),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));
assertTokenStreamContents(ts, new String[] { "Dogs", "Trees", "Birds", "Houses" });
+
+ ts = new LowerCaseFilterMock(
+ new PatternKeywordMarkerFilter(
+ new PatternKeywordMarkerFilter(
+ new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ Pattern.compile("Birds|Houses")),
+ Pattern.compile("Dogs|Trees")));
+
+ assertTokenStreamContents(ts, new String[] { "Dogs", "Trees", "Birds", "Houses" });
+
+ ts = new LowerCaseFilterMock(
+ new SetKeywordMarkerFilter(
+ new PatternKeywordMarkerFilter(
+ new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ Pattern.compile("Birds|Houses")),
+ new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));
+
+ assertTokenStreamContents(ts, new String[] { "Dogs", "Trees", "Birds", "Houses" });
}
public static final class LowerCaseFilterMock extends TokenFilter {
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java Wed Mar 20 16:27:13 2013
@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.util.S
* Simple tests to ensure the keyword marker filter factory is working.
*/
public class TestKeywordMarkerFilterFactory extends BaseTokenStreamTestCase {
+
public void testKeywords() throws IOException {
Reader reader = new StringReader("dogs cats");
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
@@ -48,6 +49,36 @@ public class TestKeywordMarkerFilterFact
TokenStream ts = new PorterStemFilter(factory.create(tokenizer));
assertTokenStreamContents(ts, new String[] { "dog", "cats" });
+
+
+ reader = new StringReader("dogs cats");
+ tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ factory = new KeywordMarkerFilterFactory();
+ args = new HashMap<String,String>();
+
+ args.put("pattern", "cats|Dogs");
+ factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
+ factory.init(args);
+ factory.inform(null);
+
+ ts = new PorterStemFilter(factory.create(tokenizer));
+ assertTokenStreamContents(ts, new String[] { "dog", "cats" });
+ }
+
+ public void testKeywordsMixed() throws IOException {
+ Reader reader = new StringReader("dogs cats birds");
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
+ Map<String,String> args = new HashMap<String,String>();
+ ResourceLoader loader = new StringMockResourceLoader("cats");
+ args.put("protected", "protwords.txt");
+ args.put("pattern", "birds|Dogs");
+ factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
+ factory.init(args);
+ factory.inform(loader);
+
+ TokenStream ts = new PorterStemFilter(factory.create(tokenizer));
+ assertTokenStreamContents(ts, new String[] { "dog", "cats", "birds" });
}
public void testKeywordsCaseInsensitive() throws IOException {
@@ -64,5 +95,36 @@ public class TestKeywordMarkerFilterFact
TokenStream ts = new PorterStemFilter(factory.create(tokenizer));
assertTokenStreamContents(ts, new String[] { "dog", "cats", "Cats" });
+
+ reader = new StringReader("dogs cats Cats");
+ tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ factory = new KeywordMarkerFilterFactory();
+ args = new HashMap<String,String>();
+
+ args.put("pattern", "Cats");
+ args.put("ignoreCase", "true");
+ factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
+ factory.init(args);
+ factory.inform(null);
+
+ ts = new PorterStemFilter(factory.create(tokenizer));
+ assertTokenStreamContents(ts, new String[] { "dog", "cats", "Cats" });
+ }
+
+ public void testKeywordsCaseInsensitiveMixed() throws IOException {
+ Reader reader = new StringReader("dogs cats Cats Birds birds");
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
+ Map<String,String> args = new HashMap<String,String>();
+ ResourceLoader loader = new StringMockResourceLoader("cats");
+ args.put("protected", "protwords.txt");
+ args.put("pattern", "birds");
+ args.put("ignoreCase", "true");
+ factory.setLuceneMatchVersion(TEST_VERSION_CURRENT);
+ factory.init(args);
+ factory.inform(loader);
+
+ TokenStream ts = new PorterStemFilter(factory.create(tokenizer));
+ assertTokenStreamContents(ts, new String[] { "dog", "cats", "Cats", "Birds", "birds" });
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java Wed Mar 20 16:27:13 2013
@@ -36,7 +36,7 @@ public class TestStemmerOverrideFilter e
dictionary.put("booked", "books");
Tokenizer tokenizer = new KeywordTokenizer(new StringReader("booked"));
TokenStream stream = new PorterStemFilter(
- new StemmerOverrideFilter(TEST_VERSION_CURRENT, tokenizer, dictionary));
+ new StemmerOverrideFilter(tokenizer, dictionary));
assertTokenStreamContents(stream, new String[] { "books" });
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -57,7 +57,7 @@ public class TestNorwegianLightStemFilte
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new NorwegianLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -57,7 +57,7 @@ public class TestNorwegianMinimalStemFil
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -103,7 +103,7 @@ public class TestPortugueseLightStemFilt
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java Wed Mar 20 16:27:13 2013
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.Tokeni
import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -77,7 +77,7 @@ public class TestPortugueseMinimalStemFi
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java Wed Mar 20 16:27:13 2013
@@ -29,7 +29,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -76,7 +76,7 @@ public class TestPortugueseStemFilter ex
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestRussianLightStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new RussianLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java Wed Mar 20 16:27:13 2013
@@ -26,7 +26,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,7 +55,7 @@ public class TestSwedishLightStemFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new SwedishLightStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Wed Mar 20 16:27:13 2013
@@ -55,9 +55,13 @@ public class TestSynonymMapFilter extend
if (VERBOSE) {
System.out.println(" add input=" + input + " output=" + output + " keepOrig=" + keepOrig);
}
- b.add(new CharsRef(input.replaceAll(" +", "\u0000")),
- new CharsRef(output.replaceAll(" +", "\u0000")),
- keepOrig);
+ CharsRef inputCharsRef = new CharsRef();
+ SynonymMap.Builder.join(input.split(" +"), inputCharsRef);
+
+ CharsRef outputCharsRef = new CharsRef();
+ SynonymMap.Builder.join(output.split(" +"), outputCharsRef);
+
+ b.add(inputCharsRef, outputCharsRef, keepOrig);
}
private void assertEquals(CharTermAttribute term, String expected) {
Modified: lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java Wed Mar 20 16:27:13 2013
@@ -62,6 +62,8 @@ public final class ICUTokenizer extends
* Reader.
* <p>
* The default script-specific handling is used.
+ * <p>
+ * The default attribute factory is used.
*
* @param input Reader containing text to tokenize.
* @see DefaultICUTokenizerConfig
@@ -73,12 +75,26 @@ public final class ICUTokenizer extends
/**
* Construct a new ICUTokenizer that breaks text into words from the given
* Reader, using a tailored BreakIterator configuration.
+ * <p>
+ * The default attribute factory is used.
*
* @param input Reader containing text to tokenize.
* @param config Tailored BreakIterator configuration
*/
public ICUTokenizer(Reader input, ICUTokenizerConfig config) {
- super(input);
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, config);
+ }
+
+ /**
+ * Construct a new ICUTokenizer that breaks text into words from the given
+ * Reader, using a tailored BreakIterator configuration.
+ *
+ * @param factory AttributeFactory to use
+ * @param input Reader containing text to tokenize.
+ * @param config Tailored BreakIterator configuration
+ */
+ public ICUTokenizer(AttributeFactory factory, Reader input, ICUTokenizerConfig config) {
+ super(factory, input);
this.config = config;
breaker = new CompositeBreakIterator(config);
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -25,11 +25,11 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import com.ibm.icu.lang.UCharacter;
@@ -144,8 +144,8 @@ public class ICUTokenizerFactory extends
}
@Override
- public Tokenizer create(Reader input) {
+ public ICUTokenizer create(AttributeFactory factory, Reader input) {
assert config != null : "inform must be called first!";
- return new ICUTokenizer(input, config);
+ return new ICUTokenizer(factory, input, config);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseBaseFormFilter.java Wed Mar 20 16:27:13 2013
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ja.tokenattributes.BaseFormAttribute;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
@@ -32,7 +32,7 @@ import org.apache.lucene.analysis.tokena
* This acts as a lemmatizer for verbs and adjectives.
* <p>
* To prevent terms from being stemmed use an instance of
- * {@link KeywordMarkerFilter} or a custom {@link TokenFilter} that sets
+ * {@link SetKeywordMarkerFilter} or a custom {@link TokenFilter} that sets
* the {@link KeywordAttribute} before this {@link TokenStream}.
* </p>
*/
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseKatakanaStemFilter.java Wed Mar 20 16:27:13 2013
@@ -35,7 +35,7 @@ import java.io.IOException;
* </p>
* <p>
* In order to prevent terms from being stemmed, use an instance of
- * {@link org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter}
+ * {@link org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter}
* or a custom {@link TokenFilter} that sets the {@link KeywordAttribute}
* before this {@link TokenStream}.
* </p>
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java Wed Mar 20 16:27:13 2013
@@ -187,6 +187,8 @@ public final class JapaneseTokenizer ext
/**
* Create a new JapaneseTokenizer.
+ * <p>
+ * Uses the default AttributeFactory.
*
* @param input Reader containing text
* @param userDictionary Optional: if non-null, user dictionary.
@@ -194,7 +196,21 @@ public final class JapaneseTokenizer ext
* @param mode tokenization mode.
*/
public JapaneseTokenizer(Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
- super(input);
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, userDictionary, discardPunctuation, mode);
+ }
+
+ /**
+ * Create a new JapaneseTokenizer.
+ *
+ * @param factory the AttributeFactory to use
+ * @param input Reader containing text
+ * @param userDictionary Optional: if non-null, user dictionary.
+ * @param discardPunctuation true if punctuation tokens should be dropped from the output.
+ * @param mode tokenization mode.
+ */
+ public JapaneseTokenizer
+ (AttributeFactory factory, Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
+ super(factory, input);
dictionary = TokenInfoDictionary.getInstance();
fst = dictionary.getFST();
unkDictionary = UnknownDictionary.getInstance();
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -27,11 +27,10 @@ import java.nio.charset.CodingErrorActio
import java.util.Locale;
import java.util.Map;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.ja.JapaneseTokenizer;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -89,8 +88,8 @@ public class JapaneseTokenizerFactory ex
}
@Override
- public Tokenizer create(Reader input) {
- return new JapaneseTokenizer(input, userDictionary, discardPunctuation, mode);
+ public JapaneseTokenizer create(AttributeFactory factory, Reader input) {
+ return new JapaneseTokenizer(factory, input, userDictionary, discardPunctuation, mode);
}
private Mode getMode(Map<String, String> args) {
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java Wed Mar 20 16:27:13 2013
@@ -25,7 +25,7 @@ import org.apache.lucene.analysis.BaseTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase {
@@ -49,7 +49,7 @@ public class TestJapaneseBaseFormFilter
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.DEFAULT_MODE);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new JapaneseBaseFormFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java Wed Mar 20 16:27:13 2013
@@ -23,7 +23,7 @@ import org.apache.lucene.analysis.MockTo
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import java.io.IOException;
@@ -70,7 +70,7 @@ public class TestJapaneseKatakanaStemFil
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(sink));
}
};
Modified: lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Wed Mar 20 16:27:13 2013
@@ -52,10 +52,6 @@ public final class SentenceTokenizer ext
super(reader);
}
- public SentenceTokenizer(AttributeSource source, Reader reader) {
- super(source, reader);
- }
-
public SentenceTokenizer(AttributeFactory factory, Reader reader) {
super(factory, reader);
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -19,9 +19,8 @@ package org.apache.lucene.analysis.cn.sm
import java.io.Reader;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
/**
* Factory for the SmartChineseAnalyzer {@link SentenceTokenizer}
@@ -29,7 +28,7 @@ import org.apache.lucene.analysis.util.T
*/
public class SmartChineseSentenceTokenizerFactory extends TokenizerFactory {
@Override
- public Tokenizer create(Reader input) {
- return new SentenceTokenizer(input);
+ public SentenceTokenizer create(AttributeFactory factory, Reader input) {
+ return new SentenceTokenizer(factory, input);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java Wed Mar 20 16:27:13 2013
@@ -23,7 +23,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.standard.StandardFilter;
@@ -112,7 +112,7 @@ public final class PolishAnalyzer extend
/**
* Builds an analyzer with the given stop words. If a non-empty stem exclusion set is
- * provided this analyzer will add a {@link KeywordMarkerFilter} before
+ * provided this analyzer will add a {@link SetKeywordMarkerFilter} before
* stemming.
*
* @param matchVersion lucene compatibility version
@@ -135,7 +135,7 @@ public final class PolishAnalyzer extend
* {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
* built from an {@link StandardTokenizer} filtered with
* {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}
- * , {@link KeywordMarkerFilter} if a stem exclusion set is
+ * , {@link SetKeywordMarkerFilter} if a stem exclusion set is
* provided and {@link StempelFilter}.
*/
@Override
@@ -146,7 +146,7 @@ public final class PolishAnalyzer extend
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if(!stemExclusionSet.isEmpty())
- result = new KeywordMarkerFilter(result, stemExclusionSet);
+ result = new SetKeywordMarkerFilter(result, stemExclusionSet);
result = new StempelFilter(result, new StempelStemmer(stemTable));
return new TokenStreamComponents(source, result);
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java Wed Mar 20 16:27:13 2013
@@ -44,8 +44,9 @@ public abstract class BaseUIMATokenizer
protected AnalysisEngine ae;
protected CAS cas;
- protected BaseUIMATokenizer(Reader reader, String descriptorPath, Map<String, Object> configurationParameters) {
- super(reader);
+ protected BaseUIMATokenizer
+ (AttributeFactory factory, Reader reader, String descriptorPath, Map<String, Object> configurationParameters) {
+ super(factory, reader);
this.descriptorPath = descriptorPath;
this.configurationParameters = configurationParameters;
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java Wed Mar 20 16:27:13 2013
@@ -43,7 +43,12 @@ public final class UIMAAnnotationsTokeni
private int finalOffset = 0;
public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Map<String, Object> configurationParameters, Reader input) {
- super(input, descriptorPath, configurationParameters);
+ this(descriptorPath, tokenType, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
+ }
+
+ public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Map<String, Object> configurationParameters,
+ AttributeFactory factory, Reader input) {
+ super(factory, input, descriptorPath, configurationParameters);
this.tokenTypeString = tokenType;
this.termAttr = addAttribute(CharTermAttribute.class);
this.offsetAttr = addAttribute(OffsetAttribute.class);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -17,9 +17,8 @@ package org.apache.lucene.analysis.uima;
* limitations under the License.
*/
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.HashMap;
@@ -54,7 +53,7 @@ public class UIMAAnnotationsTokenizerFac
}
@Override
- public Tokenizer create(Reader input) {
- return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, input);
+ public UIMAAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
+ return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, factory, input);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java Wed Mar 20 16:27:13 2013
@@ -53,7 +53,12 @@ public final class UIMATypeAwareAnnotati
private int finalOffset = 0;
public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath, Map<String, Object> configurationParameters, Reader input) {
- super(input, descriptorPath, configurationParameters);
+ this(descriptorPath, tokenType, typeAttributeFeaturePath, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
+ }
+
+ public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath,
+ Map<String, Object> configurationParameters, AttributeFactory factory, Reader input) {
+ super(factory, input, descriptorPath, configurationParameters);
this.tokenTypeString = tokenType;
this.termAttr = addAttribute(CharTermAttribute.class);
this.typeAttr = addAttribute(TypeAttribute.class);
Modified: lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -17,8 +17,8 @@ package org.apache.lucene.analysis.uima;
* limitations under the License.
*/
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import java.io.Reader;
import java.util.HashMap;
@@ -55,7 +55,8 @@ public class UIMATypeAwareAnnotationsTok
}
@Override
- public Tokenizer create(Reader input) {
- return new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, configurationParameters, input);
+ public UIMATypeAwareAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
+ return new UIMATypeAwareAnnotationsTokenizer
+ (descriptorPath, tokenType, featurePath, configurationParameters, factory, input);
}
}
Modified: lucene/dev/branches/lucene4258/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AnalyzerFactoryTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AnalyzerFactoryTask.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AnalyzerFactoryTask.java (original)
+++ lucene/dev/branches/lucene4258/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AnalyzerFactoryTask.java Wed Mar 20 16:27:13 2013
@@ -376,6 +376,9 @@ public class AnalyzerFactoryTask extends
instance.init(argMap);
if (instance instanceof ResourceLoaderAware) {
File baseDir = new File(getRunData().getConfig().get("work.dir", "work")).getAbsoluteFile();
+ if ( ! baseDir.isDirectory()) {
+ baseDir = new File(".").getAbsoluteFile();
+ }
((ResourceLoaderAware)instance).inform(new FilesystemResourceLoader(baseDir));
}
if (CharFilterFactory.class.isAssignableFrom(clazz)) {
Modified: lucene/dev/branches/lucene4258/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/build.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/build.xml (original)
+++ lucene/dev/branches/lucene4258/lucene/build.xml Wed Mar 20 16:27:13 2013
@@ -439,7 +439,7 @@
<svn-export-source source.dir="."/>
<!-- Exclude javadoc package-list files under licenses incompatible with the ASL -->
- <delete dir="${svn.export.dir}/tools/javadoc/java6"/>
+ <delete dir="${svn.export.dir}/tools/javadoc/java7"/>
<!-- Exclude clover license files incompatible with the ASL -->
<delete dir="${svn.export.dir}/tools/clover"/>
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Wed Mar 20 16:27:13 2013
@@ -1488,6 +1488,11 @@ public final class DirectPostingsFormat
}
return docID();
}
+
+ @Override
+ public long cost() {
+ return postings.length;
+ }
}
// Docs + freqs:
@@ -1553,6 +1558,11 @@ public final class DirectPostingsFormat
}
return docID();
}
+
+ @Override
+ public long cost() {
+ return postings.length / 2;
+ }
}
// Docs + freqs + positions/offets:
@@ -1634,6 +1644,12 @@ public final class DirectPostingsFormat
}
return docID();
}
+
+ @Override
+ public long cost() {
+ // TODO: could do a better estimate
+ return postings.length / 2;
+ }
}
private final static class LowFreqDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -1786,6 +1802,12 @@ public final class DirectPostingsFormat
return null;
}
}
+
+ @Override
+ public long cost() {
+ // TODO: could do a better estimate
+ return postings.length / 2;
+ }
}
// Docs + freqs:
@@ -1959,6 +1981,11 @@ public final class DirectPostingsFormat
return docID = docIDs[upto];
}
}
+
+ @Override
+ public long cost() {
+ return docIDs.length;
+ }
}
// TODO: specialize offsets and not
@@ -2192,5 +2219,10 @@ public final class DirectPostingsFormat
return payload;
}
}
+
+ @Override
+ public long cost() {
+ return docIDs.length;
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Wed Mar 20 16:27:13 2013
@@ -431,6 +431,11 @@ public final class MemoryPostingsFormat
public int freq() {
return freq;
}
+
+ @Override
+ public long cost() {
+ return numDocs;
+ }
}
private final static class FSTDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -618,6 +623,11 @@ public final class MemoryPostingsFormat
public int freq() {
return freq;
}
+
+ @Override
+ public long cost() {
+ return numDocs;
+ }
}
private final static class FSTTermsEnum extends TermsEnum {
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Wed Mar 20 16:27:13 2013
@@ -260,6 +260,7 @@ public class PulsingPostingsReader exten
private int accum;
private int freq;
private int payloadLength;
+ private int cost;
public PulsingDocsEnum(FieldInfo fieldInfo) {
indexOptions = fieldInfo.getIndexOptions();
@@ -283,6 +284,7 @@ public class PulsingPostingsReader exten
docID = -1;
accum = 0;
freq = 1;
+ cost = termState.docFreq;
payloadLength = 0;
this.liveDocs = liveDocs;
return this;
@@ -367,6 +369,11 @@ public class PulsingPostingsReader exten
}
return docID = NO_MORE_DOCS;
}
+
+ @Override
+ public long cost() {
+ return cost;
+ }
}
private static class PulsingDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -390,6 +397,7 @@ public class PulsingPostingsReader exten
private int offsetLength;
private boolean payloadRetrieved;
+ private int cost;
public PulsingDocsAndPositionsEnum(FieldInfo fieldInfo) {
indexOptions = fieldInfo.getIndexOptions();
@@ -415,6 +423,7 @@ public class PulsingPostingsReader exten
posPending = 0;
docID = -1;
accum = 0;
+ cost = termState.docFreq;
startOffset = storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
offsetLength = 0;
//System.out.println("PR d&p reset storesPayloads=" + storePayloads + " bytes=" + bytes.length + " this=" + this);
@@ -551,6 +560,11 @@ public class PulsingPostingsReader exten
return null;
}
}
+
+ @Override
+ public long cost() {
+ return cost;
+ }
}
@Override
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Wed Mar 20 16:27:13 2013
@@ -441,6 +441,11 @@ public class SepPostingsReader extends P
return doc;
}
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
}
class SepDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -717,5 +722,10 @@ public class SepPostingsReader extends P
pendingPayloadBytes = 0;
return payload;
}
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Wed Mar 20 16:27:13 2013
@@ -199,7 +199,7 @@ class SimpleTextFieldsReader extends Fie
} else {
docsEnum = new SimpleTextDocsEnum();
}
- return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
+ return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY, docFreq);
}
@Override
@@ -216,7 +216,7 @@ class SimpleTextFieldsReader extends Fie
} else {
docsAndPositionsEnum = new SimpleTextDocsAndPositionsEnum();
}
- return docsAndPositionsEnum.reset(docsStart, liveDocs, indexOptions);
+ return docsAndPositionsEnum.reset(docsStart, liveDocs, indexOptions, docFreq);
}
@Override
@@ -234,6 +234,7 @@ class SimpleTextFieldsReader extends Fie
private Bits liveDocs;
private final BytesRef scratch = new BytesRef(10);
private final CharsRef scratchUTF16 = new CharsRef(10);
+ private int cost;
public SimpleTextDocsEnum() {
this.inStart = SimpleTextFieldsReader.this.in;
@@ -244,12 +245,13 @@ class SimpleTextFieldsReader extends Fie
return in == inStart;
}
- public SimpleTextDocsEnum reset(long fp, Bits liveDocs, boolean omitTF) throws IOException {
+ public SimpleTextDocsEnum reset(long fp, Bits liveDocs, boolean omitTF, int docFreq) throws IOException {
this.liveDocs = liveDocs;
in.seek(fp);
this.omitTF = omitTF;
docID = -1;
tf = 1;
+ cost = docFreq;
return this;
}
@@ -316,6 +318,11 @@ class SimpleTextFieldsReader extends Fie
while(nextDoc() < target);
return docID;
}
+
+ @Override
+ public long cost() {
+ return cost;
+ }
}
private class SimpleTextDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -334,6 +341,7 @@ class SimpleTextFieldsReader extends Fie
private boolean readPositions;
private int startOffset;
private int endOffset;
+ private int cost;
public SimpleTextDocsAndPositionsEnum() {
this.inStart = SimpleTextFieldsReader.this.in;
@@ -344,7 +352,7 @@ class SimpleTextFieldsReader extends Fie
return in == inStart;
}
- public SimpleTextDocsAndPositionsEnum reset(long fp, Bits liveDocs, IndexOptions indexOptions) {
+ public SimpleTextDocsAndPositionsEnum reset(long fp, Bits liveDocs, IndexOptions indexOptions, int docFreq) {
this.liveDocs = liveDocs;
nextDocStart = fp;
docID = -1;
@@ -354,6 +362,7 @@ class SimpleTextFieldsReader extends Fie
startOffset = -1;
endOffset = -1;
}
+ cost = docFreq;
return this;
}
@@ -471,6 +480,11 @@ class SimpleTextFieldsReader extends Fie
public BytesRef getPayload() {
return payload;
}
+
+ @Override
+ public long cost() {
+ return cost;
+ }
}
static class TermData {
Modified: lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Wed Mar 20 16:27:13 2013
@@ -444,6 +444,11 @@ public class SimpleTextTermVectorsReader
this.doc = -1;
didNext = false;
}
+
+ @Override
+ public long cost() {
+ return 1;
+ }
}
private static class SimpleTVDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -535,5 +540,10 @@ public class SimpleTextTermVectorsReader
return endOffsets[nextPos-1];
}
}
+
+ @Override
+ public long cost() {
+ return 1;
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/common-build.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/common-build.xml (original)
+++ lucene/dev/branches/lucene4258/lucene/common-build.xml Wed Mar 20 16:27:13 2013
@@ -143,16 +143,16 @@
<property name="javac.deprecation" value="off"/>
<property name="javac.debug" value="on"/>
- <property name="javac.source" value="1.6"/>
- <property name="javac.target" value="1.6"/>
- <property name="javac.source.backwards" value="1.6"/>
- <property name="javac.target.backwards" value="1.6"/>
+ <property name="javac.source" value="1.7"/>
+ <property name="javac.target" value="1.7"/>
+ <property name="javac.source.backwards" value="1.7"/>
+ <property name="javac.target.backwards" value="1.7"/>
<property name="javac.args" value="-Xlint -Xlint:-deprecation -Xlint:-serial -Xlint:-options"/>
<property name="bootclasspath" value=""/>
- <property name="javadoc.link" value="http://download.oracle.com/javase/6/docs/api/"/>
+ <property name="javadoc.link" value="http://download.oracle.com/javase/7/docs/api/"/>
<property name="javadoc.link.junit" value="http://junit.sourceforge.net/javadoc/"/>
<property name="javadoc.packagelist.dir" location="${common.dir}/tools/javadoc"/>
- <available file="${javadoc.packagelist.dir}/java6/package-list" property="javadoc.java6.packagelist.exists"/>
+ <available file="${javadoc.packagelist.dir}/java7/package-list" property="javadoc.java7.packagelist.exists"/>
<property name="javadoc.access" value="protected"/>
<property name="javadoc.charset" value="utf-8"/>
<property name="javadoc.dir" location="${common.dir}/build/docs"/>
@@ -253,7 +253,7 @@
</propertyset>
<patternset id="lucene.local.src.package.patterns"
- excludes="**/pom.xml,**/*.iml,**/*.jar,build/**,dist/**,benchmark/work/**,benchmark/temp/**,tools/javadoc/java6/**,tools/clover/**"
+ excludes="**/pom.xml,**/*.iml,**/*.jar,build/**,dist/**,benchmark/work/**,benchmark/temp/**,tools/javadoc/java7/**,tools/clover/**"
/>
<!-- Default exclude sources and javadoc jars from Ivy fetch to save time and bandwidth -->
@@ -282,10 +282,7 @@
<condition property="build.java.runtime" value="1.7">
<hasmethod classname="java.lang.Throwable" method="getSuppressed"/>
</condition>
- <condition property="build.java.runtime" value="1.6">
- <hasmethod classname="java.lang.String" method="isEmpty"/>
- </condition>
- <fail message="Minimum supported Java version is 1.6." unless="build.java.runtime"/>
+ <fail message="Minimum supported Java version is 1.7." unless="build.java.runtime"/>
<condition property="documentation-lint.supported">
<and>
@@ -1239,7 +1236,7 @@ ${tests-output}/junit4-*.suites - pe
]]></fail>
<echo>Code coverage with Atlassian Clover enabled.</echo>
- <ivy:cachepath organisation="com.cenqua.clover" module="clover" revision="2.6.3"
+ <ivy:cachepath organisation="com.cenqua.clover" module="clover" revision="3.1.10"
inline="true" conf="master" type="jar" pathid="clover.classpath"/>
<taskdef resource="cloverlib.xml" classpathref="clover.classpath" />
<mkdir dir="${clover.db.dir}"/>
@@ -1662,7 +1659,7 @@ ${tests-output}/junit4-*.suites - pe
<attribute name="overview" default="${src.dir}/overview.html"/>
<attribute name="linksource" default="no"/>
<sequential>
- <antcall target="download-java6-javadoc-packagelist"/>
+ <antcall target="download-java7-javadoc-packagelist"/>
<delete file="@{destdir}/stylesheet.css" failonerror="false"/>
<copy todir="@{destdir}" file="${prettify.dir}/prettify.js" overwrite="false" />
<record name="@{destdir}/log_javadoc.txt" action="start" append="no"/>
@@ -1691,8 +1688,8 @@ ${tests-output}/junit4-*.suites - pe
description="WARNING: This API is experimental and might change in incompatible ways in the next release."/>
<tag name="lucene.internal"
description="NOTE: This API is for internal purposes only and might change in incompatible ways in the next release."/>
- <link offline="true" packagelistLoc="${javadoc.dir}"/>
- <link offline="true" href="${javadoc.link}" packagelistLoc="${javadoc.packagelist.dir}/java6"/>
+ <link offline="true" packagelistLoc="${javadoc.dir}"/>
+ <link offline="true" href="${javadoc.link}" packagelistLoc="${javadoc.packagelist.dir}/java7"/>
<bottom><![CDATA[
<i>Copyright © ${year} Apache Software Foundation. All Rights Reserved.</i>
<script src='{@docRoot}/prettify.js' type='text/javascript'></script>
@@ -1762,10 +1759,10 @@ ${tests-output}/junit4-*.suites - pe
</sequential>
</macrodef>
- <target name="download-java6-javadoc-packagelist" unless="javadoc.java6.packagelist.exists">
- <mkdir dir="${javadoc.packagelist.dir}/java6"/>
+ <target name="download-java7-javadoc-packagelist" unless="javadoc.java7.packagelist.exists">
+ <mkdir dir="${javadoc.packagelist.dir}/java7"/>
<get src="${javadoc.link}/package-list"
- dest="${javadoc.packagelist.dir}/java6/package-list" ignoreerrors="true"/>
+ dest="${javadoc.packagelist.dir}/java7/package-list" ignoreerrors="true"/>
</target>
<!-- VALIDATION work -->
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/TokenStreamToAutomaton.java Wed Mar 20 16:27:13 2013
@@ -23,6 +23,7 @@ import java.io.OutputStreamWriter;
import java.io.Writer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
@@ -37,7 +38,9 @@ import org.apache.lucene.util.automaton.
/** Consumes a TokenStream and creates an {@link Automaton}
* where the transition labels are UTF8 bytes from the {@link
* TermToBytesRefAttribute}. Between tokens we insert
- * POS_SEP and for holes we insert HOLE. */
+ * POS_SEP and for holes we insert HOLE.
+ *
+ * @lucene.experimental */
public class TokenStreamToAutomaton {
/** Sole constructor. */
@@ -89,6 +92,7 @@ public class TokenStreamToAutomaton {
final TermToBytesRefAttribute termBytesAtt = in.addAttribute(TermToBytesRefAttribute.class);
final PositionIncrementAttribute posIncAtt = in.addAttribute(PositionIncrementAttribute.class);
final PositionLengthAttribute posLengthAtt = in.addAttribute(PositionLengthAttribute.class);
+ final OffsetAttribute offsetAtt = in.addAttribute(OffsetAttribute.class);
final BytesRef term = termBytesAtt.getBytesRef();
@@ -101,7 +105,7 @@ public class TokenStreamToAutomaton {
int pos = -1;
Position posData = null;
-
+ int maxOffset = 0;
while (in.incrementToken()) {
int posInc = posIncAtt.getPositionIncrement();
assert pos > -1 || posInc > 0;
@@ -157,13 +161,26 @@ public class TokenStreamToAutomaton {
state.addTransition(new Transition(term2.bytes[term2.offset + byteIDX] & 0xff, nextState));
state = nextState;
}
+
+ maxOffset = Math.max(maxOffset, offsetAtt.endOffset());
+ }
+
+ in.end();
+ State endState = null;
+ if (offsetAtt.endOffset() > maxOffset) {
+ endState = new State();
+ endState.setAccept(true);
}
pos++;
while (pos <= positions.getMaxPos()) {
posData = positions.get(pos);
if (posData.arriving != null) {
- posData.arriving.setAccept(true);
+ if (endState != null) {
+ posData.arriving.addTransition(new Transition(POS_SEP, endState));
+ } else {
+ posData.arriving.setAccept(true);
+ }
}
pos++;
}
Modified: lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java Wed Mar 20 16:27:13 2013
@@ -47,13 +47,6 @@ public abstract class Tokenizer extends
this.input = input;
}
- /** Construct a token stream processing the given input using the given AttributeSource. */
- protected Tokenizer(AttributeSource source, Reader input) {
- super(source);
- assert input != null: "input must not be null";
- this.input = input;
- }
-
/**
* {@inheritDoc}
* <p>