You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/07/20 03:01:52 UTC
svn commit: r1363608 [2/10] - in /lucene/dev/branches/LUCENE-2878: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/mave...
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java Fri Jul 20 01:01:39 2012
@@ -66,10 +66,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
@@ -80,10 +81,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -95,10 +97,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -110,10 +113,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @param strictAffixParsing Affix strict parsing enabled or not (an error while reading a rule causes exception or is ignored)
@@ -194,7 +198,6 @@ public class HunspellDictionary {
flagParsingStrategy = getFlagParsingStrategy(line);
}
}
- reader.close();
}
/**
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java Fri Jul 20 01:01:39 2012
@@ -299,13 +299,12 @@ public class HunspellStemmer {
// ================================================= Entry Point ===================================================
- /**
+ /*
* HunspellStemmer entry point. Accepts two arguments: location of affix file and location of dic file
*
* @param args Program arguments. Should contain location of affix file and location of dic file
* @throws IOException Can be thrown while reading from the files
* @throws ParseException Can be thrown while parsing the files
- */
public static void main(String[] args) throws IOException, ParseException {
boolean ignoreCase = false;
int offset = 0;
@@ -347,12 +346,10 @@ public class HunspellStemmer {
}
}
- /**
* Prints the results of the stemming of a word
*
* @param originalWord Word that has been stemmed
* @param stems Stems of the word
- */
private static void printStemResults(String originalWord, List<Stem> stems) {
StringBuilder builder = new StringBuilder().append("stem(").append(originalWord).append(")").append("\n");
@@ -382,13 +379,12 @@ public class HunspellStemmer {
System.out.println(builder);
}
- /**
* Simple utility to check if the given String has any text
*
* @param str String to check if it has any text
* @return {@code true} if the String has text, {@code false} otherwise
- */
private static boolean hasText(String str) {
return str != null && str.length() > 0;
}
+ */
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java Fri Jul 20 01:01:39 2012
@@ -23,7 +23,6 @@ import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
/**
@@ -54,7 +53,7 @@ public class PatternReplaceCharFilter ex
private final String replacement;
private Reader transformedInput;
- public PatternReplaceCharFilter(Pattern pattern, String replacement, CharStream in) {
+ public PatternReplaceCharFilter(Pattern pattern, String replacement, Reader in) {
super(in);
this.pattern = pattern;
this.replacement = replacement;
@@ -64,16 +63,29 @@ public class PatternReplaceCharFilter ex
public int read(char[] cbuf, int off, int len) throws IOException {
// Buffer all input on the first call.
if (transformedInput == null) {
- StringBuilder buffered = new StringBuilder();
- char [] temp = new char [1024];
- for (int cnt = input.read(temp); cnt > 0; cnt = input.read(temp)) {
- buffered.append(temp, 0, cnt);
- }
- transformedInput = new StringReader(processPattern(buffered).toString());
+ fill();
}
return transformedInput.read(cbuf, off, len);
}
+
+ private void fill() throws IOException {
+ StringBuilder buffered = new StringBuilder();
+ char [] temp = new char [1024];
+ for (int cnt = in.read(temp); cnt > 0; cnt = in.read(temp)) {
+ buffered.append(temp, 0, cnt);
+ }
+ transformedInput = new StringReader(processPattern(buffered).toString());
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (transformedInput == null) {
+ fill();
+ }
+
+ return transformedInput.read();
+ }
@Override
protected int correct(int currentOff) {
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java Fri Jul 20 01:01:39 2012
@@ -136,8 +136,8 @@ public final class PatternTokenizer exte
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
fillBuffer(str, input);
matcher.reset(str);
index = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java Fri Jul 20 01:01:39 2012
@@ -132,7 +132,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new CharArraySet(Version.LUCENE_50,
Arrays.asList(exceptions), false);
@@ -156,7 +156,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new char[exceptions.length][];
for (int i = 0; i < exceptions.length; i++)
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Fri Jul 20 01:01:39 2012
@@ -114,9 +114,9 @@ public final class ClassicAnalyzer exten
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java Fri Jul 20 01:01:39 2012
@@ -175,8 +175,8 @@ public final class ClassicTokenizer exte
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(reader);
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Fri Jul 20 01:01:39 2012
@@ -115,9 +115,9 @@ public final class StandardAnalyzer exte
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Fri Jul 20 01:01:39 2012
@@ -183,8 +183,8 @@ public final class StandardTokenizer ext
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(reader);
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java Fri Jul 20 01:01:39 2012
@@ -104,9 +104,9 @@ public final class UAX29URLEmailAnalyzer
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java Fri Jul 20 01:01:39 2012
@@ -162,8 +162,8 @@ public final class UAX29URLEmailTokenize
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(reader);
}
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java Fri Jul 20 01:01:39 2012
@@ -17,13 +17,15 @@ package org.apache.lucene.analysis.util;
* limitations under the License.
*/
-import org.apache.lucene.analysis.CharStream;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.CharFilter;
/**
- * Abstract parent class for analysis factories that create {@link CharStream}
+ * Abstract parent class for analysis factories that create {@link CharFilter}
* instances.
*/
public abstract class CharFilterFactory extends AbstractAnalysisFactory {
- public abstract CharStream create(CharStream input);
+ public abstract CharFilter create(Reader input);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java Fri Jul 20 01:01:39 2012
@@ -162,8 +162,8 @@ public abstract class CharTokenizer exte
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
bufferIndex = 0;
offset = 0;
dataLen = 0;
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java Fri Jul 20 01:01:39 2012
@@ -325,8 +325,8 @@ public final class WikipediaTokenizer ex
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(input);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/tartarus/snowball/SnowballProgram.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/tartarus/snowball/SnowballProgram.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/tartarus/snowball/SnowballProgram.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/org/tartarus/snowball/SnowballProgram.java Fri Jul 20 01:01:39 2012
@@ -435,7 +435,7 @@ public abstract class SnowballProgram {
bra > ket ||
ket > limit)
{
- System.err.println("faulty slice operation");
+ throw new IllegalArgumentException("faulty slice operation: bra=" + bra + ",ket=" + ket + ",limit=" + limit);
// FIXME: report error somehow.
/*
fprintf(stderr, "faulty slice operation:\n");
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/overview.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/overview.html?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/overview.html (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/java/overview.html Fri Jul 20 01:01:39 2012
@@ -24,7 +24,7 @@
For an introduction to Lucene's analysis API, see the {@link org.apache.lucene.analysis} package documentation.
</p>
<p>
- This module contains concrete components ({@link org.apache.lucene.analysis.charfilter.CharFilter}s,
+ This module contains concrete components ({@link org.apache.lucene.analysis.CharFilter}s,
{@link org.apache.lucene.analysis.Tokenizer}s, and ({@link org.apache.lucene.analysis.TokenFilter}s) for
analyzing different types of content. It also provides a number of {@link org.apache.lucene.analysis.Analyzer}s
for different languages that you can use to get started quickly.
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java Fri Jul 20 01:01:39 2012
@@ -29,7 +29,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.util._TestUtil;
@@ -46,7 +45,7 @@ public class HTMLStripCharFilterTest ext
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new HTMLStripCharFilter(CharReader.get(reader));
+ return new HTMLStripCharFilter(reader);
}
};
}
@@ -60,7 +59,7 @@ public class HTMLStripCharFilterTest ext
String gold = "\nthis is some text\n here is a link and " +
"another link. " +
"This is an entity: & plus a <. Here is an &. ";
- HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new StringReader(html)));
+ HTMLStripCharFilter reader = new HTMLStripCharFilter(new StringReader(html));
StringBuilder builder = new StringBuilder();
int ch = -1;
char [] goldArray = gold.toCharArray();
@@ -79,7 +78,7 @@ public class HTMLStripCharFilterTest ext
//Some sanity checks, but not a full-fledged check
public void testHTML() throws Exception {
InputStream stream = getClass().getResourceAsStream("htmlStripReaderTest.html");
- HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8")));
+ HTMLStripCharFilter reader = new HTMLStripCharFilter(new InputStreamReader(stream, "UTF-8"));
StringBuilder builder = new StringBuilder();
int ch = -1;
while ((ch = reader.read()) != -1){
@@ -96,7 +95,7 @@ public class HTMLStripCharFilterTest ext
public void testMSWord14GeneratedHTML() throws Exception {
InputStream stream = getClass().getResourceAsStream("MS-Word 14 generated.htm");
- HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new InputStreamReader(stream, "UTF-8")));
+ HTMLStripCharFilter reader = new HTMLStripCharFilter(new InputStreamReader(stream, "UTF-8"));
String gold = "This is a test";
StringBuilder builder = new StringBuilder();
int ch = 0;
@@ -117,7 +116,7 @@ public class HTMLStripCharFilterTest ext
String gold = "\u0393";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -132,7 +131,7 @@ public class HTMLStripCharFilterTest ext
String gold = " <foo> \u00DCbermensch = \u0393 bar \u0393";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -147,7 +146,7 @@ public class HTMLStripCharFilterTest ext
String gold = " <junk/> ! @ and â";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -161,7 +160,7 @@ public class HTMLStripCharFilterTest ext
String test = "aaa bbb <reserved ccc=\"ddddd\"> eeee </reserved> ffff <reserved ggg=\"hhhh\"/> <other/>";
Set<String> set = new HashSet<String>();
set.add("reserved");
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set);
+ Reader reader = new HTMLStripCharFilter(new StringReader(test), set);
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -346,7 +345,7 @@ public class HTMLStripCharFilterTest ext
for (int i = 0 ; i < testGold.length ; i += 2) {
String test = testGold[i];
String gold = testGold[i + 1];
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -370,7 +369,7 @@ public class HTMLStripCharFilterTest ext
testBuilder.append("-->foo");
String gold = "foo";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString())));
+ Reader reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString()));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -388,7 +387,7 @@ public class HTMLStripCharFilterTest ext
appendChars(testBuilder, HTMLStripCharFilter.getInitialBufferSize() + 500);
testBuilder.append("?>");
gold = "";
- reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString())));
+ reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString()));
ch = 0;
builder = new StringBuilder();
try {
@@ -406,7 +405,7 @@ public class HTMLStripCharFilterTest ext
appendChars(testBuilder, HTMLStripCharFilter.getInitialBufferSize() + 500);
testBuilder.append("/>");
gold = "";
- reader = new HTMLStripCharFilter(CharReader.get(new StringReader(testBuilder.toString())));
+ reader = new HTMLStripCharFilter(new StringReader(testBuilder.toString()));
ch = 0;
builder = new StringBuilder();
try {
@@ -430,7 +429,7 @@ public class HTMLStripCharFilterTest ext
private void processBuffer(String test, String assertMsg) throws IOException {
// System.out.println("-------------------processBuffer----------");
- Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader
+ Reader reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -448,7 +447,7 @@ public class HTMLStripCharFilterTest ext
String test = "<!--- three dashes, still a valid comment ---> ";
String gold = " ";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader
+ Reader reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(test)));//force the use of BufferedReader
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -464,7 +463,7 @@ public class HTMLStripCharFilterTest ext
public void doTestOffsets(String in) throws Exception {
- HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(in))));
+ HTMLStripCharFilter reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(in)));
int ch = 0;
int off = 0; // offset in the reader
int strOff = -1; // offset in the original string
@@ -491,7 +490,7 @@ public class HTMLStripCharFilterTest ext
static void assertLegalOffsets(String in) throws Exception {
int length = in.length();
- HTMLStripCharFilter reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(in))));
+ HTMLStripCharFilter reader = new HTMLStripCharFilter(new BufferedReader(new StringReader(in)));
int ch = 0;
int off = 0;
while ((ch = reader.read()) != -1) {
@@ -526,7 +525,7 @@ public class HTMLStripCharFilterTest ext
+ " alt = \"Alt: <!--#echo var='${IMAGE_CAPTION:<!--comment-->\\'Comment\\'}' -->\"\n\n"
+ " title=\"Title: <!--#echo var=\"IMAGE_CAPTION\"-->\">two";
String gold = "onetwo";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -540,7 +539,7 @@ public class HTMLStripCharFilterTest ext
test = "one<script><!-- <!--#config comment=\"<!-- \\\"comment\\\"-->\"--> --></script>two";
gold = "one\ntwo";
- reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ reader = new HTMLStripCharFilter(new StringReader(test));
ch = 0;
builder = new StringBuilder();
try {
@@ -557,7 +556,7 @@ public class HTMLStripCharFilterTest ext
public void testScriptQuotes() throws Exception {
String test = "one<script attr= bare><!-- action('<!-- comment -->', \"\\\"-->\\\"\"); --></script>two";
String gold = "one\ntwo";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -572,7 +571,7 @@ public class HTMLStripCharFilterTest ext
test = "hello<script><!-- f('<!--internal--></script>'); --></script>";
gold = "hello\n";
- reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ reader = new HTMLStripCharFilter(new StringReader(test));
ch = 0;
builder = new StringBuilder();
try {
@@ -591,7 +590,7 @@ public class HTMLStripCharFilterTest ext
String gold = "one<script no-value-attr></script>two";
Set<String> escapedTags = new HashSet<String>(Arrays.asList("SCRIPT"));
Reader reader = new HTMLStripCharFilter
- (CharReader.get(new StringReader(test)), escapedTags);
+ (new StringReader(test), escapedTags);
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -612,7 +611,7 @@ public class HTMLStripCharFilterTest ext
+ "-->\n"
+ "</style>two";
String gold = "one\ntwo";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -631,7 +630,7 @@ public class HTMLStripCharFilterTest ext
String gold = "one<style type=\"text/css\"></style>two";
Set<String> escapedTags = new HashSet<String>(Arrays.asList("STYLE"));
Reader reader = new HTMLStripCharFilter
- (CharReader.get(new StringReader(test)), escapedTags);
+ (new StringReader(test), escapedTags);
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -656,7 +655,7 @@ public class HTMLStripCharFilterTest ext
for (int i = 0 ; i < testGold.length ; i += 2) {
String test = testGold[i];
String gold = testGold[i + 1];
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
StringBuilder builder = new StringBuilder();
int ch = 0;
while ((ch = reader.read()) != -1){
@@ -671,7 +670,7 @@ public class HTMLStripCharFilterTest ext
String gold = "one<BR class='whatever'>two</\nBR\n>";
Set<String> escapedTags = new HashSet<String>(Arrays.asList("BR"));
Reader reader = new HTMLStripCharFilter
- (CharReader.get(new StringReader(test)), escapedTags);
+ (new StringReader(test), escapedTags);
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -688,7 +687,7 @@ public class HTMLStripCharFilterTest ext
public void testInlineTagsNoSpace() throws Exception {
String test = "one<sPAn class=\"invisible\">two<sup>2<sup>e</sup></sup>.</SpaN>three";
String gold = "onetwo2e.three";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -705,7 +704,7 @@ public class HTMLStripCharFilterTest ext
public void testCDATA() throws Exception {
String test = "one<![CDATA[<one><two>three<four></four></two></one>]]>two";
String gold = "one<one><two>three<four></four></two></one>two";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -720,7 +719,7 @@ public class HTMLStripCharFilterTest ext
test = "one<![CDATA[two<![CDATA[three]]]]><![CDATA[>four]]>five";
gold = "onetwo<![CDATA[three]]>fourfive";
- reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ reader = new HTMLStripCharFilter(new StringReader(test));
ch = 0;
builder = new StringBuilder();
try {
@@ -737,7 +736,7 @@ public class HTMLStripCharFilterTest ext
public void testUppercaseCharacterEntityVariants() throws Exception {
String test = " "-©>><<®&";
String gold = " \"-\u00A9>><<\u00AE&";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -754,7 +753,7 @@ public class HTMLStripCharFilterTest ext
public void testMSWordMalformedProcessingInstruction() throws Exception {
String test = "one<?xml:namespace prefix = o ns = \"urn:schemas-microsoft-com:office:office\" />two";
String gold = "onetwo";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -771,7 +770,7 @@ public class HTMLStripCharFilterTest ext
public void testSupplementaryCharsInTags() throws Exception {
String test = "one<ð©¬
è±éä¹æ¯ç>two<çæ¯ð©¬
>three çæ¯ð©¬
</çæ¯ð©¬
>four</ð©¬
è±éä¹æ¯ç>five<ð ð >six<ð ð />seven";
String gold = "one\ntwo\nthree çæ¯ð©¬
\nfour\nfive\nsix\nseven";
- Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)));
+ Reader reader = new HTMLStripCharFilter(new StringReader(test));
int ch = 0;
StringBuilder builder = new StringBuilder();
try {
@@ -822,7 +821,7 @@ public class HTMLStripCharFilterTest ext
}
}
Reader reader = new HTMLStripCharFilter
- (CharReader.get(new StringReader(text.toString())));
+ (new StringReader(text.toString()));
while (reader.read() != -1);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java Fri Jul 20 01:01:39 2012
@@ -29,8 +29,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -60,7 +59,7 @@ public class TestMappingCharFilter exten
}
public void testReaderReset() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
char[] buf = new char[10];
int len = cs.read(buf, 0, 10);
assertEquals( 1, len );
@@ -76,55 +75,55 @@ public class TestMappingCharFilter exten
}
public void testNothingChange() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"x"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to1() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "h" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "h" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"i"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to2() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "j" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "j" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"jj"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to3() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "k" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "k" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"kkk"}, new int[]{0}, new int[]{1}, 1);
}
public void test2to4() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "ll" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "ll" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"llll"}, new int[]{0}, new int[]{2}, 2);
}
public void test2to1() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "aa" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "aa" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"a"}, new int[]{0}, new int[]{2}, 2);
}
public void test3to1() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"b"}, new int[]{0}, new int[]{3}, 3);
}
public void test4to2() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[]{"cc"}, new int[]{0}, new int[]{4}, 4);
}
public void test5to0() throws Exception {
- CharStream cs = new MappingCharFilter( normMap, new StringReader( "empty" ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( "empty" ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts, new String[0], new int[]{}, new int[]{}, 5);
}
@@ -149,7 +148,7 @@ public class TestMappingCharFilter exten
//
public void testTokenStream() throws Exception {
String testString = "h i j k ll cccc bbb aa";
- CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( testString ) ) );
+ CharFilter cs = new MappingCharFilter( normMap, new StringReader( testString ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[]{"i","i","jj","kkk","llll","cc","b","a"},
@@ -171,8 +170,8 @@ public class TestMappingCharFilter exten
// h,8,9 => i,8,9
public void testChained() throws Exception {
String testString = "aaaa ll h";
- CharStream cs = new MappingCharFilter( normMap,
- new MappingCharFilter( normMap, CharReader.get( new StringReader( testString ) ) ) );
+ CharFilter cs = new MappingCharFilter( normMap,
+ new MappingCharFilter( normMap, new StringReader( testString ) ) );
TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
assertTokenStreamContents(ts,
new String[]{"a","llllllll","i"},
@@ -193,7 +192,7 @@ public class TestMappingCharFilter exten
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MappingCharFilter(normMap, CharReader.get(reader));
+ return new MappingCharFilter(normMap, reader);
}
};
@@ -219,7 +218,7 @@ public class TestMappingCharFilter exten
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MappingCharFilter(map, CharReader.get(reader));
+ return new MappingCharFilter(map, reader);
}
};
@@ -241,7 +240,7 @@ public class TestMappingCharFilter exten
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MappingCharFilter(map, CharReader.get(reader));
+ return new MappingCharFilter(map, reader);
}
};
int numRounds = 100;
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java Fri Jul 20 01:01:39 2012
@@ -23,7 +23,6 @@ import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -216,7 +215,7 @@ public class TestCJKAnalyzer extends Bas
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MappingCharFilter(norm, CharReader.get(reader));
+ return new MappingCharFilter(norm, reader);
}
};
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java Fri Jul 20 01:01:39 2012
@@ -48,7 +48,7 @@ public class CommonGramsFilterTest exten
assertTrue(cgf.incrementToken());
assertEquals("the_s", term.toString());
- wt.reset(new StringReader(input));
+ wt.setReader(new StringReader(input));
cgf.reset();
assertTrue(cgf.incrementToken());
assertEquals("How", term.toString());
@@ -66,7 +66,7 @@ public class CommonGramsFilterTest exten
assertTrue(nsf.incrementToken());
assertEquals("the_s", term.toString());
- wt.reset(new StringReader(input));
+ wt.setReader(new StringReader(input));
nsf.reset();
assertTrue(nsf.incrementToken());
assertEquals("How_the", term.toString());
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Fri Jul 20 01:01:39 2012
@@ -24,7 +24,6 @@ import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -240,7 +239,7 @@ public class TestCompoundWordTokenFilter
assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
assertTrue(tf.incrementToken());
assertEquals("Rind", termAtt.toString());
- wsTokenizer.reset(new StringReader("Rindfleischüberwachungsgesetz"));
+ wsTokenizer.setReader(new StringReader("Rindfleischüberwachungsgesetz"));
tf.reset();
assertTrue(tf.incrementToken());
assertEquals("Rindfleischüberwachungsgesetz", termAtt.toString());
@@ -327,7 +326,7 @@ public class TestCompoundWordTokenFilter
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MappingCharFilter(normMap, CharReader.get(reader));
+ return new MappingCharFilter(normMap, reader);
}
};
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java Fri Jul 20 01:01:39 2012
@@ -163,7 +163,7 @@ public class TestAnalyzers extends BaseT
filter.reset();
String highSurEndingUpper = "BogustermBoguster\ud801";
String highSurEndingLower = "bogustermboguster\ud801";
- tokenizer.reset(new StringReader(highSurEndingUpper));
+ tokenizer.setReader(new StringReader(highSurEndingUpper));
assertTokenStreamContents(filter, new String[] {highSurEndingLower});
assertTrue(filter.hasAttribute(CharTermAttribute.class));
char[] termBuffer = filter.getAttribute(CharTermAttribute.class).buffer();
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java Fri Jul 20 01:01:39 2012
@@ -1,11 +1,12 @@
package org.apache.lucene.analysis.core;
import java.io.Reader;
+import java.io.StringReader;
import java.nio.CharBuffer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockCharFilter;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
@@ -64,7 +65,7 @@ public class TestBugInSomething extends
checkAnalysisConsistency(random(), a, false, "wmgddzunizdomqyj");
}
- CharStream wrappedStream = new CharStream() {
+ CharFilter wrappedStream = new CharFilter(new StringReader("bogus")) {
@Override
public void mark(int readAheadLimit) {
@@ -107,8 +108,8 @@ public class TestBugInSomething extends
}
@Override
- public int correctOffset(int currentOff) {
- throw new UnsupportedOperationException("correctOffset(int)");
+ public int correct(int currentOff) {
+ throw new UnsupportedOperationException("correct(int)");
}
@Override
@@ -123,7 +124,7 @@ public class TestBugInSomething extends
};
public void testWrapping() throws Exception {
- CharStream cs = new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(wrappedStream);
+ CharFilter cs = new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(wrappedStream);
try {
cs.mark(1);
fail();
@@ -177,7 +178,7 @@ public class TestBugInSomething extends
cs.correctOffset(1);
fail();
} catch (Exception e) {
- assertEquals("correctOffset(int)", e.getMessage());
+ assertEquals("correct(int)", e.getMessage());
}
try {
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java Fri Jul 20 01:01:39 2012
@@ -44,8 +44,7 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.EmptyTokenizer;
import org.apache.lucene.analysis.MockGraphTokenFilter;
import org.apache.lucene.analysis.MockRandomLookaheadTokenFilter;
@@ -101,7 +100,7 @@ public class TestRandomChains extends Ba
static List<Constructor<? extends Tokenizer>> tokenizers;
static List<Constructor<? extends TokenFilter>> tokenfilters;
- static List<Constructor<? extends CharStream>> charfilters;
+ static List<Constructor<? extends CharFilter>> charfilters;
// TODO: fix those and remove
private static final Set<Class<?>> brokenComponents = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
@@ -170,7 +169,7 @@ public class TestRandomChains extends Ba
getClassesForPackage("org.apache.lucene.analysis", analysisClasses);
tokenizers = new ArrayList<Constructor<? extends Tokenizer>>();
tokenfilters = new ArrayList<Constructor<? extends TokenFilter>>();
- charfilters = new ArrayList<Constructor<? extends CharStream>>();
+ charfilters = new ArrayList<Constructor<? extends CharFilter>>();
for (final Class<?> c : analysisClasses) {
final int modifiers = c.getModifiers();
if (
@@ -179,7 +178,7 @@ public class TestRandomChains extends Ba
|| c.isSynthetic() || c.isAnonymousClass() || c.isMemberClass() || c.isInterface()
|| brokenComponents.contains(c)
|| c.isAnnotationPresent(Deprecated.class)
- || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharStream.class.isAssignableFrom(c))
+ || !(Tokenizer.class.isAssignableFrom(c) || TokenFilter.class.isAssignableFrom(c) || CharFilter.class.isAssignableFrom(c))
) {
continue;
}
@@ -197,10 +196,10 @@ public class TestRandomChains extends Ba
assertTrue(ctor.toGenericString() + " has unsupported parameter types",
allowedTokenFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
tokenfilters.add(castConstructor(TokenFilter.class, ctor));
- } else if (CharStream.class.isAssignableFrom(c)) {
+ } else if (CharFilter.class.isAssignableFrom(c)) {
assertTrue(ctor.toGenericString() + " has unsupported parameter types",
allowedCharFilterArgs.containsAll(Arrays.asList(ctor.getParameterTypes())));
- charfilters.add(castConstructor(CharStream.class, ctor));
+ charfilters.add(castConstructor(CharFilter.class, ctor));
} else {
fail("Cannot get here");
}
@@ -524,7 +523,6 @@ public class TestRandomChains extends Ba
allowedCharFilterArgs = Collections.newSetFromMap(new IdentityHashMap<Class<?>,Boolean>());
allowedCharFilterArgs.addAll(argProducers.keySet());
allowedCharFilterArgs.add(Reader.class);
- allowedCharFilterArgs.add(CharStream.class);
}
@SuppressWarnings("unchecked")
@@ -560,8 +558,6 @@ public class TestRandomChains extends Ba
Class<?> paramType = paramTypes[i];
if (paramType == Reader.class) {
args[i] = reader;
- } else if (paramType == CharStream.class) {
- args[i] = CharReader.get(reader);
} else {
args[i] = newRandomArg(random, paramType);
}
@@ -701,7 +697,7 @@ public class TestRandomChains extends Ba
int numFilters = random.nextInt(3);
for (int i = 0; i < numFilters; i++) {
while (true) {
- final Constructor<? extends CharStream> ctor = charfilters.get(random.nextInt(charfilters.size()));
+ final Constructor<? extends CharFilter> ctor = charfilters.get(random.nextInt(charfilters.size()));
final Object args[] = newCharFilterArgs(random, spec.reader, ctor.getParameterTypes());
reader = createComponent(ctor, args, descr);
if (reader != null) {
@@ -760,24 +756,16 @@ public class TestRandomChains extends Ba
}
}
- // wants charfilter to be a filterreader...
- // do *NOT*, do *NOT* refactor me to be a charfilter: LUCENE-3990
- static class CheckThatYouDidntReadAnythingReaderWrapper extends CharStream {
+ static class CheckThatYouDidntReadAnythingReaderWrapper extends CharFilter {
boolean readSomething;
- CharStream in;
CheckThatYouDidntReadAnythingReaderWrapper(Reader in) {
- this.in = CharReader.get(in);
+ super(in);
}
@Override
- public int correctOffset(int currentOff) {
- return in.correctOffset(currentOff);
- }
-
- @Override
- public void close() throws IOException {
- in.close();
+ public int correct(int currentOff) {
+ return currentOff; // we don't change any offsets
}
@Override
@@ -799,32 +787,12 @@ public class TestRandomChains extends Ba
}
@Override
- public void mark(int readAheadLimit) throws IOException {
- in.mark(readAheadLimit);
- }
-
- @Override
- public boolean markSupported() {
- return in.markSupported();
- }
-
- @Override
public int read(char[] cbuf) throws IOException {
readSomething = true;
return in.read(cbuf);
}
@Override
- public boolean ready() throws IOException {
- return in.ready();
- }
-
- @Override
- public void reset() throws IOException {
- in.reset();
- }
-
- @Override
public long skip(long n) throws IOException {
readSomething = true;
return in.skip(n);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -46,6 +49,19 @@ public class TestGermanLightStemFilter e
assertVocabulary(analyzer, getDataFile("delighttestdata.zip"), "delight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GermanLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sängerinnen", "sängerinnen");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -53,6 +56,19 @@ public class TestGermanMinimalStemFilter
checkOneTerm(analyzer, "äpfel", "apfel");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GermanMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sängerinnen", "sängerinnen");
+ }
+
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("deminimaltestdata.zip"), "deminimal.txt");
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,9 +23,13 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -58,6 +62,19 @@ public class TestGermanStemFilter extend
assertAnalyzesTo(analyzer, "", new String[] { "" });
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GermanStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "sängerinnen", "sängerinnen");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -46,6 +49,19 @@ public class TestFinnishLightStemFilter
assertVocabulary(analyzer, getDataFile("filighttestdata.zip"), "filight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("edeltäjistään"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "edeltäjistään", "edeltäjistään");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -176,6 +179,19 @@ public class TestFrenchLightStemFilter e
assertVocabulary(analyzer, getDataFile("frlighttestdata.zip"), "frlight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "chevaux", "chevaux");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), analyzer, 1000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -55,6 +58,19 @@ public class TestFrenchMinimalStemFilter
checkOneTerm(analyzer, "baron", "baron");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "chevaux", "chevaux");
+ }
+
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("frminimaltestdata.zip"), "frminimal.txt");
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
/**
* Simple tests for {@link GalicianMinimalStemmer}
@@ -50,6 +53,19 @@ public class TestGalicianMinimalStemFilt
checkOneTerm(a, "barcelonês", "barcelonês");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("elefantes"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "elefantes", "elefantes");
+ }
+
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER);
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java Fri Jul 20 01:01:39 2012
@@ -23,8 +23,11 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import static org.apache.lucene.analysis.VocabularyAssert.*;
@@ -46,6 +49,19 @@ public class TestHungarianLightStemFilte
assertVocabulary(analyzer, getDataFile("hulighttestdata.zip"), "hulight.txt");
}
+ public void testKeyword() throws IOException {
+ final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("babakocsi"), false);
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream sink = new KeywordMarkerFilter(source, exclusionSet);
+ return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
+ }
+ };
+ checkOneTerm(a, "babakocsi", "babakocsi");
+ }
+
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellDictionaryTest.java Fri Jul 20 01:01:39 2012
@@ -17,19 +17,98 @@ package org.apache.lucene.analysis.hunsp
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.Version;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.io.IOException;
import java.io.InputStream;
import java.text.ParseException;
import java.util.Arrays;
-import static junit.framework.Assert.assertEquals;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Assert;
+import org.junit.Test;
public class HunspellDictionaryTest extends LuceneTestCase {
+
+ private class CloseCheckInputStream extends InputStream {
+ private InputStream delegate;
+
+ private boolean closed = false;
+
+ public CloseCheckInputStream(InputStream delegate) {
+ super();
+ this.delegate = delegate;
+ }
+
+ public int read() throws IOException {
+ return delegate.read();
+ }
+
+ public int hashCode() {
+ return delegate.hashCode();
+ }
+
+ public int read(byte[] b) throws IOException {
+ return delegate.read(b);
+ }
+
+ public boolean equals(Object obj) {
+ return delegate.equals(obj);
+ }
+
+ public int read(byte[] b, int off, int len) throws IOException {
+ return delegate.read(b, off, len);
+ }
+
+ public long skip(long n) throws IOException {
+ return delegate.skip(n);
+ }
+
+ public String toString() {
+ return delegate.toString();
+ }
+
+ public int available() throws IOException {
+ return delegate.available();
+ }
+
+ public void close() throws IOException {
+ this.closed = true;
+ delegate.close();
+ }
+
+ public void mark(int readlimit) {
+ delegate.mark(readlimit);
+ }
+
+ public void reset() throws IOException {
+ delegate.reset();
+ }
+
+ public boolean markSupported() {
+ return delegate.markSupported();
+ }
+
+ public boolean isClosed() {
+ return this.closed;
+ }
+
+ }
+
+ @Test
+ public void testResourceCleanup() throws IOException, ParseException {
+ CloseCheckInputStream affixStream = new CloseCheckInputStream(getClass().getResourceAsStream("testCompressed.aff"));
+ CloseCheckInputStream dictStream = new CloseCheckInputStream(getClass().getResourceAsStream("testCompressed.dic"));
+
+ new HunspellDictionary(affixStream, dictStream, TEST_VERSION_CURRENT);
+
+ assertFalse(affixStream.isClosed());
+ assertFalse(dictStream.isClosed());
+
+ affixStream.close();
+ dictStream.close();
+
+ assertTrue(affixStream.isClosed());
+ assertTrue(dictStream.isClosed());
+ }
@Test
public void testHunspellDictionary_loadDicAff() throws IOException, ParseException {
@@ -40,7 +119,7 @@ public class HunspellDictionaryTest exte
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
-
+
affixStream.close();
dictStream.close();
}
@@ -54,7 +133,7 @@ public class HunspellDictionaryTest exte
assertEquals(3, dictionary.lookupSuffix(new char[]{'e'}, 0, 1).size());
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
-
+
affixStream.close();
dictStream.close();
}
@@ -69,7 +148,9 @@ public class HunspellDictionaryTest exte
assertEquals(1, dictionary.lookupPrefix(new char[]{'s'}, 0, 1).size());
assertEquals(1, dictionary.lookupWord(new char[]{'o', 'l', 'r'}, 0, 3).size());
//strict parsing disabled: malformed rule is not loaded
- assertNull(dictionary.lookupPrefix(new char[]{'a'}, 0, 1));
+ assertNull(dictionary.lookupPrefix(new char[]{'a'}, 0, 1));
+ affixStream.close();
+ dictStream.close();
affixStream = getClass().getResourceAsStream("testWrongAffixRule.aff");
dictStream = getClass().getResourceAsStream("test.dic");
@@ -81,7 +162,7 @@ public class HunspellDictionaryTest exte
Assert.assertEquals("The affix file contains a rule with less than five elements", e.getMessage());
Assert.assertEquals(23, e.getErrorOffset());
}
-
+
affixStream.close();
dictStream.close();
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Fri Jul 20 01:01:39 2012
@@ -68,7 +68,7 @@ public class TestPerFieldAnalyzerWrapper
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MockCharFilter(CharReader.get(reader), 7);
+ return new MockCharFilter(reader, 7);
}
};
assertAnalyzesTo(a, "ab",
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Fri Jul 20 01:01:39 2012
@@ -116,7 +116,7 @@ public class EdgeNGramTokenFilterTest ex
WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3});
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Fri Jul 20 01:01:39 2012
@@ -96,7 +96,7 @@ public class EdgeNGramTokenizerTest exte
public void testReset() throws Exception {
EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 3);
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
}
Modified: lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=1363608&r1=1363607&r2=1363608&view=diff
==============================================================================
--- lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/dev/branches/LUCENE-2878/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Fri Jul 20 01:01:39 2012
@@ -98,7 +98,7 @@ public class NGramTokenFilterTest extend
WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
NGramTokenFilter filter = new NGramTokenFilter(tokenizer, 1, 1);
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
- tokenizer.reset(new StringReader("abcde"));
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5});
}