You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by ma...@apache.org on 2009/11/09 05:02:02 UTC
svn commit: r833968 [2/10] - in /lucene/java/branches/flex_1458: ./ contrib/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/
contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/
contrib/analyzers/common/src/java/org/apache/...
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -149,6 +149,7 @@
return tree;
}
+ @Override
protected void decomposeInternal(final Token token) {
// get the hyphenation points
Hyphenation hyphens = hyphenator.hyphenate(token.termBuffer(), 0, token
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/CharVector.java Mon Nov 9 04:01:50 2009
@@ -81,6 +81,7 @@
n = 0;
}
+ @Override
public Object clone() {
CharVector cv = new CharVector((char[]) array.clone(), blockSize);
cv.n = this.n;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/Hyphen.java Mon Nov 9 04:01:50 2009
@@ -51,6 +51,7 @@
postBreak = null;
}
+ @Override
public String toString() {
if (noBreak == null && postBreak == null && preBreak != null
&& preBreak.equals("-")) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java Mon Nov 9 04:01:50 2009
@@ -466,6 +466,7 @@
insert(pattern, (char) k);
}
+ @Override
public void printStats() {
System.out.println("Value space size = "
+ Integer.toString(vspace.length()));
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java Mon Nov 9 04:01:50 2009
@@ -266,6 +266,7 @@
//
// EntityResolver methods
//
+ @Override
public InputSource resolveEntity(String publicId, String systemId) {
return HyphenationDTDGenerator.generateDTD();
}
@@ -278,6 +279,7 @@
* @see org.xml.sax.ContentHandler#startElement(java.lang.String,
* java.lang.String, java.lang.String, org.xml.sax.Attributes)
*/
+ @Override
public void startElement(String uri, String local, String raw,
Attributes attrs) {
if (local.equals("hyphen-char")) {
@@ -307,6 +309,7 @@
* @see org.xml.sax.ContentHandler#endElement(java.lang.String,
* java.lang.String, java.lang.String)
*/
+ @Override
public void endElement(String uri, String local, String raw) {
if (token.length() > 0) {
@@ -343,6 +346,7 @@
/**
* @see org.xml.sax.ContentHandler#characters(char[], int, int)
*/
+ @Override
public void characters(char ch[], int start, int length) {
StringBuffer chars = new StringBuffer(length);
chars.append(ch, start, length);
@@ -376,6 +380,7 @@
/**
* @see org.xml.sax.ErrorHandler#warning(org.xml.sax.SAXParseException)
*/
+ @Override
public void warning(SAXParseException ex) {
errMsg = "[Warning] " + getLocationString(ex) + ": " + ex.getMessage();
}
@@ -383,6 +388,7 @@
/**
* @see org.xml.sax.ErrorHandler#error(org.xml.sax.SAXParseException)
*/
+ @Override
public void error(SAXParseException ex) {
errMsg = "[Error] " + getLocationString(ex) + ": " + ex.getMessage();
}
@@ -390,6 +396,7 @@
/**
* @see org.xml.sax.ErrorHandler#fatalError(org.xml.sax.SAXParseException)
*/
+ @Override
public void fatalError(SAXParseException ex) throws SAXException {
errMsg = "[Fatal Error] " + getLocationString(ex) + ": " + ex.getMessage();
throw ex;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java Mon Nov 9 04:01:50 2009
@@ -348,6 +348,7 @@
return length;
}
+ @Override
public Object clone() {
TernaryTree t = new TernaryTree();
t.lo = (char[]) this.lo.clone();
@@ -483,6 +484,7 @@
child = c;
}
+ @Override
public Object clone() {
return new Item(parent, child);
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -140,6 +140,7 @@
* @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
* {@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
*/
+ @Override
public final TokenStream tokenStream( String fieldName, Reader reader ) {
TokenStream result = new StandardTokenizer( matchVersion, reader );
result = new StandardFilter( result );
@@ -161,6 +162,7 @@
* @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
* {@link StandardFilter}, {@link LowerCaseFilter}, and {@link StopFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -149,6 +149,7 @@
* {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}, and
* {@link GermanStemFilter}
*/
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
@@ -172,6 +173,7 @@
* {@link StandardFilter}, {@link LowerCaseFilter}, {@link StopFilter}, and
* {@link GermanStemFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
if (overridesTokenStreamMethod) {
// LUCENE-1678: force fallback to tokenStream() if we
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java Mon Nov 9 04:01:50 2009
@@ -61,6 +61,7 @@
/**
* @return Returns true for next token in the stream, or false at EOS
*/
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String term = termAtt.term();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -99,6 +99,7 @@
* @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
* {@link GreekLowerCaseFilter} and {@link StopFilter}
*/
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader)
{
TokenStream result = new StandardTokenizer(matchVersion, reader);
@@ -120,6 +121,7 @@
* @return A {@link TokenStream} built from a {@link StandardTokenizer} filtered with
* {@link GreekLowerCaseFilter} and {@link StopFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java Mon Nov 9 04:01:50 2009
@@ -37,6 +37,7 @@
termAtt = addAttribute(TermAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
char[] chArray = termAtt.termBuffer();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -152,6 +152,7 @@
* {@link ArabicNormalizationFilter},
* {@link PersianNormalizationFilter} and Persian Stop words
*/
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new ArabicLetterTokenizer(reader);
result = new LowerCaseFilter(result);
@@ -181,6 +182,7 @@
* {@link ArabicNormalizationFilter},
* {@link PersianNormalizationFilter} and Persian Stop words
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilter.java Mon Nov 9 04:01:50 2009
@@ -40,6 +40,7 @@
termAtt = addAttribute(TermAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
final int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java Mon Nov 9 04:01:50 2009
@@ -19,11 +19,9 @@
import java.io.IOException;
import java.util.Set;
-import java.util.HashSet;
import java.util.Arrays;
-import java.util.Iterator;
import org.apache.lucene.analysis.standard.StandardTokenizer; // for javadocs
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
@@ -37,17 +35,16 @@
* @see <a href="http://fr.wikipedia.org/wiki/%C3%89lision">Elision in Wikipedia</a>
*/
public final class ElisionFilter extends TokenFilter {
- private Set articles = null;
+ private CharArraySet articles = null;
private TermAttribute termAtt;
private static char[] apostrophes = {'\'', 'â'};
- public void setArticles(Set articles) {
- this.articles = new HashSet();
- Iterator iter = articles.iterator();
- while (iter.hasNext()) {
- this.articles.add(((String) iter.next()).toLowerCase());
- }
+ public void setArticles(Set<?> articles) {
+ if (articles instanceof CharArraySet)
+ this.articles = (CharArraySet) articles;
+ else
+ this.articles = new CharArraySet(articles, true);
}
/**
@@ -55,15 +52,15 @@
*/
protected ElisionFilter(TokenStream input) {
super(input);
- this.articles = new HashSet(Arrays.asList(new String[] { "l", "m", "t",
- "qu", "n", "s", "j" }));
+ this.articles = new CharArraySet(Arrays.asList(
+ "l", "m", "t", "qu", "n", "s", "j"), true);
termAtt = addAttribute(TermAttribute.class);
}
/**
* Constructs an elision filter with a Set of stop words
*/
- public ElisionFilter(TokenStream input, Set articles) {
+ public ElisionFilter(TokenStream input, Set<?> articles) {
super(input);
setArticles(articles);
termAtt = addAttribute(TermAttribute.class);
@@ -74,13 +71,14 @@
*/
public ElisionFilter(TokenStream input, String[] articles) {
super(input);
- setArticles(new HashSet(Arrays.asList(articles)));
+ this.articles = new CharArraySet(Arrays.asList(articles), true);
termAtt = addAttribute(TermAttribute.class);
}
/**
* Increments the {@link TokenStream} with a {@link TermAttribute} without elisioned start
*/
+ @Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
char[] termBuffer = termAtt.termBuffer();
@@ -100,7 +98,7 @@
// An apostrophe has been found. If the prefix is an article strip it off.
if (minPoz != Integer.MAX_VALUE
- && articles.contains(new String(termAtt.termBuffer(), 0, minPoz).toLowerCase())) {
+ && articles.contains(termAtt.termBuffer(), 0, minPoz)) {
termAtt.setTermBuffer(termAtt.termBuffer(), minPoz + 1, termAtt.termLength() - (minPoz + 1));
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -155,6 +155,7 @@
* filtered with {@link StandardFilter}, {@link StopFilter},
* {@link FrenchStemFilter} and {@link LowerCaseFilter}
*/
+ @Override
public final TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
@@ -179,6 +180,7 @@
* filtered with {@link StandardFilter}, {@link StopFilter},
* {@link FrenchStemFilter} and {@link LowerCaseFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java Mon Nov 9 04:01:50 2009
@@ -59,6 +59,7 @@
/**
* @return Returns true for the next token in the stream, or false at EOS
*/
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String term = termAtt.term();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java Mon Nov 9 04:01:50 2009
@@ -27,6 +27,7 @@
*/
public final class EmptyTokenStream extends TokenStream {
+ @Override
public final boolean incrementToken() throws IOException {
return false;
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAndSuffixAwareTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -35,11 +35,13 @@
public PrefixAndSuffixAwareTokenFilter(TokenStream prefix, TokenStream input, TokenStream suffix) {
super(suffix);
prefix = new PrefixAwareTokenFilter(prefix, input) {
+ @Override
public Token updateSuffixToken(Token suffixToken, Token lastInputToken) {
return PrefixAndSuffixAwareTokenFilter.this.updateInputToken(suffixToken, lastInputToken);
}
};
this.suffix = new PrefixAwareTokenFilter(prefix, suffix) {
+ @Override
public Token updateSuffixToken(Token suffixToken, Token lastInputToken) {
return PrefixAndSuffixAwareTokenFilter.this.updateSuffixToken(suffixToken, lastInputToken);
}
@@ -59,15 +61,18 @@
}
+ @Override
public final boolean incrementToken() throws IOException {
return suffix.incrementToken();
}
+ @Override
public void reset() throws IOException {
suffix.reset();
}
+ @Override
public void close() throws IOException {
suffix.close();
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -84,6 +84,7 @@
private boolean prefixExhausted;
+ @Override
public final boolean incrementToken() throws IOException {
if (!prefixExhausted) {
Token nextToken = getNextPrefixInputToken(reusableToken);
@@ -156,11 +157,13 @@
return suffixToken;
}
+ @Override
public void close() throws IOException {
prefix.close();
suffix.close();
}
+ @Override
public void reset() throws IOException {
super.reset();
if (prefix != null) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/SingleTokenTokenStream.java Mon Nov 9 04:01:50 2009
@@ -45,6 +45,7 @@
assert (tokenAtt instanceof Token);
}
+ @Override
public final boolean incrementToken() throws IOException {
if (exhausted) {
return false;
@@ -56,6 +57,7 @@
}
}
+ @Override
public void reset() throws IOException {
exhausted = false;
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -39,10 +39,16 @@
public static enum Side {
/** Get the n-gram from the front of the input */
- FRONT { public String getLabel() { return "front"; } },
+ FRONT {
+ @Override
+ public String getLabel() { return "front"; }
+ },
/** Get the n-gram from the end of the input */
- BACK { public String getLabel() { return "back"; } };
+ BACK {
+ @Override
+ public String getLabel() { return "back"; }
+ };
public abstract String getLabel();
@@ -117,6 +123,7 @@
this(input, Side.getSide(sideLabel), minGram, maxGram);
}
+ @Override
public final boolean incrementToken() throws IOException {
while (true) {
if (curTermBuffer == null) {
@@ -144,6 +151,7 @@
}
}
+ @Override
public void reset() throws IOException {
super.reset();
curTermBuffer = null;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java Mon Nov 9 04:01:50 2009
@@ -44,10 +44,16 @@
public static enum Side {
/** Get the n-gram from the front of the input */
- FRONT { public String getLabel() { return "front"; } },
+ FRONT {
+ @Override
+ public String getLabel() { return "front"; }
+ },
/** Get the n-gram from the end of the input */
- BACK { public String getLabel() { return "back"; } };
+ BACK {
+ @Override
+ public String getLabel() { return "back"; }
+ };
public abstract String getLabel();
@@ -174,14 +180,15 @@
}
/** Returns the next token in the stream, or null at EOS. */
+ @Override
public final boolean incrementToken() throws IOException {
clearAttributes();
// if we are just starting, read the whole input
if (!started) {
started = true;
char[] chars = new char[1024];
- input.read(chars);
- inStr = new String(chars).trim(); // remove any leading or trailing spaces
+ int charsRead = input.read(chars);
+ inStr = new String(chars, 0, charsRead).trim(); // remove any leading or trailing spaces
inLen = inStr.length();
gramSize = minGram;
}
@@ -205,17 +212,20 @@
return true;
}
+ @Override
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
+ @Override
public void reset(Reader input) throws IOException {
super.reset(input);
reset();
}
+ @Override
public void reset() throws IOException {
super.reset();
started = false;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -72,6 +72,7 @@
}
/** Returns the next token in the stream, or null at EOS. */
+ @Override
public final boolean incrementToken() throws IOException {
while (true) {
if (curTermBuffer == null) {
@@ -98,6 +99,7 @@
}
}
+ @Override
public void reset() throws IOException {
super.reset();
curTermBuffer = null;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java Mon Nov 9 04:01:50 2009
@@ -101,6 +101,7 @@
}
/** Returns the next token in the stream, or null at EOS. */
+ @Override
public final boolean incrementToken() throws IOException {
clearAttributes();
if (!started) {
@@ -128,17 +129,20 @@
return true;
}
+ @Override
public final void end() {
// set final offset
final int finalOffset = inLen;
this.offsetAtt.setOffset(finalOffset, finalOffset);
}
+ @Override
public void reset(Reader input) throws IOException {
super.reset(input);
reset();
}
+ @Override
public void reset() throws IOException {
super.reset();
started = false;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -188,6 +188,7 @@
* filtered with {@link StandardFilter}, {@link StopFilter},
* and {@link DutchStemFilter}
*/
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new StandardTokenizer(matchVersion, reader);
result = new StandardFilter(result);
@@ -210,6 +211,7 @@
* filtered with {@link StandardFilter}, {@link StopFilter},
* and {@link DutchStemFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
if (overridesTokenStreamMethod) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java Mon Nov 9 04:01:50 2009
@@ -69,6 +69,7 @@
/**
* Returns the next token in the stream, or null at EOS
*/
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String term = termAtt.term();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -59,6 +59,7 @@
this.encoder = encoder;
}
+ @Override
public boolean incrementToken() throws IOException {
boolean result = false;
if (input.incrementToken()) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -48,6 +48,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (typeAtt.type().equals(typeMatch))
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -43,6 +43,7 @@
payAtt = addAttribute(PayloadAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
byte[] data = new byte[8];
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -44,6 +44,7 @@
}
+ @Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
String type = typeAtt.type();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java Mon Nov 9 04:01:50 2009
@@ -62,6 +62,7 @@
this.positionIncrement = positionIncrement;
}
+ @Override
public final boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (firstTokenPositioned) {
@@ -75,6 +76,7 @@
}
}
+ @Override
public void reset() throws IOException {
super.reset();
firstTokenPositioned = false;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -172,6 +172,7 @@
return stopWords.size();
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result;
try {
@@ -198,6 +199,7 @@
TokenStream withStopFilter;
};
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
if (overridesTokenStreamMethod) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java Mon Nov 9 04:01:50 2009
@@ -88,6 +88,7 @@
termAtt = addAttribute(TermAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
int len = termAtt.termLength();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -97,6 +97,7 @@
* {@link RussianLowerCaseFilter}, {@link StopFilter},
* and {@link RussianStemFilter}
*/
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader)
{
TokenStream result = new RussianLetterTokenizer(reader);
@@ -121,6 +122,7 @@
* {@link RussianLowerCaseFilter}, {@link StopFilter},
* and {@link RussianStemFilter}
*/
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java Mon Nov 9 04:01:50 2009
@@ -49,6 +49,7 @@
* Collects only characters which satisfy
* {@link Character#isLetter(char)}.
*/
+ @Override
protected boolean isTokenChar(char c)
{
if (Character.isLetter(c) || (c >= '0' && c <= '9'))
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java Mon Nov 9 04:01:50 2009
@@ -51,6 +51,7 @@
/**
* Returns the next token in the stream, or null at EOS
*/
+ @Override
public final boolean incrementToken() throws IOException
{
if (input.incrementToken()) {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java Mon Nov 9 04:01:50 2009
@@ -98,6 +98,7 @@
this.outputUnigrams = outputUnigrams;
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream wrapped;
try {
@@ -116,6 +117,7 @@
ShingleFilter shingle;
};
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
if (overridesTokenStreamMethod) {
// LUCENE-1678: force fallback to tokenStream() if we
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java Mon Nov 9 04:01:50 2009
@@ -163,6 +163,7 @@
/* (non-Javadoc)
* @see org.apache.lucene.analysis.TokenStream#next()
*/
+ @Override
public final boolean incrementToken() throws IOException {
while (true) {
if (nextToken == null) {
@@ -328,6 +329,7 @@
return true;
}
+ @Override
public void reset() throws IOException {
super.reset();
nextToken = null;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java Mon Nov 9 04:01:50 2009
@@ -116,7 +116,7 @@
*/
public final class ShingleMatrixFilter extends TokenStream {
- public static Character defaultSpacerCharacter = new Character('_');
+ public static Character defaultSpacerCharacter = Character.valueOf('_');
public static TokenSettingsCodec defaultSettingsCodec = new OneDimensionalNonWeightedTokenSettingsCodec();
public static boolean ignoringSinglePrefixOrSuffixShingleByDefault = false;
@@ -345,6 +345,7 @@
private Set<List<Token>> shinglesSeen = new HashSet<List<Token>>();
+ @Override
public void reset() throws IOException {
permutations = null;
shinglesSeen.clear();
@@ -355,6 +356,7 @@
private Token reusableToken = new Token();
+ @Override
public final boolean incrementToken() throws IOException {
if (matrix == null) {
matrix = new Matrix();
@@ -743,6 +745,7 @@
return Matrix.this.columns.indexOf(this);
}
+ @Override
public String toString() {
return "Column{" +
"first=" + first +
@@ -807,6 +810,7 @@
// return ret;
// }
+ @Override
public String toString() {
return "Row{" +
"index=" + getIndex() +
@@ -863,6 +867,7 @@
};
}
+ @Override
public String toString() {
return "Matrix{" +
"columns=" + columns +
@@ -920,17 +925,21 @@
*/
public static class OneDimensionalNonWeightedTokenSettingsCodec extends TokenSettingsCodec {
+ @Override
public TokenPositioner getTokenPositioner(Token token) throws IOException {
return TokenPositioner.newColumn;
}
+ @Override
public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
}
+ @Override
public float getWeight(Token token) {
return 1f;
}
+ @Override
public void setWeight(Token token, float weight) {
}
@@ -944,6 +953,7 @@
*/
public static class TwoDimensionalNonWeightedSynonymTokenSettingsCodec extends TokenSettingsCodec {
+ @Override
public TokenPositioner getTokenPositioner(Token token) throws IOException {
if (token.getPositionIncrement() == 0) {
return TokenPositioner.newRow;
@@ -952,14 +962,17 @@
}
}
+ @Override
public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
throw new UnsupportedOperationException();
}
+ @Override
public float getWeight(Token token) {
return 1f;
}
+ @Override
public void setWeight(Token token, float weight) {
}
@@ -981,6 +994,7 @@
* @return the token flags int value as TokenPosition
* @throws IOException
*/
+ @Override
public TokenPositioner getTokenPositioner(Token token) throws IOException {
switch (token.getFlags()) {
case 0:
@@ -999,6 +1013,7 @@
* @param token
* @param tokenPositioner
*/
+ @Override
public void setTokenPositioner(Token token, TokenPositioner tokenPositioner) {
token.setFlags(tokenPositioner.getIndex());
}
@@ -1009,6 +1024,7 @@
* @param token
* @return 32 bit float
*/
+ @Override
public float getWeight(Token token) {
if (token.getPayload() == null || token.getPayload().getData() == null) {
return 1f;
@@ -1022,6 +1038,7 @@
* @param token
* @param weight
*/
+ @Override
public void setWeight(Token token, float weight) {
if (weight == 1f) {
token.setPayload(null);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/DateRecognizerSinkFilter.java Mon Nov 9 04:01:50 2009
@@ -49,6 +49,7 @@
this.dateFormat = dateFormat;
}
+ @Override
public boolean accept(AttributeSource source) {
if (termAtt == null) {
termAtt = source.addAttribute(TermAttribute.class);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenRangeSinkFilter.java Mon Nov 9 04:01:50 2009
@@ -37,6 +37,7 @@
}
+ @Override
public boolean accept(AttributeSource source) {
try {
if (count >= lower && count < upper){
@@ -48,6 +49,7 @@
}
}
+ @Override
public void reset() throws IOException {
count = 0;
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/sinks/TokenTypeSinkFilter.java Mon Nov 9 04:01:50 2009
@@ -29,6 +29,7 @@
this.typeToMatch = typeToMatch;
}
+ @Override
public boolean accept(AttributeSource source) {
if (typeAtt == null) {
typeAtt = source.addAttribute(TypeAttribute.class);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -43,6 +43,7 @@
this.matchVersion = matchVersion;
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream ts = new StandardTokenizer(matchVersion, reader);
ts = new StandardFilter(ts);
@@ -57,6 +58,7 @@
TokenStream result;
};
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
if (overridesTokenStreamMethod) {
// LUCENE-1678: force fallback to tokenStream() if we
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Mon Nov 9 04:01:50 2009
@@ -48,6 +48,7 @@
offsetAtt = addAttribute(OffsetAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
if (thaiState != null) {
int start = breaker.current();
@@ -82,6 +83,7 @@
return false;
}
+ @Override
public void reset() throws IOException {
super.reset();
thaiState = null;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java Mon Nov 9 04:01:50 2009
@@ -65,6 +65,7 @@
* convenience to show the behavior of the tokenizer
*/
private class JustChineseTokenizerAnalyzer extends Analyzer {
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new ChineseTokenizer(reader);
}
@@ -75,6 +76,7 @@
* convenience to show the behavior of the filter.
*/
private class JustChineseFilterAnalyzer extends Analyzer {
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new ChineseFilter(new WhitespaceTokenizer(reader));
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -51,6 +51,7 @@
private static byte[] patternsFileContent;
+ @Override
protected void setUp() throws Exception {
super.setUp();
getHyphenationPatternFileContents();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -51,6 +51,7 @@
* An input stream that always throws IOException for testing.
*/
private class UnreliableInputStream extends InputStream {
+ @Override
public int read() throws IOException {
throw new IOException();
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java Mon Nov 9 04:01:50 2009
@@ -76,6 +76,7 @@
super(matchVersion);
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new WhitespaceTokenizer(reader);
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java Mon Nov 9 04:01:50 2009
@@ -29,6 +29,7 @@
public class EdgeNGramTokenFilterTest extends BaseTokenStreamTestCase {
private TokenStream input;
+ @Override
public void setUp() throws Exception {
super.setUp();
input = new WhitespaceTokenizer(new StringReader("abcde"));
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java Mon Nov 9 04:01:50 2009
@@ -28,6 +28,7 @@
public class EdgeNGramTokenizerTest extends BaseTokenStreamTestCase {
private StringReader input;
+ @Override
public void setUp() throws Exception {
super.setUp();
input = new StringReader("abcde");
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java Mon Nov 9 04:01:50 2009
@@ -30,6 +30,7 @@
public class NGramTokenFilterTest extends BaseTokenStreamTestCase {
private TokenStream input;
+ @Override
public void setUp() throws Exception {
super.setUp();
input = new WhitespaceTokenizer(new StringReader("abcde"));
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Mon Nov 9 04:01:50 2009
@@ -30,6 +30,7 @@
public class NGramTokenizerTest extends BaseTokenStreamTestCase {
private StringReader input;
+ @Override
public void setUp() throws Exception {
super.setUp();
input = new StringReader("abcde");
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/nl/TestDutchStemmer.java Mon Nov 9 04:01:50 2009
@@ -134,6 +134,7 @@
public DutchSubclassAnalyzer(Version matchVersion) {
super(matchVersion);
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new WhitespaceTokenizer(reader);
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java Mon Nov 9 04:01:50 2009
@@ -69,6 +69,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
if (termAtt.term().equals("dogs"))
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java Mon Nov 9 04:01:50 2009
@@ -65,6 +65,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (input.incrementToken()) {
typeAtt.setType(String.valueOf(Character.toUpperCase(termAtt.termBuffer()[0])));
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/position/PositionFilterTest.java Mon Nov 9 04:01:50 2009
@@ -38,6 +38,7 @@
termAtt = addAttribute(TermAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
clearAttributes();
if (index < testToken.length) {
@@ -47,6 +48,7 @@
return false;
}
}
+ @Override
public void reset() {
index = 0;
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzerTest.java Mon Nov 9 04:01:50 2009
@@ -47,6 +47,7 @@
IndexReader reader;
QueryAutoStopWordAnalyzer protectedAnalyzer;
+ @Override
protected void setUp() throws Exception {
super.setUp();
dir = new RAMDirectory();
@@ -66,6 +67,7 @@
protectedAnalyzer = new QueryAutoStopWordAnalyzer(Version.LUCENE_CURRENT, appAnalyzer);
}
+ @Override
protected void tearDown() throws Exception {
reader.close();
super.tearDown();
@@ -154,6 +156,7 @@
super(matchVersion, new WhitespaceAnalyzer());
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new WhitespaceTokenizer(reader);
}
@@ -172,6 +175,7 @@
*/
private class NonreusableAnalyzer extends Analyzer {
int invocationCount = 0;
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
if (++invocationCount % 2 == 0)
return new WhitespaceTokenizer(reader);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -42,6 +42,7 @@
private File dataDir;
+ @Override
protected void setUp() throws Exception
{
super.setUp();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ru/TestRussianStem.java Mon Nov 9 04:01:50 2009
@@ -37,6 +37,7 @@
/**
* @see TestCase#setUp()
*/
+ @Override
protected void setUp() throws Exception
{
super.setUp();
@@ -73,6 +74,7 @@
/**
* @see TestCase#tearDown()
*/
+ @Override
protected void tearDown() throws Exception
{
super.tearDown();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapperTest.java Mon Nov 9 04:01:50 2009
@@ -221,6 +221,7 @@
super(org.apache.lucene.util.Version.LUCENE_CURRENT);
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new WhitespaceTokenizer(reader);
}
@@ -240,6 +241,7 @@
*/
private class NonreusableAnalyzer extends Analyzer {
int invocationCount = 0;
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
if (++invocationCount % 2 == 0)
return new WhitespaceTokenizer(reader);
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java Mon Nov 9 04:01:50 2009
@@ -48,6 +48,7 @@
this.typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
clearAttributes();
if (index < testToken.length) {
@@ -201,6 +202,7 @@
};
+ @Override
protected void setUp() throws Exception {
super.setUp();
testTokenWithHoles = new Token[] {
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java Mon Nov 9 04:01:50 2009
@@ -537,6 +537,7 @@
private Iterator iterator;
+ @Override
public boolean incrementToken() throws IOException {
if (iterator == null) {
iterator = tokens.iterator();
@@ -556,6 +557,7 @@
}
+ @Override
public void reset() throws IOException {
iterator = null;
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java Mon Nov 9 04:01:50 2009
@@ -76,6 +76,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public final boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
Modified: lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -90,6 +90,26 @@
new String[] { "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²", "quick", "brown", "fox", "jumped", "over", "lazy", "dogs" });
}
+ /*
+ * Test that position increments are adjusted correctly for stopwords.
+ */
+ public void testPositionIncrements() throws Exception {
+ ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT);
+
+ assertAnalyzesTo(analyzer, "à¸à¸£à¸°à¹à¸¢à¸à¸§à¹à¸² the à¸à¸£à¸°à¹à¸¢à¸à¸§à¹à¸²",
+ new String[] { "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²", "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²" },
+ new int[] { 0, 6, 14, 20 },
+ new int[] { 6, 9, 20, 23 },
+ new int[] { 1, 1, 2, 1 });
+
+ // case that a stopword is adjacent to thai text, with no whitespace
+ assertAnalyzesTo(analyzer, "à¸à¸£à¸°à¹à¸¢à¸à¸§à¹à¸²theà¸à¸£à¸°à¹à¸¢à¸à¸§à¹à¸²",
+ new String[] { "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²", "à¸à¸£à¸°à¹à¸¢à¸", "วà¹à¸²" },
+ new int[] { 0, 6, 12, 18 },
+ new int[] { 6, 9, 18, 21 },
+ new int[] { 1, 1, 2, 1 });
+ }
+
public void testReusableTokenStream() throws Exception {
ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_CURRENT);
assertAnalyzesToReuse(analyzer, "", new String[] {});
@@ -112,6 +132,7 @@
public ThaiSubclassAnalyzer(Version matchVersion) {
super(matchVersion);
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new WhitespaceTokenizer(reader);
}
Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java Mon Nov 9 04:01:50 2009
@@ -73,6 +73,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
clearAttributes();
buffer.setLength(0);
@@ -122,11 +123,13 @@
}
}
+ @Override
public void reset() throws IOException {
super.reset();
tokenStart = tokenEnd = 0;
}
+ @Override
public void reset(Reader input) throws IOException {
super.reset(input);
reset();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java Mon Nov 9 04:01:50 2009
@@ -60,7 +60,7 @@
*/
public class SmartChineseAnalyzer extends Analyzer {
- private final Set stopWords;
+ private final Set<?> stopWords;
private static final String DEFAULT_STOPWORD_FILE = "stopwords.txt";
@@ -144,6 +144,7 @@
this.matchVersion = matchVersion;
}
+ @Override
public TokenStream tokenStream(String fieldName, Reader reader) {
TokenStream result = new SentenceTokenizer(reader);
result = new WordTokenFilter(result);
@@ -163,6 +164,7 @@
TokenStream filteredTokenStream;
}
+ @Override
public TokenStream reusableTokenStream(String fieldName, Reader reader)
throws IOException {
SavedStreams streams = (SavedStreams) getPreviousTokenStream();
Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java Mon Nov 9 04:01:50 2009
@@ -17,7 +17,7 @@
package org.apache.lucene.analysis.cn.smart;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.lucene.analysis.cn.smart.hhmm.HHMMSegmenter;
@@ -45,18 +45,19 @@
* @param startOffset start offset of sentence
* @return {@link List} of {@link SegToken}
*/
- public List segmentSentence(String sentence, int startOffset) {
-
- List segTokenList = hhmmSegmenter.process(sentence);
-
- List result = new ArrayList();
+ public List<SegToken> segmentSentence(String sentence, int startOffset) {
+ List<SegToken> segTokenList = hhmmSegmenter.process(sentence);
// tokens from sentence, excluding WordType.SENTENCE_BEGIN and WordType.SENTENCE_END
- for (int i = 1; i < segTokenList.size() - 1; i++) {
- result.add(convertSegToken((SegToken) segTokenList.get(i), sentence, startOffset));
- }
+ List<SegToken> result = Collections.emptyList();
+
+ if (segTokenList.size() > 2) // if its not an empty sentence
+ result = segTokenList.subList(1, segTokenList.size() - 1);
+
+ for (SegToken st : result)
+ convertSegToken(st, sentence, startOffset);
+
return result;
-
}
/**
Modified: lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java?rev=833968&r1=833967&r2=833968&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (original)
+++ lucene/java/branches/flex_1458/contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java Mon Nov 9 04:01:50 2009
@@ -40,9 +40,9 @@
private WordSegmenter wordSegmenter;
- private Iterator tokenIter;
+ private Iterator<SegToken> tokenIter;
- private List tokenBuffer;
+ private List<SegToken> tokenBuffer;
private TermAttribute termAtt;
private OffsetAttribute offsetAtt;
@@ -61,6 +61,7 @@
typeAtt = addAttribute(TypeAttribute.class);
}
+ @Override
public boolean incrementToken() throws IOException {
if (tokenIter == null || !tokenIter.hasNext()) {
// there are no remaining tokens from the current sentence... are there more sentences?
@@ -78,15 +79,17 @@
return false; // no more sentences, end of stream!
}
}
-
+ // WordTokenFilter must clear attributes, as it is creating new tokens.
+ clearAttributes();
// There are remaining tokens from the current sentence, return the next one.
- SegToken nextWord = (SegToken) tokenIter.next();
+ SegToken nextWord = tokenIter.next();
termAtt.setTermBuffer(nextWord.charArray, 0, nextWord.charArray.length);
offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
typeAtt.setType("word");
return true;
}
+ @Override
public void reset() throws IOException {
super.reset();
tokenIter = null;