You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2014/11/01 20:59:57 UTC
svn commit: r1636025 - in /lucene/dev/branches/branch_5x/lucene: ./
analysis/common/src/java/org/apache/lucene/analysis/standard/
analysis/common/src/java/org/apache/lucene/analysis/standard/std40/
analysis/common/src/test/org/apache/lucene/analysis/st...
Author: rjernst
Date: Sat Nov 1 19:59:56 2014
New Revision: 1636025
URL: http://svn.apache.org/r1636025
Log:
LUCENE-6043: Fix backcompat support for UAX29URLEmailTokenizer
Modified:
lucene/dev/branches/branch_5x/lucene/CHANGES.txt
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1636025&r1=1636024&r2=1636025&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Sat Nov 1 19:59:56 2014
@@ -187,6 +187,9 @@ Bug Fixes
org.apache.lucene.index (Simon Willnauer, Robert Muir, Mike
McCandless)
+* LUCENE-6043: Fix backcompat support for UAX29URLEmailTokenizer
+ (Ryan Ernst)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java?rev=1636025&r1=1636024&r2=1636025&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java Sat Nov 1 19:59:56 2014
@@ -18,11 +18,14 @@ package org.apache.lucene.analysis.stand
*/
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopAnalyzer;
import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.standard.std40.UAX29URLEmailTokenizer40;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
@@ -84,15 +87,27 @@ public final class UAX29URLEmailAnalyzer
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
- final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer();
- src.setMaxTokenLength(maxTokenLength);
+ final Tokenizer src;
+ if (getVersion().onOrAfter(Version.LUCENE_4_7_0)) {
+ src = new UAX29URLEmailTokenizer();
+ ((UAX29URLEmailTokenizer)src).setMaxTokenLength(maxTokenLength);
+ } else {
+ src = new UAX29URLEmailTokenizer40();
+ ((UAX29URLEmailTokenizer40)src).setMaxTokenLength(maxTokenLength);
+ }
+
TokenStream tok = new StandardFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
protected void setReader(final Reader reader) throws IOException {
- src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
+ int m = UAX29URLEmailAnalyzer.this.maxTokenLength;
+ if (src instanceof UAX29URLEmailTokenizer) {
+ ((UAX29URLEmailTokenizer)src).setMaxTokenLength(m);
+ } else {
+ ((UAX29URLEmailTokenizer40)src).setMaxTokenLength(m);
+ }
super.setReader(reader);
}
};
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java?rev=1636025&r1=1636024&r2=1636025&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java Sat Nov 1 19:59:56 2014
@@ -17,10 +17,12 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.standard.std40.UAX29URLEmailTokenizer40;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
+import org.apache.lucene.util.Version;
-import java.io.Reader;
import java.util.Map;
/**
@@ -45,9 +47,15 @@ public class UAX29URLEmailTokenizerFacto
}
@Override
- public UAX29URLEmailTokenizer create(AttributeFactory factory) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(factory);
- tokenizer.setMaxTokenLength(maxTokenLength);
- return tokenizer;
+ public Tokenizer create(AttributeFactory factory) {
+ if (luceneMatchVersion.onOrAfter(Version.LUCENE_4_7_0)) {
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(factory);
+ tokenizer.setMaxTokenLength(maxTokenLength);
+ return tokenizer;
+ } else {
+ UAX29URLEmailTokenizer40 tokenizer40 = new UAX29URLEmailTokenizer40(factory);
+ tokenizer40.setMaxTokenLength(maxTokenLength);
+ return tokenizer40;
+ }
}
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java?rev=1636025&r1=1636024&r2=1636025&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/std40/UAX29URLEmailTokenizerImpl40.java Sat Nov 1 19:59:56 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.analysis.stand
* limitations under the License.
*/
-import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@Deprecated
@@ -3994,10 +3993,10 @@ public final class UAX29URLEmailTokenize
/* user code: */
/** Alphanumeric sequences */
- public static final int WORD_TYPE = UAX29URLEmailTokenizer.ALPHANUM;
+ public static final int WORD_TYPE = UAX29URLEmailTokenizer40.ALPHANUM;
/** Numbers */
- public static final int NUMERIC_TYPE = UAX29URLEmailTokenizer.NUM;
+ public static final int NUMERIC_TYPE = UAX29URLEmailTokenizer40.NUM;
/**
* Chars in class \p{Line_Break = Complex_Context} are from South East Asian
@@ -4007,19 +4006,19 @@ public final class UAX29URLEmailTokenize
* <p>
* See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
*/
- public static final int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer.SOUTHEAST_ASIAN;
+ public static final int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer40.SOUTHEAST_ASIAN;
- public static final int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer.IDEOGRAPHIC;
+ public static final int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer40.IDEOGRAPHIC;
- public static final int HIRAGANA_TYPE = UAX29URLEmailTokenizer.HIRAGANA;
+ public static final int HIRAGANA_TYPE = UAX29URLEmailTokenizer40.HIRAGANA;
- public static final int KATAKANA_TYPE = UAX29URLEmailTokenizer.KATAKANA;
+ public static final int KATAKANA_TYPE = UAX29URLEmailTokenizer40.KATAKANA;
- public static final int HANGUL_TYPE = UAX29URLEmailTokenizer.HANGUL;
+ public static final int HANGUL_TYPE = UAX29URLEmailTokenizer40.HANGUL;
- public static final int EMAIL_TYPE = UAX29URLEmailTokenizer.EMAIL;
+ public static final int EMAIL_TYPE = UAX29URLEmailTokenizer40.EMAIL;
- public static final int URL_TYPE = UAX29URLEmailTokenizer.URL;
+ public static final int URL_TYPE = UAX29URLEmailTokenizer40.URL;
public final int yychar()
{
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java?rev=1636025&r1=1636024&r2=1636025&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailAnalyzer.java Sat Nov 1 19:59:56 2014
@@ -19,7 +19,7 @@ package org.apache.lucene.analysis.stand
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.standard.UAX29URLEmailAnalyzer;
+import org.apache.lucene.util.Version;
import java.io.IOException;
import java.util.Arrays;
@@ -340,9 +340,15 @@ public class TestUAX29URLEmailAnalyzer e
new String[] { "<URL>" });
}
-
/** blast some random strings through the analyzer */
public void testRandomStrings() throws Exception {
checkRandomData(random(), new UAX29URLEmailAnalyzer(), 1000*RANDOM_MULTIPLIER);
}
+
+ public void testBackcompat40() throws IOException {
+ UAX29URLEmailAnalyzer a = new UAX29URLEmailAnalyzer();
+ a.setVersion(Version.LUCENE_4_6_1);
+ // this is just a test to see the correct unicode version is being used, not actually testing hebrew
+ assertAnalyzesTo(a, "×\"×", new String[] {"×", "×"});
+ }
}