You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2018/11/19 10:10:19 UTC
[2/2] lucene-solr:master: LUCENE-8497: Replace
MultiTermAwareComponent with normalize() method
LUCENE-8497: Replace MultiTermAwareComponent with normalize() method
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65486442
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65486442
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65486442
Branch: refs/heads/master
Commit: 65486442c4a893a17cd70c9a865fa1af7c160aa3
Parents: 759af01
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Nov 19 09:47:52 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Nov 19 10:09:37 2018 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 ++
lucene/MIGRATE.txt | 5 ++
.../ar/ArabicNormalizationFilterFactory.java | 10 +--
.../bn/BengaliNormalizationFilterFactory.java | 14 ++-
.../charfilter/MappingCharFilterFactory.java | 13 ++-
.../analysis/cjk/CJKWidthFilterFactory.java | 10 +--
.../ckb/SoraniNormalizationFilterFactory.java | 10 +--
.../core/DecimalDigitFilterFactory.java | 10 +--
.../analysis/core/LowerCaseFilterFactory.java | 10 +--
.../analysis/core/UpperCaseFilterFactory.java | 10 +--
.../lucene/analysis/custom/CustomAnalyzer.java | 17 +---
.../de/GermanNormalizationFilterFactory.java | 10 +--
.../el/GreekLowerCaseFilterFactory.java | 10 +--
.../analysis/fa/PersianCharFilterFactory.java | 11 +--
.../fa/PersianNormalizationFilterFactory.java | 12 ++-
.../ga/IrishLowerCaseFilterFactory.java | 8 +-
.../hi/HindiNormalizationFilterFactory.java | 10 +--
.../in/IndicNormalizationFilterFactory.java | 10 +--
.../ASCIIFoldingFilterFactory.java | 28 +++---
.../ScandinavianFoldingFilterFactory.java | 15 ++--
.../ScandinavianNormalizationFilterFactory.java | 13 ++-
.../miscellaneous/TrimFilterFactory.java | 10 +--
.../PatternReplaceCharFilterFactory.java | 11 +--
.../sr/SerbianNormalizationFilterFactory.java | 9 +-
.../tr/TurkishLowerCaseFilterFactory.java | 8 +-
.../lucene/analysis/util/CharFilterFactory.java | 9 ++
.../analysis/util/ElisionFilterFactory.java | 8 +-
.../analysis/util/MultiTermAwareComponent.java | 36 --------
.../analysis/util/TokenFilterFactory.java | 9 ++
.../lucene/analysis/core/TestFactories.java | 28 ------
.../analysis/custom/TestCustomAnalyzer.java | 33 ++-----
.../TestAsciiFoldingFilterFactory.java | 7 +-
.../analysis/icu/ICUFoldingFilterFactory.java | 13 ++-
.../icu/ICUNormalizer2CharFilterFactory.java | 12 +--
.../icu/ICUNormalizer2FilterFactory.java | 13 ++-
.../analysis/icu/ICUTransformFilterFactory.java | 13 ++-
.../JapaneseIterationMarkCharFilterFactory.java | 15 ++--
.../lucene/analysis/ja/TestFactories.java | 28 ------
...tJapaneseIterationMarkCharFilterFactory.java | 10 +--
.../apache/solr/analysis/TokenizerChain.java | 39 ++++++--
.../solr/schema/FieldTypePluginLoader.java | 59 +------------
.../org/apache/solr/schema/MultiTermTest.java | 93 --------------------
.../solr/search/TestFoldingMultitermQuery.java | 12 ---
43 files changed, 205 insertions(+), 511 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a0cfa71..16e9107 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -87,6 +87,11 @@ API Changes
* LUCENE-8513: MultiFields.getFields is now removed. Please avoid this class,
and Fields in general, when possible. (David Smiley)
+* LUCENE-8497: MultiTermAwareComponent has been removed, and in its place
+ TokenFilterFactory and CharFilterFactory now expose type-safe normalize()
+ methods. This decouples normalization from tokenization entirely.
+ (Mayya Sharipova, Alan Woodward)
+
Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 0515ff8..53187aa 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -145,3 +145,8 @@ use a TokenFilter chain as you would with any other Tokenizer.
Both Highlighter and FastVectorHighlighter need a custom WeightedSpanTermExtractor or FieldQuery respectively
in order to support ToParent/ToChildBlockJoinQuery.
+
+## MultiTermAwareComponent replaced by CharFilterFactory#normalize() and TokenFilterFactory#normalize() ##
+
+Normalization is now type-safe, with CharFilterFactory#normalize() returning a Reader and
+TokenFilterFactory#normalize() returning a TokenFilter.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java
index 8e8a83b..3bf2227 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.ar;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -36,7 +34,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
*
* @since 3.1
*/
-public class ArabicNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class ArabicNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new ArabicNormalizationFilterFactory */
public ArabicNormalizationFilterFactory(Map<String,String> args) {
@@ -47,12 +45,12 @@ public class ArabicNormalizationFilterFactory extends TokenFilterFactory impleme
}
@Override
- public ArabicNormalizationFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new ArabicNormalizationFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
index 045b585..d537f22 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliNormalizationFilterFactory.java
@@ -17,13 +17,11 @@
package org.apache.lucene.analysis.bn;
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Map;
-
/**
* Factory for {@link BengaliNormalizationFilter}.
* <pre class="prettyprint">
@@ -35,7 +33,7 @@ import java.util.Map;
* </fieldType></pre>
* @since 7.1.0
*/
-public class BengaliNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class BengaliNormalizationFilterFactory extends TokenFilterFactory {
public BengaliNormalizationFilterFactory(Map<String,String> args) {
super(args);
@@ -48,9 +46,9 @@ public class BengaliNormalizationFilterFactory extends TokenFilterFactory implem
public TokenStream create(TokenStream input) {
return new BengaliNormalizationFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
index 7aba3b2..de0e22b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
@@ -25,9 +25,7 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
@@ -44,7 +42,7 @@ import org.apache.lucene.analysis.util.ResourceLoaderAware;
* @since Solr 1.4
*/
public class MappingCharFilterFactory extends CharFilterFactory implements
- ResourceLoaderAware, MultiTermAwareComponent {
+ ResourceLoaderAware {
protected NormalizeCharMap normMap;
private final String mapping;
@@ -86,6 +84,11 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
return normMap == null ? input : new MappingCharFilter(normMap,input);
}
+ @Override
+ public Reader normalize(Reader input) {
+ return create(input);
+ }
+
// "source" => "target"
static Pattern p = Pattern.compile( "\"(.*)\"\\s*=>\\s*\"(.*)\"\\s*$" );
@@ -131,8 +134,4 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
return new String( out, 0, writePos );
}
- @Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
index fbad5f4..6754bf5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKWidthFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.cjk;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -37,7 +35,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.6.0
*/
-public class CJKWidthFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class CJKWidthFilterFactory extends TokenFilterFactory {
/** Creates a new CJKWidthFilterFactory */
public CJKWidthFilterFactory(Map<String,String> args) {
@@ -51,9 +49,9 @@ public class CJKWidthFilterFactory extends TokenFilterFactory implements MultiTe
public TokenStream create(TokenStream input) {
return new CJKWidthFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
index 1c5781d..b552ae9 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.ckb;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 4.7.0
*/
-public class SoraniNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class SoraniNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new SoraniNormalizationFilterFactory */
public SoraniNormalizationFilterFactory(Map<String,String> args) {
@@ -46,12 +44,12 @@ public class SoraniNormalizationFilterFactory extends TokenFilterFactory impleme
}
@Override
- public SoraniNormalizationFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new SoraniNormalizationFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/DecimalDigitFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/DecimalDigitFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/DecimalDigitFilterFactory.java
index a7606e2..2db789a 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/DecimalDigitFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/DecimalDigitFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.core;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 5.4.0
*/
-public class DecimalDigitFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class DecimalDigitFilterFactory extends TokenFilterFactory {
/** Creates a new DecimalDigitFilterFactory */
public DecimalDigitFilterFactory(Map<String,String> args) {
@@ -46,12 +44,12 @@ public class DecimalDigitFilterFactory extends TokenFilterFactory implements Mul
}
@Override
- public DecimalDigitFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new DecimalDigitFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java
index 52a2090..d0e0791 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.core;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -36,7 +34,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
*
* @since 3.1
*/
-public class LowerCaseFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class LowerCaseFilterFactory extends TokenFilterFactory {
/** Creates a new LowerCaseFilterFactory */
public LowerCaseFilterFactory(Map<String,String> args) {
@@ -47,12 +45,12 @@ public class LowerCaseFilterFactory extends TokenFilterFactory implements MultiT
}
@Override
- public LowerCaseFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new LowerCaseFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java
index 9e3b372..24515d2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/UpperCaseFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.core;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -40,7 +38,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* general search matching
* @since 4.7.0
*/
-public class UpperCaseFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class UpperCaseFilterFactory extends TokenFilterFactory {
/** Creates a new UpperCaseFilterFactory */
public UpperCaseFilterFactory(Map<String,String> args) {
@@ -51,12 +49,12 @@ public class UpperCaseFilterFactory extends TokenFilterFactory implements MultiT
}
@Override
- public UpperCaseFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new UpperCaseFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
index db5a29d..5a8ae9f 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/custom/CustomAnalyzer.java
@@ -40,7 +40,6 @@ import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.FilesystemResourceLoader;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -143,10 +142,7 @@ public final class CustomAnalyzer extends Analyzer {
@Override
protected Reader initReaderForNormalization(String fieldName, Reader reader) {
for (CharFilterFactory charFilter : charFilters) {
- if (charFilter instanceof MultiTermAwareComponent) {
- charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
- reader = charFilter.create(reader);
- }
+ reader = charFilter.normalize(reader);
}
return reader;
}
@@ -164,17 +160,8 @@ public final class CustomAnalyzer extends Analyzer {
@Override
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = in;
- // tokenizers can return a tokenfilter if the tokenizer does normalization,
- // although this is really bogus/abstraction violation...
- if (tokenizer instanceof MultiTermAwareComponent) {
- TokenFilterFactory filter = (TokenFilterFactory) ((MultiTermAwareComponent) tokenizer).getMultiTermComponent();
- result = filter.create(result);
- }
for (TokenFilterFactory filter : tokenFilters) {
- if (filter instanceof MultiTermAwareComponent) {
- filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
- result = filter.create(result);
- }
+ result = filter.normalize(result);
}
return result;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanNormalizationFilterFactory.java
index 77c3f55..bed3203 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.de;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -36,7 +34,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.6.0
*/
-public class GermanNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class GermanNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new GermanNormalizationFilterFactory */
public GermanNormalizationFilterFactory(Map<String,String> args) {
@@ -50,9 +48,9 @@ public class GermanNormalizationFilterFactory extends TokenFilterFactory impleme
public TokenStream create(TokenStream input) {
return new GermanNormalizationFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java
index c47bfd3..17e7adf 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.el;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -36,7 +34,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
*
* @since 3.1
*/
-public class GreekLowerCaseFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class GreekLowerCaseFilterFactory extends TokenFilterFactory {
/** Creates a new GreekLowerCaseFilterFactory */
public GreekLowerCaseFilterFactory(Map<String,String> args) {
@@ -47,13 +45,13 @@ public class GreekLowerCaseFilterFactory extends TokenFilterFactory implements M
}
@Override
- public GreekLowerCaseFilter create(TokenStream in) {
+ public TokenStream create(TokenStream in) {
return new GreekLowerCaseFilter(in);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java
index 8a661f3..1f59ecf 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java
@@ -20,10 +20,7 @@ package org.apache.lucene.analysis.fa;
import java.io.Reader;
import java.util.Map;
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
/**
* Factory for {@link PersianCharFilter}.
@@ -37,7 +34,7 @@ import org.apache.lucene.analysis.util.MultiTermAwareComponent;
*
* @since 3.1
*/
-public class PersianCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
+public class PersianCharFilterFactory extends CharFilterFactory {
/** Creates a new PersianCharFilterFactory */
public PersianCharFilterFactory(Map<String,String> args) {
@@ -48,12 +45,12 @@ public class PersianCharFilterFactory extends CharFilterFactory implements Multi
}
@Override
- public CharFilter create(Reader input) {
+ public Reader create(Reader input) {
return new PersianCharFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public Reader normalize(Reader input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilterFactory.java
index 9775f48..256b501 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.fa;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -37,7 +35,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
*
* @since 3.1
*/
-public class PersianNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class PersianNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new PersianNormalizationFilterFactory */
public PersianNormalizationFilterFactory(Map<String,String> args) {
@@ -48,13 +46,13 @@ public class PersianNormalizationFilterFactory extends TokenFilterFactory implem
}
@Override
- public PersianNormalizationFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new PersianNormalizationFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishLowerCaseFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishLowerCaseFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishLowerCaseFilterFactory.java
index 5ece365..abaad3d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishLowerCaseFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishLowerCaseFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.ga;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.6.0
*/
-public class IrishLowerCaseFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class IrishLowerCaseFilterFactory extends TokenFilterFactory {
/** Creates a new IrishLowerCaseFilterFactory */
public IrishLowerCaseFilterFactory(Map<String,String> args) {
@@ -52,7 +50,7 @@ public class IrishLowerCaseFilterFactory extends TokenFilterFactory implements M
// this will 'mostly work', except for special cases, just like most other filters
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilterFactory.java
index 80d50a4..fbc0a24 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.hi;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.1.0
*/
-public class HindiNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class HindiNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new HindiNormalizationFilterFactory */
public HindiNormalizationFilterFactory(Map<String,String> args) {
@@ -49,9 +47,9 @@ public class HindiNormalizationFilterFactory extends TokenFilterFactory implemen
public TokenStream create(TokenStream input) {
return new HindiNormalizationFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilterFactory.java
index 07ac214..e4a6900 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/in/IndicNormalizationFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.in;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.1.0
*/
-public class IndicNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class IndicNormalizationFilterFactory extends TokenFilterFactory {
/** Creates a new IndicNormalizationFilterFactory */
public IndicNormalizationFilterFactory(Map<String,String> args) {
@@ -49,9 +47,9 @@ public class IndicNormalizationFilterFactory extends TokenFilterFactory implemen
public TokenStream create(TokenStream input) {
return new IndicNormalizationFilter(input);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
index b42c878..4dabd88 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ASCIIFoldingFilterFactory.java
@@ -17,13 +17,10 @@
package org.apache.lucene.analysis.miscellaneous;
-import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link ASCIIFoldingFilter}.
@@ -37,7 +34,7 @@ import org.apache.lucene.analysis.TokenStream;
*
* @since 3.1
*/
-public class ASCIIFoldingFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class ASCIIFoldingFilterFactory extends TokenFilterFactory {
private static final String PRESERVE_ORIGINAL = "preserveOriginal";
private final boolean preserveOriginal;
@@ -52,23 +49,18 @@ public class ASCIIFoldingFilterFactory extends TokenFilterFactory implements Mul
}
@Override
- public ASCIIFoldingFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new ASCIIFoldingFilter(input, preserveOriginal);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- if (preserveOriginal) {
- // The main use-case for using preserveOriginal is to match regardless of
- // case but to give better scores to exact matches. Since most multi-term
- // queries return constant scores anyway, the multi-term component only
- // emits the folded token
- Map<String, String> args = new HashMap<>(getOriginalArgs());
- args.remove(PRESERVE_ORIGINAL);
- return new ASCIIFoldingFilterFactory(args);
- } else {
- return this;
- }
+ public TokenStream normalize(TokenStream input) {
+ // The main use-case for using preserveOriginal is to match regardless of
+ // case and to give better scores to exact matches. Since most multi-term
+ // queries return constant scores anyway, for normalization we
+ // emit only the folded token
+ return new ASCIIFoldingFilter(input, false);
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianFoldingFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianFoldingFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianFoldingFilterFactory.java
index 92f9465..ae6465d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianFoldingFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianFoldingFilterFactory.java
@@ -17,13 +17,11 @@
package org.apache.lucene.analysis.miscellaneous;
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Map;
-
/**
* Factory for {@link ScandinavianFoldingFilter}.
* <pre class="prettyprint">
@@ -35,8 +33,7 @@ import java.util.Map;
* </fieldType></pre>
* @since 4.4.0
*/
-public class ScandinavianFoldingFilterFactory extends TokenFilterFactory
- implements MultiTermAwareComponent {
+public class ScandinavianFoldingFilterFactory extends TokenFilterFactory {
public ScandinavianFoldingFilterFactory(Map<String,String> args) {
super(args);
@@ -46,12 +43,12 @@ public class ScandinavianFoldingFilterFactory extends TokenFilterFactory
}
@Override
- public ScandinavianFoldingFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new ScandinavianFoldingFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianNormalizationFilterFactory.java
index 80f0a8f..f58d42d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/ScandinavianNormalizationFilterFactory.java
@@ -17,13 +17,11 @@
package org.apache.lucene.analysis.miscellaneous;
+import java.util.Map;
+
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import java.util.Map;
-
/**
* Factory for {@link org.apache.lucene.analysis.miscellaneous.ScandinavianNormalizationFilter}.
* <pre class="prettyprint">
@@ -35,8 +33,7 @@ import java.util.Map;
* </fieldType></pre>
* @since 4.4.0
*/
-public class ScandinavianNormalizationFilterFactory extends TokenFilterFactory
- implements MultiTermAwareComponent {
+public class ScandinavianNormalizationFilterFactory extends TokenFilterFactory {
public ScandinavianNormalizationFilterFactory(Map<String, String> args) {
super(args);
@@ -51,7 +48,7 @@ public class ScandinavianNormalizationFilterFactory extends TokenFilterFactory
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
index 63abf07..4f00e41 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/TrimFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.miscellaneous;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -38,7 +36,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
*
* @since 3.1
*/
-public class TrimFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class TrimFilterFactory extends TokenFilterFactory {
/** Creates a new TrimFilterFactory */
public TrimFilterFactory(Map<String,String> args) {
@@ -49,12 +47,12 @@ public class TrimFilterFactory extends TokenFilterFactory implements MultiTermAw
}
@Override
- public TrimFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new TrimFilter(input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java
index 5319cbc..6b71c76 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java
@@ -21,10 +21,7 @@ import java.io.Reader;
import java.util.Map;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
/**
* Factory for {@link PatternReplaceCharFilter}.
@@ -39,7 +36,7 @@ import org.apache.lucene.analysis.util.MultiTermAwareComponent;
*
* @since Solr 3.1
*/
-public class PatternReplaceCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
+public class PatternReplaceCharFilterFactory extends CharFilterFactory {
private final Pattern pattern;
private final String replacement;
@@ -54,12 +51,12 @@ public class PatternReplaceCharFilterFactory extends CharFilterFactory implement
}
@Override
- public CharFilter create(Reader input) {
+ public Reader create(Reader input) {
return new PatternReplaceCharFilter(pattern, replacement, input);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public Reader normalize(Reader input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
index f5b608d..c47d471 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianNormalizationFilterFactory.java
@@ -21,8 +21,6 @@ import java.util.Arrays;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -38,7 +36,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 5.0.0
*/
-public class SerbianNormalizationFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class SerbianNormalizationFilterFactory extends TokenFilterFactory {
final String haircut;
/** Creates a new SerbianNormalizationFilterFactory */
@@ -61,8 +59,7 @@ public class SerbianNormalizationFilterFactory extends TokenFilterFactory implem
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilterFactory.java
index 5565a2a..be43d34 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishLowerCaseFilterFactory.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.tr;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
@@ -35,7 +33,7 @@ import org.apache.lucene.analysis.util.TokenFilterFactory;
* </fieldType></pre>
* @since 3.1.0
*/
-public class TurkishLowerCaseFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class TurkishLowerCaseFilterFactory extends TokenFilterFactory {
/** Creates a new TurkishLowerCaseFilterFactory */
public TurkishLowerCaseFilterFactory(Map<String,String> args) {
@@ -51,7 +49,7 @@ public class TurkishLowerCaseFilterFactory extends TokenFilterFactory implement
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
index 181beb5..7a5eccb 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharFilterFactory.java
@@ -73,4 +73,13 @@ public abstract class CharFilterFactory extends AbstractAnalysisFactory {
/** Wraps the given Reader with a CharFilter. */
public abstract Reader create(Reader input);
+
+ /**
+ * Normalize the specified input Reader
+ * While the default implementation returns input unchanged,
+ * char filters that should be applied at normalization time can delegate to {@code create} method.
+ */
+ public Reader normalize(Reader input) {
+ return input;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/ElisionFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/ElisionFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/ElisionFilterFactory.java
index e27fbc9..3e67c3c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/ElisionFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/ElisionFilterFactory.java
@@ -38,7 +38,7 @@ import org.apache.lucene.analysis.fr.FrenchAnalyzer;
*
* @since 3.1
*/
-public class ElisionFilterFactory extends TokenFilterFactory implements ResourceLoaderAware, MultiTermAwareComponent {
+public class ElisionFilterFactory extends TokenFilterFactory implements ResourceLoaderAware {
private final String articlesFile;
private final boolean ignoreCase;
private CharArraySet articles;
@@ -63,13 +63,13 @@ public class ElisionFilterFactory extends TokenFilterFactory implements Resource
}
@Override
- public ElisionFilter create(TokenStream input) {
+ public TokenStream create(TokenStream input) {
return new ElisionFilter(input, articles);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
deleted file mode 100644
index 1c1c4c8..0000000
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.util;
-
-
-/** Add to any analysis factory component to allow returning an
- * analysis component factory for use with partial terms in prefix queries,
- * wildcard queries, range query endpoints, regex queries, etc.
- *
- * Components implementing this interface should not add or remove tokens from
- * the token stream, and should be able to deal with special characters
- * indicating that multi-term queries are required (eg slashes for regex, wildcard
- * characters, etc)
- *
- * @lucene.experimental
- */
-public interface MultiTermAwareComponent {
- /** Returns an analysis component to handle analysis if multi-term queries.
- * The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory.
- */
- public AbstractAnalysisFactory getMultiTermComponent();
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
index 180126d..f18b4de 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenFilterFactory.java
@@ -73,4 +73,13 @@ public abstract class TokenFilterFactory extends AbstractAnalysisFactory {
/** Transform the specified input TokenStream */
public abstract TokenStream create(TokenStream input);
+
+ /**
+ * Normalize the specified input TokenStream
+ * While the default implementation returns input unchanged,
+ * filters that should be applied at normalization time can delegate to {@code create} method.
+ */
+ public TokenStream normalize(TokenStream input) {
+ return input;
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
index 499774e..bd834ec 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
@@ -34,7 +34,6 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -78,15 +77,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
TokenizerFactory factory = (TokenizerFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return e.g. a charfilter here: but a tokenizer could wrap a filter around it
- assertFalse(mtc instanceof CharFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
@@ -102,15 +92,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
TokenFilterFactory factory = (TokenFilterFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return a charfilter or tokenizer here, this makes no sense
- assertTrue(mtc instanceof TokenFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
@@ -126,15 +107,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
CharFilterFactory factory = (CharFilterFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return a tokenizer or tokenfilter here, this makes no sense
- assertTrue(mtc instanceof CharFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index a4e1ac5..0a0287a 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -39,9 +39,7 @@ import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
import org.apache.lucene.analysis.reverse.ReverseStringFilterFactory;
import org.apache.lucene.analysis.standard.ClassicTokenizerFactory;
import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
@@ -397,17 +395,16 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
}
- public static class DummyMultiTermAwareCharFilterFactory extends DummyCharFilterFactory implements MultiTermAwareComponent {
+ public static class DummyMultiTermAwareCharFilterFactory extends DummyCharFilterFactory {
public DummyMultiTermAwareCharFilterFactory(Map<String,String> args) {
super(args);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return new DummyCharFilterFactory(Collections.emptyMap(), '0', '2');
+ public Reader normalize(Reader input) {
+ return create(input);
}
-
}
public static class DummyTokenizerFactory extends TokenizerFactory {
@@ -423,19 +420,6 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
}
- public static class DummyMultiTermAwareTokenizerFactory extends DummyTokenizerFactory implements MultiTermAwareComponent {
-
- public DummyMultiTermAwareTokenizerFactory(Map<String,String> args) {
- super(args);
- }
-
- @Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return new DummyTokenFilterFactory(Collections.emptyMap());
- }
-
- }
-
public static class DummyTokenFilterFactory extends TokenFilterFactory {
public DummyTokenFilterFactory(Map<String,String> args) {
@@ -449,15 +433,15 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
}
- public static class DummyMultiTermAwareTokenFilterFactory extends DummyTokenFilterFactory implements MultiTermAwareComponent {
+ public static class DummyMultiTermAwareTokenFilterFactory extends DummyTokenFilterFactory {
public DummyMultiTermAwareTokenFilterFactory(Map<String,String> args) {
super(args);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return new ASCIIFoldingFilterFactory(Collections.emptyMap());
+ public TokenStream normalize(TokenStream input) {
+ return new ASCIIFoldingFilterFactory(Collections.emptyMap()).normalize(input);
}
}
@@ -472,12 +456,13 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
assertEquals(new BytesRef("0À"), analyzer1.normalize("dummy", "0À"));
CustomAnalyzer analyzer2 = CustomAnalyzer.builder()
+ // this component in not multi-term aware so it should not be applied
+ .withTokenizer(DummyTokenizerFactory.class, Collections.emptyMap())
// these components are multi-term aware so they should be applied
- .withTokenizer(DummyMultiTermAwareTokenizerFactory.class, Collections.emptyMap())
.addCharFilter(DummyMultiTermAwareCharFilterFactory.class, Collections.emptyMap())
.addTokenFilter(DummyMultiTermAwareTokenFilterFactory.class, Collections.emptyMap())
.build();
- assertEquals(new BytesRef("2A"), analyzer2.normalize("dummy", "0À"));
+ assertEquals(new BytesRef("1A"), analyzer2.normalize("dummy", "0À"));
}
public void testNormalizationWithMultipleTokenFilters() throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestAsciiFoldingFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestAsciiFoldingFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestAsciiFoldingFilterFactory.java
index 87d8760..5990e59 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestAsciiFoldingFilterFactory.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestAsciiFoldingFilterFactory.java
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
public class TestAsciiFoldingFilterFactory extends BaseTokenStreamFactoryTestCase {
@@ -35,9 +34,8 @@ public class TestAsciiFoldingFilterFactory extends BaseTokenStreamFactoryTestCas
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "Ete" });
- factory = (TokenFilterFactory) ((MultiTermAwareComponent) factory).getMultiTermComponent();
stream = new CannedTokenStream(new Token("Été", 0, 3));
- stream = factory.create(stream);
+ stream = factory.normalize(stream);
assertTokenStreamContents(stream, new String[] { "Ete" });
factory = new ASCIIFoldingFilterFactory(new HashMap<>(Collections.singletonMap("preserveOriginal", "true")));
@@ -45,9 +43,8 @@ public class TestAsciiFoldingFilterFactory extends BaseTokenStreamFactoryTestCas
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "Ete", "Été" });
- factory = (TokenFilterFactory) ((MultiTermAwareComponent) factory).getMultiTermComponent();
stream = new CannedTokenStream(new Token("Été", 0, 3));
- stream = factory.create(stream);
+ stream = factory.normalize(stream);
assertTokenStreamContents(stream, new String[] { "Ete" });
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
index c35ab61..8825c66 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUFoldingFilterFactory.java
@@ -19,14 +19,11 @@ package org.apache.lucene.analysis.icu;
import java.util.Map;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-import org.apache.lucene.analysis.util.TokenFilterFactory;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link ICUFoldingFilter}.
@@ -39,7 +36,7 @@ import com.ibm.icu.text.UnicodeSet;
* </fieldType></pre>
* @since 3.1.0
*/
-public class ICUFoldingFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class ICUFoldingFilterFactory extends TokenFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUFoldingFilterFactory */
@@ -67,7 +64,7 @@ public class ICUFoldingFilterFactory extends TokenFilterFactory implements Multi
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
index 2f43b51..57a8ee1 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2CharFilterFactory.java
@@ -21,13 +21,10 @@ import java.io.Reader;
import java.util.Arrays;
import java.util.Map;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
+import org.apache.lucene.analysis.util.CharFilterFactory;
/**
* Factory for {@link ICUNormalizer2CharFilter}
@@ -47,7 +44,7 @@ import com.ibm.icu.text.UnicodeSet;
*
* @since 4.10.0
*/
-public class ICUNormalizer2CharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
+public class ICUNormalizer2CharFilterFactory extends CharFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUNormalizer2CharFilterFactory */
@@ -78,8 +75,7 @@ public class ICUNormalizer2CharFilterFactory extends CharFilterFactory implement
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public Reader normalize(Reader input) {
+ return create(input);
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
index 2905055..93b9f40 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUNormalizer2FilterFactory.java
@@ -20,14 +20,11 @@ package org.apache.lucene.analysis.icu;
import java.util.Arrays;
import java.util.Map;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-import org.apache.lucene.analysis.util.TokenFilterFactory;
-
import com.ibm.icu.text.FilteredNormalizer2;
import com.ibm.icu.text.Normalizer2;
import com.ibm.icu.text.UnicodeSet;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
/**
* Factory for {@link ICUNormalizer2Filter}
@@ -46,7 +43,7 @@ import com.ibm.icu.text.UnicodeSet;
* @see FilteredNormalizer2
* @since 3.1.0
*/
-public class ICUNormalizer2FilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class ICUNormalizer2FilterFactory extends TokenFilterFactory {
private final Normalizer2 normalizer;
/** Creates a new ICUNormalizer2FilterFactory */
@@ -79,7 +76,7 @@ public class ICUNormalizer2FilterFactory extends TokenFilterFactory implements M
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
index 4776160..b70d477 100644
--- a/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
+++ b/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilterFactory.java
@@ -20,13 +20,10 @@ package org.apache.lucene.analysis.icu;
import java.util.Arrays;
import java.util.Map;
+import com.ibm.icu.text.Transliterator;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
-import com.ibm.icu.text.Transliterator;
-
/**
* Factory for {@link ICUTransformFilter}.
* <p>
@@ -38,7 +35,7 @@ import com.ibm.icu.text.Transliterator;
* @see Transliterator
* @since 3.1.0
*/
-public class ICUTransformFilterFactory extends TokenFilterFactory implements MultiTermAwareComponent {
+public class ICUTransformFilterFactory extends TokenFilterFactory {
private final Transliterator transliterator;
// TODO: add support for custom rules
@@ -58,9 +55,9 @@ public class ICUTransformFilterFactory extends TokenFilterFactory implements Mul
public TokenStream create(TokenStream input) {
return new ICUTransformFilter(input, transliterator);
}
-
+
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
index 8ee5457..568989a 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
@@ -17,14 +17,11 @@
package org.apache.lucene.analysis.ja;
-import org.apache.lucene.analysis.CharFilter;
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-
import java.io.Reader;
import java.util.Map;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+
/**
* Factory for {@link org.apache.lucene.analysis.ja.JapaneseIterationMarkCharFilter}.
* <pre class="prettyprint">
@@ -37,7 +34,7 @@ import java.util.Map;
*
* @since 4.0.0
*/
-public class JapaneseIterationMarkCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
+public class JapaneseIterationMarkCharFilterFactory extends CharFilterFactory {
private static final String NORMALIZE_KANJI_PARAM = "normalizeKanji";
private static final String NORMALIZE_KANA_PARAM = "normalizeKana";
@@ -56,12 +53,12 @@ public class JapaneseIterationMarkCharFilterFactory extends CharFilterFactory im
}
@Override
- public CharFilter create(Reader input) {
+ public Reader create(Reader input) {
return new JapaneseIterationMarkCharFilter(input, normalizeKanji, normalizeKana);
}
@Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- return this;
+ public Reader normalize(Reader input) {
+ return create(input);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java
index 0a467a7..40e9dee 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestFactories.java
@@ -34,7 +34,6 @@ import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory;
import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -75,15 +74,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
TokenizerFactory factory = (TokenizerFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return e.g. a charfilter here: but a tokenizer could wrap a filter around it
- assertFalse(mtc instanceof CharFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
@@ -99,15 +89,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
TokenFilterFactory factory = (TokenFilterFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return a charfilter or tokenizer here, this makes no sense
- assertTrue(mtc instanceof TokenFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
@@ -123,15 +104,6 @@ public class TestFactories extends BaseTokenStreamTestCase {
CharFilterFactory factory = (CharFilterFactory) initialize(factoryClazz);
if (factory != null) {
// we managed to fully create an instance. check a few more things:
-
- // if it implements MultiTermAware, sanity check its impl
- if (factory instanceof MultiTermAwareComponent) {
- AbstractAnalysisFactory mtc = ((MultiTermAwareComponent) factory).getMultiTermComponent();
- assertNotNull(mtc);
- // it's not ok to return a tokenizer or tokenfilter here, this makes no sense
- assertTrue(mtc instanceof CharFilterFactory);
- }
-
if (!EXCLUDE_FACTORIES_RANDOM_DATA.contains(factory.getClass())) {
// beast it just a little, it shouldnt throw exceptions:
// (it should have thrown them in initialize)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
index e14c822..d05cceb 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
@@ -18,12 +18,12 @@ package org.apache.lucene.analysis.ja;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import java.io.IOException;
+import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
@@ -36,7 +36,7 @@ public class TestJapaneseIterationMarkCharFilterFactory extends BaseTokenStreamT
public void testIterationMarksWithKeywordTokenizer() throws IOException {
final String text = "時々馬鹿々々しいところゞゝゝミスヾ";
JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(new HashMap<String,String>());
- CharFilter filter = filterFactory.create(new StringReader(text));
+ Reader filter = filterFactory.create(new StringReader(text));
TokenStream tokenStream = new MockTokenizer(MockTokenizer.KEYWORD, false);
((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"時時馬鹿馬鹿しいところどころミスズ"});
@@ -47,7 +47,7 @@ public class TestJapaneseIterationMarkCharFilterFactory extends BaseTokenStreamT
tokenizerFactory.inform(new StringMockResourceLoader(""));
JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(new HashMap<String,String>());
- CharFilter filter = filterFactory.create(
+ Reader filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
@@ -64,7 +64,7 @@ public class TestJapaneseIterationMarkCharFilterFactory extends BaseTokenStreamT
filterArgs.put("normalizeKana", "false");
JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(filterArgs);
- CharFilter filter = filterFactory.create(
+ Reader filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
@@ -81,7 +81,7 @@ public class TestJapaneseIterationMarkCharFilterFactory extends BaseTokenStreamT
filterArgs.put("normalizeKana", "true");
JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(filterArgs);
- CharFilter filter = filterFactory.create(
+ Reader filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
TokenStream tokenStream = tokenizerFactory.create(newAttributeFactory());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65486442/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
index d27e946..95755e1 100644
--- a/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
+++ b/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
@@ -18,10 +18,11 @@ package org.apache.solr.analysis;
import java.io.Reader;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.util.CharFilterFactory;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
@@ -91,10 +92,7 @@ public final class TokenizerChain extends SolrAnalyzer {
protected Reader initReaderForNormalization(String fieldName, Reader reader) {
if (charFilters != null && charFilters.length > 0) {
for (CharFilterFactory charFilter : charFilters) {
- if (charFilter instanceof MultiTermAwareComponent) {
- charFilter = (CharFilterFactory) ((MultiTermAwareComponent) charFilter).getMultiTermComponent();
- reader = charFilter.create(reader);
- }
+ reader = charFilter.normalize(reader);
}
}
return reader;
@@ -114,10 +112,7 @@ public final class TokenizerChain extends SolrAnalyzer {
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = in;
for (TokenFilterFactory filter : filters) {
- if (filter instanceof MultiTermAwareComponent) {
- filter = (TokenFilterFactory) ((MultiTermAwareComponent) filter).getMultiTermComponent();
- result = filter.create(result);
- }
+ result = filter.normalize(result);
}
return result;
}
@@ -138,4 +133,30 @@ public final class TokenizerChain extends SolrAnalyzer {
return sb.toString();
}
+ public Analyzer getMultiTermAnalyzer() {
+ return new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tk = new KeywordTokenizer();
+ TokenStream ts = tk;
+ for (TokenFilterFactory filter : filters) {
+ ts = filter.normalize(ts);
+ }
+ return new TokenStreamComponents(tk, ts);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ if (charFilters != null && charFilters.length > 0) {
+ Reader cs = reader;
+ for (CharFilterFactory charFilter : charFilters) {
+ cs = charFilter.normalize(cs);
+ }
+ reader = cs;
+ }
+ return reader;
+ }
+ };
+ }
+
}