You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2018/03/28 04:38:04 UTC
lucene-solr:master: SOLR-12035: edimax should include charfilters in
nostopanalyzer
Repository: lucene-solr
Updated Branches:
refs/heads/master f8af27478 -> 3e29c7dbd
SOLR-12035: edimax should include charfilters in nostopanalyzer
This closes #329
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e29c7db
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e29c7db
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e29c7db
Branch: refs/heads/master
Commit: 3e29c7dbd507032315aa698702daef1c7a370f75
Parents: f8af274
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Tue Mar 27 21:37:19 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Tue Mar 27 21:37:19 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../solr/search/ExtendedDismaxQParser.java | 2 +-
.../solr/collection1/conf/schema12.xml | 12 ++++++++++++
.../solr/search/TestExtendedDismaxParser.java | 19 ++++++++++++++++++-
4 files changed, 34 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e29c7db/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3039ce3..fcf1f9c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -83,6 +83,9 @@ Bug Fixes
* SOLR-11551: Standardize CoreAdmin API success/failure status codes (Jason Gerlowski, Steve Rowe)
+* SOLR-12035: ExtendedDismaxQParser fails to include charfilters in nostopanalyzer (Tim Allison via
+ Tomás Fernández Löbbe)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e29c7db/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
index 004d1c0..9d6cd59 100644
--- a/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
+++ b/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
@@ -1412,7 +1412,7 @@ public class ExtendedDismaxQParser extends QParser {
newtf[j++] = facs[i];
}
- TokenizerChain newa = new TokenizerChain(tcq.getTokenizerFactory(), newtf);
+ TokenizerChain newa = new TokenizerChain(tcq.getCharFilterFactories(), tcq.getTokenizerFactory(), newtf);
newa.setPositionIncrementGap(tcq.getPositionIncrementGap(fieldName));
return newa;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e29c7db/solr/core/src/test-files/solr/collection1/conf/schema12.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
index 97b6ed0..de1b998 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
@@ -272,6 +272,17 @@
<tokenizer class="solr.StandardTokenizerFactory"/>
</analyzer>
</fieldType>
+ <fieldType name="isocharfilter" class="solr.TextField">
+ <analyzer type="index">
+ <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ </analyzer>
+ <analyzer type="query">
+ <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
+ <tokenizer class="solr.MockTokenizerFactory"/>
+ <filter class="solr.StopFilterFactory" ignoreCase="true"/>
+ </analyzer>
+ </fieldType>
<fieldType name="HTMLwhitetok" class="solr.TextField">
<analyzer>
<charFilter class="solr.HTMLStripCharFilterFactory"/>
@@ -590,6 +601,7 @@
<field name="lettertok" type="lettertok" indexed="true" stored="true"/>
<field name="whitetok" type="whitetok" indexed="true" stored="true"/>
<field name="HTMLwhitetok" type="HTMLwhitetok" indexed="true" stored="true"/>
+ <field name="isocharfilter" type="isocharfilter" indexed="true" stored="true"/>
<field name="standardtokfilt" type="standardtokfilt" indexed="true" stored="true"/>
<field name="standardfilt" type="standardfilt" indexed="true" stored="true"/>
<field name="lowerfilt" type="lowerfilt" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e29c7db/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
index 21d47db..f77baaf 100644
--- a/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
@@ -96,6 +96,7 @@ public class TestExtendedDismaxParser extends SolrTestCaseJ4 {
assertU(adoc("id", "71", "text_sw", "ties"));
assertU(adoc("id", "72", "text_sw", "wifi ATM"));
assertU(adoc("id", "73", "shingle23", "A B X D E"));
+ assertU(adoc("id", "74", "isocharfilter", "niño"));
// assertU(adoc("id", "74", "text_pick_best", "tabby"));
// assertU(adoc("id", "74", "text_as_distinct", "persian"));
@@ -210,7 +211,23 @@ public class TestExtendedDismaxParser extends SolrTestCaseJ4 {
, "*[count(//doc)=1]");
}
}
-
+
+ public void testCharFilter() throws Exception {
+ // test that charfilter was applied by the indexer
+ assertQ(req("defType", "edismax",
+ "stopwords","false",
+ "qf", "isocharfilter",
+ "q","nino"), "*[count(//doc)=1]"
+ );
+
+ // test that charfilter was applied to the query
+ assertQ(req("defType", "edismax",
+ "stopwords","false",
+ "qf", "isocharfilter",
+ "q","niño"), "*[count(//doc)=1]"
+ );
+ }
+
// test the edismax query parser based on the dismax parser
public void testFocusQueryParser() {
String allq = "id:[42 TO 51]";