You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/11/07 04:40:13 UTC
svn commit: r1406439 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/CHANGES.txt solr/core/
solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
Author: rmuir
Date: Wed Nov 7 03:40:13 2012
New Revision: 1406439
URL: http://svn.apache.org/viewvc?rev=1406439&view=rev
Log:
SOLR-3589: Edismax parser does not honor mm parameter if analyzer splits a token
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1406439&r1=1406438&r2=1406439&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Nov 7 03:40:13 2012
@@ -131,6 +131,9 @@ Bug Fixes
thrown by PingRequestHandler. Do not log exceptions if a user tries to view a
hidden file using ShowFileRequestHandler. (Tomás Fernández Löbbe via James Dyer)
+* SOLR-3589: Edismax parser does not honor mm parameter if analyzer splits a token.
+ (Tom Burton-West, Robert Muir)
+
Other Changes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java?rev=1406439&r1=1406438&r2=1406439&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParserPlugin.java Wed Nov 7 03:40:13 2012
@@ -243,6 +243,8 @@ class ExtendedDismaxQParser extends QPar
// For correct lucene queries, turn off mm processing if there
// were explicit operators (except for AND).
boolean doMinMatched = (numOR + numNOT + numPluses + numMinuses) == 0;
+ // but always for unstructured implicit bqs created by getFieldQuery
+ up.minShouldMatch = minShouldMatch;
try {
up.setRemoveStopFilter(!stopwords);
@@ -888,6 +890,7 @@ class ExtendedDismaxQParser extends QPar
private Map<String, Analyzer> nonStopFilterAnalyzerPerField;
private boolean removeStopFilter;
+ String minShouldMatch; // for inner boolean queries produced from a single fieldQuery
/**
* Where we store a map from field name we expect to see in our query
@@ -1161,6 +1164,18 @@ class ExtendedDismaxQParser extends QPar
case FIELD: // fallthrough
case PHRASE:
Query query = super.getFieldQuery(field, val, type == QType.PHRASE);
+ // A BooleanQuery is only possible from getFieldQuery if it came from
+ // a single whitespace separated term. In this case, check the coordination
+ // factor on the query: if its enabled, that means we aren't a set of synonyms
+ // but instead multiple terms from one whitespace-separated term, we must
+ // apply minShouldMatch here so that it works correctly with other things
+ // like aliasing.
+ if (query instanceof BooleanQuery) {
+ BooleanQuery bq = (BooleanQuery) query;
+ if (!bq.isCoordDisabled()) {
+ SolrPluginUtils.setMinShouldMatch(bq, minShouldMatch);
+ }
+ }
if (query instanceof PhraseQuery) {
PhraseQuery pq = (PhraseQuery)query;
if (minClauseSize > 1 && pq.getTerms().length < minClauseSize) return null;
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java?rev=1406439&r1=1406438&r2=1406439&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/search/TestExtendedDismaxParser.java Wed Nov 7 03:40:13 2012
@@ -57,6 +57,13 @@ public class TestExtendedDismaxParser ex
assertU(adoc("id", "52", "text_sw", "tekna theou klethomen"));
assertU(adoc("id", "53", "text_sw", "nun tekna theou esmen"));
assertU(adoc("id", "54", "text_sw", "phanera estin ta tekna tou theou"));
+ assertU(adoc("id", "55", "standardtok", "大"));
+ assertU(adoc("id", "56", "standardtok", "大äº"));
+ assertU(adoc("id", "57", "standardtok", "å¤§äºæ¹¾"));
+ assertU(adoc("id", "58", "HTMLstandardtok", "大"));
+ assertU(adoc("id", "59", "HTMLstandardtok", "大äº"));
+ assertU(adoc("id", "60", "HTMLstandardtok", "å¤§äºæ¹¾"));
+ assertU(adoc("id", "61", "text_sw", "bazaaa")); // synonyms in an expansion group
assertU(commit());
}
@Override
@@ -771,4 +778,153 @@ public class TestExtendedDismaxParser ex
, "*[count(//doc)=1]");
}
+
+ /**
+ * SOLR-3589: Edismax parser does not honor mm parameter if analyzer splits a token
+ */
+ public void testCJK() throws Exception {
+ assertQ("test cjk (disjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=3]");
+ assertQ("test cjk (minShouldMatch)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]");
+ assertQ("test cjk (conjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=1]");
+ }
+
+ /**
+ * test that minShouldMatch works with aliasing
+ * for implicit boolean queries
+ */
+ public void testCJKAliasing() throws Exception {
+ // single field
+ assertQ("test cjk (aliasing+disjunction)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=3]");
+ assertQ("test cjk (aliasing+minShouldMatch)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]");
+ assertQ("test cjk (aliasing+conjunction)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=1]");
+ // multifield
+ assertQ("test cjk (aliasing+disjunction)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok HTMLstandardtok",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=6]");
+ assertQ("test cjk (aliasing+minShouldMatch)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok HTMLstandardtok",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=4]");
+ assertQ("test cjk (aliasing+conjunction)",
+ req("q", "myalias:å¤§äºæ¹¾",
+ "f.myalias.qf", "standardtok HTMLstandardtok",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]");
+ }
+
+ /** Test that we apply boosts correctly */
+ public void testCJKBoosts() throws Exception {
+ assertQ("test cjk (disjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok^2 HTMLstandardtok",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=6]", "//result/doc[1]/str[@name='id'][.='57']");
+ assertQ("test cjk (minShouldMatch)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok^2 HTMLstandardtok",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=4]", "//result/doc[1]/str[@name='id'][.='57']");
+ assertQ("test cjk (conjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok^2 HTMLstandardtok",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]", "//result/doc[1]/str[@name='id'][.='57']");
+
+ // now boost the other field
+ assertQ("test cjk (disjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok HTMLstandardtok^2",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=6]", "//result/doc[1]/str[@name='id'][.='60']");
+ assertQ("test cjk (minShouldMatch)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok HTMLstandardtok^2",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=4]", "//result/doc[1]/str[@name='id'][.='60']");
+ assertQ("test cjk (conjunction)",
+ req("q", "å¤§äºæ¹¾",
+ "qf", "standardtok HTMLstandardtok^2",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]", "//result/doc[1]/str[@name='id'][.='60']");
+ }
+
+ /** always apply minShouldMatch to the inner booleanqueries
+ * created from whitespace, as these are never structured lucene queries
+ * but only come from unstructured text */
+ public void testCJKStructured() throws Exception {
+ assertQ("test cjk (disjunction)",
+ req("q", "å¤§äºæ¹¾ OR bogus",
+ "qf", "standardtok",
+ "mm", "0%",
+ "defType", "edismax")
+ , "*[count(//doc)=3]");
+ assertQ("test cjk (minShouldMatch)",
+ req("q", "å¤§äºæ¹¾ OR bogus",
+ "qf", "standardtok",
+ "mm", "67%",
+ "defType", "edismax")
+ , "*[count(//doc)=2]");
+ assertQ("test cjk (conjunction)",
+ req("q", "å¤§äºæ¹¾ OR bogus",
+ "qf", "standardtok",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=1]");
+ }
+
+ /**
+ * Test that we don't apply minShouldMatch to the inner boolean queries
+ * when there are synonyms (these are indicated by coordination factor)
+ */
+ public void testSynonyms() throws Exception {
+ // document only contains baraaa, but should still match.
+ assertQ("test synonyms",
+ req("q", "fooaaa",
+ "qf", "text_sw",
+ "mm", "100%",
+ "defType", "edismax")
+ , "*[count(//doc)=1]");
+ }
}