You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by kl...@apache.org on 2007/03/05 21:22:11 UTC
svn commit: r514851 - in /lucene/solr/trunk: ./
src/java/org/apache/solr/request/ src/java/org/apache/solr/search/
src/java/org/apache/solr/util/ src/test/org/apache/solr/
src/test/org/apache/solr/util/
Author: klaas
Date: Mon Mar 5 12:22:10 2007
New Revision: 514851
URL: http://svn.apache.org/viewvc?view=rev&rev=514851
Log:
enables multi-param for several dismax params (SOLR-174)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java
lucene/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java
lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java
lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java
lucene/solr/trunk/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
lucene/solr/trunk/src/test/org/apache/solr/util/SolrPluginUtilsTest.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Mon Mar 5 12:22:10 2007
@@ -137,6 +137,12 @@
IndexSchema have also been clarified.
(Erik Hatcher and hossman)
+ 4. DisMaxRequestHandler's bq, bf, qf, and pf parameters can now accept
+ multiple values (klaas).
+
+ 5. Query are re-written before highlighting is performed. This enables
+ proper highlighting of prefix and wildcard queries (klaas).
+
Optimizations
1. SOLR-114: HashDocSet specific implementations of union() and andNot()
for a 20x performance improvement for those set operations, and a new
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java Mon Mar 5 12:22:10 2007
@@ -73,6 +73,8 @@
* <li> qf - (Query Fields) fields and boosts to use when building
* DisjunctionMaxQueries from the users query. Format is:
* "<code>fieldA^1.0 fieldB^2.2</code>".
+ * This param can be specified multiple times, and the fields
+ * are additive.
* </li>
* <li> mm - (Minimum Match) this supports a wide variety of
* complex expressions.
@@ -81,6 +83,8 @@
* <li> pf - (Phrase Fields) fields/boosts to make phrase queries out
* of, to boost the users query for exact matches on the specified fields.
* Format is: "<code>fieldA^1.0 fieldB^2.2</code>".
+ * This param can be specified multiple times, and the fields
+ * are additive.
* </li>
* <li> ps - (Phrase Slop) amount of slop on phrase queries built for pf
* fields.
@@ -93,12 +97,19 @@
* with a default boost (1.0f), then the individual clauses will be
* added directly to the main query. Otherwise, the query will be
* included as is.
+ * This param can be specified multiple times, and the boosts are
+ * are additive. NOTE: the behaviour listed above is only in effect
+ * if a single <code>bq</code> paramter is specified. Hence you can
+ * disable it by specifying an additional, blank, <code>bq</code>
+ * parameter.
* </li>
* <li> bf - (Boost Functions) functions (with optional boosts) that will be
* included in the users query to influence the score.
* Format is: "<code>funcA(arg1,arg2)^1.2
* funcB(arg3,arg4)^2.2</code>". NOTE: Whitespace is not allowed
* in the function arguments.
+ * This param can be specified multiple times, and the functions
+ * are additive.
* </li>
* <li> fq - (Filter Query) a raw lucene query that can be used
* to restrict the super set of products we are interested in - more
@@ -122,7 +133,6 @@
* <pre>
* :TODO: document facet param support
*
- * :TODO: make bf,pf,qf multival params now that SolrParams supports them
* </pre>
*/
public class DisMaxRequestHandler extends RequestHandlerBase {
@@ -173,8 +183,8 @@
SolrIndexSearcher s = req.getSearcher();
IndexSchema schema = req.getSchema();
- Map<String,Float> queryFields = U.parseFieldBoosts(params.get(DMP.QF));
- Map<String,Float> phraseFields = U.parseFieldBoosts(params.get(DMP.PF));
+ Map<String,Float> queryFields = U.parseFieldBoosts(params.getParams(DMP.QF));
+ Map<String,Float> phraseFields = U.parseFieldBoosts(params.getParams(DMP.PF));
float tiebreaker = params.getFloat(DMP.TIE, 0.0f);
@@ -255,29 +265,39 @@
/* * * Boosting Query * * */
-
- String boostQuery = params.get(DMP.BQ);
- if (null != boostQuery && !boostQuery.equals("")) {
- Query tmp = p.parse(boostQuery);
- /* if the default boost was used, and we've got a BooleanQuery
- * extract the subqueries out and use them directly
- */
- if (1.0f == tmp.getBoost() && tmp instanceof BooleanQuery) {
- for (BooleanClause c : ((BooleanQuery)tmp).getClauses()) {
- query.add(c);
+ String[] boostParams = params.getParams(DMP.BQ);
+ List<Query> boostQueries = U.parseQueryStrings(req, boostParams);
+ if (null != boostQueries) {
+ if(1 == boostQueries.size() && 1 == boostParams.length) {
+ /* legacy logic */
+ Query f = boostQueries.get(0);
+ if (1.0f == f.getBoost() && f instanceof BooleanQuery) {
+ /* if the default boost was used, and we've got a BooleanQuery
+ * extract the subqueries out and use them directly
+ */
+ for (BooleanClause c : ((BooleanQuery)f).getClauses()) {
+ query.add(c);
+ }
+ } else {
+ query.add(f, BooleanClause.Occur.SHOULD);
}
} else {
- query.add(tmp, BooleanClause.Occur.SHOULD);
+ for(Query f : boostQueries) {
+ query.add(f, BooleanClause.Occur.SHOULD);
+ }
}
}
/* * * Boosting Functions * * */
- String boostFunc = params.get(DMP.BF);
- if (null != boostFunc && !boostFunc.equals("")) {
- List<Query> funcs = U.parseFuncs(schema, boostFunc);
- for (Query f : funcs) {
- query.add(f, Occur.SHOULD);
+ String[] boostFuncs = params.getParams(DMP.BF);
+ if (null != boostFuncs && 0 != boostFuncs.length) {
+ for (String boostFunc : boostFuncs) {
+ if(null == boostFunc || "".equals(boostFunc)) continue;
+ List<Query> funcs = U.parseFuncs(schema, boostFunc);
+ for (Query f : funcs) {
+ query.add(f, Occur.SHOULD);
+ }
}
}
@@ -318,22 +338,23 @@
NamedList debug = U.doStandardDebug(req, userQuery, query, results.docList);
if (null != debug) {
debug.add("altquerystring", altUserQuery);
- debug.add("boostquery", boostQuery);
- debug.add("boostfunc", boostFunc);
+ if (null != boostQueries) {
+ debug.add("boost_queries", boostParams);
+ debug.add("parsed_boost_queries",
+ QueryParsing.toString(boostQueries, req.getSchema()));
+ }
+ debug.add("boostfuncs", params.getParams(DMP.BF));
if (null != restrictions) {
debug.add("filter_queries", params.getParams(FQ));
- List<String> fqs = new ArrayList<String>(restrictions.size());
- for (Query fq : restrictions) {
- fqs.add(QueryParsing.toString(fq, req.getSchema()));
- }
- debug.add("parsed_filter_queries",fqs);
+ debug.add("parsed_filter_queries",
+ QueryParsing.toString(restrictions, req.getSchema()));
}
rsp.add("debug", debug);
}
} catch (Exception e) {
SolrException.logOnce(SolrCore.log,
- "Exception durring debug", e);
+ "Exception during debug", e);
rsp.add("exception_during_debug", SolrException.toStr(e));
}
@@ -341,8 +362,11 @@
if(HighlightingUtils.isHighlightingEnabled(req) && parsedUserQuery != null) {
String[] highFields = queryFields.keySet().toArray(new String[0]);
NamedList sumData =
- HighlightingUtils.doHighlighting(results.docList, parsedUserQuery,
- req, highFields);
+ HighlightingUtils.doHighlighting(
+ results.docList,
+ parsedUserQuery.rewrite(req.getSearcher().getReader()),
+ req,
+ highFields);
if(sumData != null)
rsp.add("highlighting", sumData);
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java Mon Mar 5 12:22:10 2007
@@ -138,9 +138,9 @@
SolrException.logOnce(SolrCore.log, "Exception during debug", e);
rsp.add("exception_during_debug", SolrException.toStr(e));
}
-
+
NamedList sumData = HighlightingUtils.doHighlighting(
- results.docList, query, req, new String[]{defaultField});
+ results.docList, query.rewrite(req.getSearcher().getReader()), req, new String[]{defaultField});
if(sumData != null)
rsp.add("highlighting", sumData);
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/QueryParsing.java Mon Mar 5 12:22:10 2007
@@ -31,6 +31,7 @@
import org.apache.solr.request.SolrParams;
import java.util.ArrayList;
+import java.util.List;
import java.util.regex.Pattern;
import java.util.logging.Level;
import java.io.IOException;
@@ -469,6 +470,16 @@
}
+ /**
+ * Builds a list of String which are stringified versions of a list of Queries
+ */
+ public static List<String> toString(List<Query> queries, IndexSchema schema) {
+ List<String> out = new ArrayList<String>(queries.size());
+ for (Query q : queries) {
+ out.add(QueryParsing.toString(q, schema));
+ }
+ return out;
+ }
private static ValueSource parseValSource(StrParser sp, IndexSchema schema) throws ParseException {
String id = sp.getId();
Modified: lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java Mon Mar 5 12:22:10 2007
@@ -483,20 +483,31 @@
* @return Map of fieldOne => 2.3, fieldTwo => null, fieldThree => -0.4
*/
public static Map<String,Float> parseFieldBoosts(String in) {
-
- if (null == in || "".equals(in.trim())) {
+ return parseFieldBoosts(new String[]{in});
+ }
+ /**
+ * Like <code>parseFieldBoosts(String)</code>, but parses all the strings
+ * in the provided array (which may be null).
+ *
+ * @param fieldList an array of Strings eg. <code>{"fieldOne^2.3", "fieldTwo"}</code>
+ * @return Map of fieldOne => 2.3, fieldThree => -0.4
+ */
+ public static Map<String,Float> parseFieldBoosts(String[] fieldLists) {
+ if (null == fieldLists || 0 == fieldLists.length) {
return new HashMap<String,Float>();
}
-
- String[] bb = in.trim().split("\\s+");
Map<String, Float> out = new HashMap<String,Float>(7);
- for (String s : bb) {
- String[] bbb = s.split("\\^");
- out.put(bbb[0], 1 == bbb.length ? null : Float.valueOf(bbb[1]));
+ for (String in : fieldLists) {
+ if (null == in || "".equals(in.trim()))
+ continue;
+ String[] bb = in.trim().split("\\s+");
+ for (String s : bb) {
+ String[] bbb = s.split("\\^");
+ out.put(bbb[0], 1 == bbb.length ? null : Float.valueOf(bbb[1]));
+ }
}
return out;
}
-
/**
* Given a string containing functions with optional boosts, returns
* an array of Queries representing those functions with the specified
@@ -804,10 +815,16 @@
* @return null if no filter queries
*/
public static List<Query> parseFilterQueries(SolrQueryRequest req) throws ParseException {
- String[] in = req.getParams().getParams(SolrParams.FQ);
-
- if (null == in || 0 == in.length) return null;
+ return parseQueryStrings(req, req.getParams().getParams(SolrParams.FQ));
+ }
+ /** Turns an array of query strings into a List of Query objects.
+ *
+ * @return null if no queries are generated
+ */
+ public static List<Query> parseQueryStrings(SolrQueryRequest req,
+ String[] queries) throws ParseException {
+ if (null == queries || 0 == queries.length) return null;
List<Query> out = new LinkedList<Query>();
SolrIndexSearcher s = req.getSearcher();
/* Ignore SolrParams.DF - could have init param FQs assuming the
@@ -815,7 +832,7 @@
* If user doesn't want schema default, they should be explicit in the FQ.
*/
SolrQueryParser qp = new SolrQueryParser(s.getSchema(), null);
- for (String q : in) {
+ for (String q : queries) {
if (null != q && 0 != q.trim().length()) {
out.add(qp.parse(q));
}
Modified: lucene/solr/trunk/src/test/org/apache/solr/DisMaxRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/DisMaxRequestHandlerTest.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/DisMaxRequestHandlerTest.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/DisMaxRequestHandlerTest.java Mon Mar 5 12:22:10 2007
@@ -29,6 +29,7 @@
import java.io.UnsupportedEncodingException;
import java.util.Map;
import java.util.HashMap;
+import java.util.regex.Pattern;
/**
* Tests some basic functionality of the DisMaxRequestHandler
@@ -46,8 +47,8 @@
"facet.field","t_s"
);
}
- public void testSomeStuff() throws Exception {
-
+ /** Add some documents to the index */
+ protected void populate() {
assertU(adoc("id", "666",
"features_t", "cool and scary stuff",
"subject", "traveling in hell",
@@ -77,7 +78,11 @@
"weight", "97.3",
"iind", "8675309"));
assertU(commit());
-
+ }
+
+ public void testSomeStuff() throws Exception {
+ populate();
+
assertQ("basic match",
req("guide")
,"//*[@numFound='2']"
@@ -94,6 +99,42 @@
,"//result/doc[2]/int[@name='id'][.='666']"
,"//result/doc[3]/int[@name='id'][.='8675309']"
);
+
+ assertQ("multi qf",
+ req("q", "cool"
+ ,"qt", "dismax"
+ ,"version", "2.0"
+ ,"qf", "subject"
+ ,"qf", "features_t"
+ )
+ ,"//*[@numFound='3']"
+ );
+
+ assertQ("boost query",
+ req("q", "cool stuff"
+ ,"qt", "dismax"
+ ,"version", "2.0"
+ ,"bq", "subject:hell^400"
+ )
+ ,"//*[@numFound='3']"
+ ,"//result/doc[1]/int[@name='id'][.='666']"
+ ,"//result/doc[2]/int[@name='id'][.='42']"
+ ,"//result/doc[3]/int[@name='id'][.='8675309']"
+ );
+
+ assertQ("multi boost query",
+ req("q", "cool stuff"
+ ,"qt", "dismax"
+ ,"version", "2.0"
+ ,"bq", "subject:hell^400"
+ ,"bq", "subject:cool^4"
+ ,"debugQuery", "true"
+ )
+ ,"//*[@numFound='3']"
+ ,"//result/doc[1]/int[@name='id'][.='666']"
+ ,"//result/doc[2]/int[@name='id'][.='8675309']"
+ ,"//result/doc[3]/int[@name='id'][.='42']"
+ );
assertQ("minimum mm is three",
req("cool stuff traveling")
@@ -133,6 +174,33 @@
"q", "\"cool chick\"" )
,"//*[@numFound='1']"
);
+ }
+
+ public void testExtraBlankBQ() throws Exception {
+ populate();
+ // if the boost queries are in their own boolean query, the clauses will be
+ // surrounded by ()'s in the debug output
+ Pattern p = Pattern.compile("subject:hell\\s*subject:cool");
+ Pattern p_bool = Pattern.compile("\\(subject:hell\\s*subject:cool\\)");
+ String resp = h.query(req("q", "cool stuff"
+ ,"qt", "dismax"
+ ,"version", "2.0"
+ ,"bq", "subject:hell OR subject:cool"
+ ,"debugQuery", "true"
+ ));
+ assertTrue(p.matcher(resp).find());
+ assertFalse(p_bool.matcher(resp).find());
+
+ resp = h.query(req("q", "cool stuff"
+ ,"qt", "dismax"
+ ,"version", "2.0"
+ ,"bq", "subject:hell OR subject:cool"
+ ,"bq",""
+ ,"debugQuery", "true"
+ ));
+ assertTrue(p.matcher(resp).find());
+ assertTrue(p_bool.matcher(resp).find());
+
}
public void testOldStyleDefaults() throws Exception {
Modified: lucene/solr/trunk/src/test/org/apache/solr/util/SolrPluginUtilsTest.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/util/SolrPluginUtilsTest.java?view=diff&rev=514851&r1=514850&r2=514851
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/util/SolrPluginUtilsTest.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/util/SolrPluginUtilsTest.java Mon Mar 5 12:22:10 2007
@@ -92,6 +92,10 @@
(" fieldOne^2.3 fieldTwo fieldThree^-0.4 "));
assertEquals("really spacey e1", e1, SolrPluginUtils.parseFieldBoosts
(" \t fieldOne^2.3 \n fieldTwo fieldThree^-0.4 "));
+ assertEquals("really spacey e1", e1, SolrPluginUtils.parseFieldBoosts
+ (new String[]{" \t fieldOne^2.3 \n",
+ " fieldTwo fieldThree^-0.4 ",
+ " "}));
Map<String,Float> e2 = new HashMap<String,Float>();
assertEquals("empty e2", e2, SolrPluginUtils.parseFieldBoosts