You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2008/02/28 15:11:26 UTC
svn commit: r631979 - in /lucene/solr/trunk: CHANGES.txt
src/java/org/apache/solr/core/SolrCore.java
src/java/org/apache/solr/search/FunctionQParser.java
src/test/org/apache/solr/search/function/TestFunctionQuery.java
Author: yonik
Date: Thu Feb 28 06:11:23 2008
New Revision: 631979
URL: http://svn.apache.org/viewvc?rev=631979&view=rev
Log:
SOLR-356: pluggable functions
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java
lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java
lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=631979&r1=631978&r2=631979&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Thu Feb 28 06:11:23 2008
@@ -207,7 +207,10 @@
in distributed mode.
(Sharad Agarwal, Patrick O'Leary, Sabyasachi Dalal, Stu Hood,
ryan, yonik)
-
+
+41. SOLR-356: Pluggable functions (value sources) that allow
+ registration of new functions via solrconfig.xml
+ (Doug Daniels via yonik)
Changes in runtime behavior
Modified: lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java?rev=631979&r1=631978&r2=631979&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/core/SolrCore.java Thu Feb 28 06:11:23 2008
@@ -63,6 +63,7 @@
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.QParserPlugin;
+import org.apache.solr.search.ValueSourceParser;
import org.apache.solr.update.DirectUpdateHandler;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.update.UpdateHandler;
@@ -329,6 +330,7 @@
initWriters();
initQParsers();
+ initValueSourceParsers();
this.searchComponents = loadSearchComponents( config );
@@ -1038,6 +1040,36 @@
if (plugin != null) return plugin;
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown query type '"+parserName+"'");
}
+
+ private final HashMap<String, ValueSourceParser> valueSourceParsers = new HashMap<String, ValueSourceParser>();
+
+ /** Configure the ValueSource (function) plugins */
+ private void initValueSourceParsers() {
+ String xpath = "valueSourceParser";
+ NodeList nodes = (NodeList) solrConfig.evaluate(xpath, XPathConstants.NODESET);
+
+ NamedListPluginLoader<ValueSourceParser> loader =
+ new NamedListPluginLoader<ValueSourceParser>( "[solrconfig.xml] "+xpath, valueSourceParsers);
+
+ loader.load( solrConfig.getResourceLoader(), nodes );
+
+ // default value source parsers
+ for (Map.Entry<String, ValueSourceParser> entry : ValueSourceParser.standardValueSourceParsers.entrySet()) {
+ try {
+ String name = entry.getKey();
+ ValueSourceParser valueSourceParser = entry.getValue();
+ valueSourceParsers.put(name, valueSourceParser);
+ valueSourceParser.init(null);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+ }
+
+ public ValueSourceParser getValueSourceParser(String parserName) {
+ return valueSourceParsers.get(parserName);
+ }
+
}
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java?rev=631979&r1=631978&r2=631979&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java Thu Feb 28 06:11:23 2008
@@ -26,18 +26,18 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
public class FunctionQParser extends QParser {
+
+ protected QueryParsing.StrParser sp;
+
public FunctionQParser(String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
}
- QueryParsing.StrParser sp;
-
public Query parse() throws ParseException {
sp = new QueryParsing.StrParser(getString());
- ValueSource vs = parseValSource();
+ ValueSource vs = parseValueSource();
/*** boost promoted to top-level query type to avoid this hack
@@ -52,236 +52,173 @@
return new FunctionQuery(vs);
}
- private abstract static class VSParser {
- abstract ValueSource parse(FunctionQParser fp) throws ParseException;
- }
-
- private static Map<String, VSParser> vsParsers = new HashMap<String, VSParser>();
- static {
- vsParsers.put("ord", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- String field = fp.sp.getId();
- return new OrdFieldSource(field);
- }
- });
- vsParsers.put("rord", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- String field = fp.sp.getId();
- return new ReverseOrdFieldSource(field);
- }
- });
- vsParsers.put("linear", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- fp.sp.expect(",");
- float slope = fp.sp.getFloat();
- fp.sp.expect(",");
- float intercept = fp.sp.getFloat();
- return new LinearFloatFunction(source,slope,intercept);
- }
- });
- vsParsers.put("max", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- fp.sp.expect(",");
- float val = fp.sp.getFloat();
- return new MaxFloatFunction(source,val);
- }
- });
- vsParsers.put("recip", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- fp.sp.expect(",");
- float m = fp.sp.getFloat();
- fp.sp.expect(",");
- float a = fp.sp.getFloat();
- fp.sp.expect(",");
- float b = fp.sp.getFloat();
- return new ReciprocalFloatFunction(source,m,a,b);
- }
- });
- vsParsers.put("scale", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- fp.sp.expect(",");
- float min = fp.sp.getFloat();
- fp.sp.expect(",");
- float max = fp.sp.getFloat();
- return new ScaleFloatFunction(source,min,max);
- }
- });
- vsParsers.put("pow", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource a = fp.parseValSource();
- fp.sp.expect(",");
- ValueSource b = fp.parseValSource();
- return new PowFloatFunction(a,b);
- }
- });
- vsParsers.put("div", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource a = fp.parseValSource();
- fp.sp.expect(",");
- ValueSource b = fp.parseValSource();
- return new DivFloatFunction(a,b);
- }
- });
- vsParsers.put("map", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- fp.sp.expect(",");
- float min = fp.sp.getFloat();
- fp.sp.expect(",");
- float max = fp.sp.getFloat();
- fp.sp.expect(",");
- float target = fp.sp.getFloat();
- return new RangeMapFloatFunction(source,min,max,target);
- }
- });
- vsParsers.put("sqrt", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- return new SimpleFloatFunction(source) {
- protected String name() {
- return "sqrt";
- }
- protected float func(int doc, DocValues vals) {
- return (float)Math.sqrt(vals.floatVal(doc));
- }
- };
- }
- });
- vsParsers.put("log", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- return new SimpleFloatFunction(source) {
- protected String name() {
- return "log";
- }
- protected float func(int doc, DocValues vals) {
- return (float)Math.log10(vals.floatVal(doc));
- }
- };
- }
- });
- vsParsers.put("abs", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- ValueSource source = fp.parseValSource();
- return new SimpleFloatFunction(source) {
- protected String name() {
- return "abs";
- }
- protected float func(int doc, DocValues vals) {
- return (float)Math.abs(vals.floatVal(doc));
- }
- };
- }
- });
- vsParsers.put("sum", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- List<ValueSource> sources = fp.parseValueSourceList();
- return new SumFloatFunction(sources.toArray(new ValueSource[sources.size()]));
- }
- });
- vsParsers.put("product", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- List<ValueSource> sources = fp.parseValueSourceList();
- return new ProductFloatFunction(sources.toArray(new ValueSource[sources.size()]));
- }
- });
- vsParsers.put("query", new VSParser() {
- // boost(query($q),rating)
- ValueSource parse(FunctionQParser fp) throws ParseException {
- Query q = fp.getNestedQuery();
- float defVal = 0.0f;
- if (fp.sp.opt(",")) {
- defVal = fp.sp.getFloat();
- }
- return new QueryValueSource(q, defVal);
- }
- });
- vsParsers.put("boost", new VSParser() {
- ValueSource parse(FunctionQParser fp) throws ParseException {
- Query q = fp.getNestedQuery();
- fp.sp.expect(",");
- ValueSource vs = fp.parseValSource();
- BoostedQuery bq = new BoostedQuery(q, vs);
-System.out.println("Constructed Boostedquery " + bq);
- return new QueryValueSource(bq, 0.0f);
- }
- });
+ /**
+ * Are there more arguments in the argument list being parsed?
+ *
+ * @return whether more args exist
+ * @throws ParseException
+ */
+ public boolean hasMoreArguments() throws ParseException {
+ int ch = sp.peek();
+ /* determine whether the function is ending with a paren or end of str */
+ return (! (ch == 0 || ch == ')') );
}
-
- private List<ValueSource> parseValueSourceList() throws ParseException {
+
+ /**
+ * TODO: Doc
+ *
+ * @return
+ * @throws ParseException
+ */
+ public String parseId() throws ParseException {
+ String value = sp.getId();
+ consumeArgumentDelimiter();
+ return value;
+ }
+
+ /**
+ * Parse a float.
+ *
+ * @return Float
+ * @throws ParseException
+ */
+ public Float parseFloat() throws ParseException {
+ float value = sp.getFloat();
+ consumeArgumentDelimiter();
+ return value;
+ }
+
+ /**
+ * Parse a list of ValueSource. Must be the final set of arguments
+ * to a ValueSource.
+ *
+ * @return List<ValueSource>
+ * @throws ParseException
+ */
+ public List<ValueSource> parseValueSourceList() throws ParseException {
List<ValueSource> sources = new ArrayList<ValueSource>(3);
for (;;) {
- sources.add(parseValSource());
- char ch = sp.peek();
- if (ch==')') break;
- sp.expect(",");
+ sources.add(parseValueSource(false));
+ if (! consumeArgumentDelimiter()) break;
}
return sources;
}
- private ValueSource parseValSource() throws ParseException {
- int ch = sp.peek();
- if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
- return new ConstValueSource(sp.getFloat());
- }
-
- String id = sp.getId();
- if (sp.opt("(")) {
- // a function... look it up.
- VSParser argParser = vsParsers.get(id);
- if (argParser==null) {
- throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
- }
- ValueSource vs = argParser.parse(this);
- sp.expect(")");
- return vs;
- }
-
- SchemaField f = req.getSchema().getField(id);
- return f.getType().getValueSource(f, this);
- }
-
- private Query getNestedQuery() throws ParseException {
+ /**
+ * Parse an individual ValueSource.
+ *
+ * @return
+ * @throws ParseException
+ */
+ public ValueSource parseValueSource() throws ParseException {
+ /* consume the delimiter afterward for an external call to parseValueSource */
+ return parseValueSource(true);
+ }
+
+ /**
+ * TODO: Doc
+ *
+ * @return
+ * @throws ParseException
+ */
+ public Query parseNestedQuery() throws ParseException {
+ Query nestedQuery;
+
if (sp.opt("$")) {
String param = sp.getId();
sp.pos += param.length();
String qstr = getParam(param);
qstr = qstr==null ? "" : qstr;
- return subQuery(qstr, null).parse();
+ nestedQuery = subQuery(qstr, null).parse();
}
-
- int start = sp.pos;
- int end = sp.pos;
- String v = sp.val;
-
- String qs = v.substring(start);
- HashMap nestedLocalParams = new HashMap<String,String>();
- end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams());
-
- QParser sub;
-
- if (end>start) {
- if (nestedLocalParams.get(QueryParsing.V) != null) {
- // value specified directly in local params... so the end of the
- // query should be the end of the local params.
- sub = subQuery(qs.substring(0, end), null);
+ else {
+ int start = sp.pos;
+ int end = sp.pos;
+ String v = sp.val;
+
+ String qs = v.substring(start);
+ HashMap nestedLocalParams = new HashMap<String,String>();
+ end = QueryParsing.parseLocalParams(qs, start, nestedLocalParams, getParams());
+
+ QParser sub;
+
+ if (end>start) {
+ if (nestedLocalParams.get(QueryParsing.V) != null) {
+ // value specified directly in local params... so the end of the
+ // query should be the end of the local params.
+ sub = subQuery(qs.substring(0, end), null);
+ } else {
+ // value here is *after* the local params... ask the parser.
+ sub = subQuery(qs, null);
+ // int subEnd = sub.findEnd(')');
+ // TODO.. implement functions to find the end of a nested query
+ throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'");
+ }
} else {
- // value here is *after* the local params... ask the parser.
- sub = subQuery(qs, null);
- // int subEnd = sub.findEnd(')');
- // TODO.. implement functions to find the end of a nested query
- throw new ParseException("Nested local params must have value in v parameter. got '" + qs + "'");
+ throw new ParseException("Nested function query must use $param or <!v=value> forms. got '" + qs + "'");
+ }
+
+ sp.pos += end-start; // advance past nested query
+ nestedQuery = sub.getQuery();
+ }
+ consumeArgumentDelimiter();
+
+ return nestedQuery;
+ }
+
+ /**
+ * Parse an individual value source.
+ *
+ * @param doConsumeDelimiter whether to consume a delimiter following the ValueSource
+ * @return
+ * @throws ParseException
+ */
+ protected ValueSource parseValueSource(boolean doConsumeDelimiter) throws ParseException {
+ ValueSource valueSource;
+
+ int ch = sp.peek();
+ if (ch>='0' && ch<='9' || ch=='.' || ch=='+' || ch=='-') {
+ valueSource = new ConstValueSource(sp.getFloat());
+ }
+ else {
+ String id = sp.getId();
+ if (sp.opt("(")) {
+ // a function... look it up.
+ ValueSourceParser argParser = req.getCore().getValueSourceParser(id);
+ if (argParser==null) {
+ throw new ParseException("Unknown function " + id + " in FunctionQuery(" + sp + ")");
+ }
+ valueSource = argParser.parse(this);
+ sp.expect(")");
}
- } else {
- throw new ParseException("Nested function query must use $param or <!v=value> forms. got '" + qs + "'");
+ else {
+ SchemaField f = req.getSchema().getField(id);
+ valueSource = f.getType().getValueSource(f, this);
+ }
+ }
+
+ if (doConsumeDelimiter)
+ consumeArgumentDelimiter();
+
+ return valueSource;
+ }
+
+ /**
+ * Consume an argument delimiter (a comma) from the token stream.
+ * Only consumes if more arguments should exist (no ending parens or end of string).
+ *
+ * @return whether a delimiter was consumed
+ * @throws ParseException
+ */
+ protected boolean consumeArgumentDelimiter() throws ParseException {
+ /* if a list of args is ending, don't expect the comma */
+ if (hasMoreArguments()) {
+ sp.expect(",");
+ return true;
}
-
- sp.pos += end-start; // advance past nested query
- return sub.getQuery();
+
+ return false;
}
+
}
Modified: lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=631979&r1=631978&r2=631979&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java Thu Feb 28 06:11:23 2008
@@ -17,7 +17,17 @@
package org.apache.solr.search.function;
+import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
+import org.apache.lucene.queryParser.ParseException;
+import org.apache.lucene.search.Query;
+import org.apache.solr.search.ValueSourceParser;
+import org.apache.solr.search.FunctionQParser;
+import org.apache.solr.search.function.DocValues;
+import org.apache.solr.search.function.QueryValueSource;
+import org.apache.solr.search.function.SimpleFloatFunction;
+import org.apache.solr.search.function.ValueSource;
import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import java.util.ArrayList;
@@ -36,7 +46,7 @@
public class TestFunctionQuery extends AbstractSolrTestCase {
public String getSchemaFile() { return "schema11.xml"; }
- public String getSolrConfigFile() { return "solrconfig.xml"; }
+ public String getSolrConfigFile() { return "solrconfig-functionquery.xml"; }
public String getCoreName() { return "basic"; }
public void setUp() throws Exception {
@@ -153,6 +163,11 @@
// test that infinity doesn't mess up scale function
singleTest(field,"scale(log(\0),-1000,1000)",100,1000);
+ // test use of an ValueSourceParser plugin: nvl function
+ singleTest(field,"nvl(\0,1)", 0, 1, 100, 100);
+
+ // compose the ValueSourceParser plugin function with another function
+ singleTest(field, "nvl(sum(0,\0),1)", 0, 1, 100, 100);
}
public void testFunctions() {
@@ -232,5 +247,4 @@
}
}
-
}