You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by yo...@apache.org on 2009/08/21 00:05:10 UTC
svn commit: r806374 - in /lucene/solr/trunk: ./
src/java/org/apache/solr/schema/ src/java/org/apache/solr/search/
src/java/org/apache/solr/search/function/
src/test/org/apache/solr/search/function/
Author: yonik
Date: Thu Aug 20 22:05:09 2009
New Revision: 806374
URL: http://svn.apache.org/viewvc?rev=806374&view=rev
Log:
SOLR-1368: add ms() and sub() functions
Added:
lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java (with props)
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java
lucene/solr/trunk/src/java/org/apache/solr/schema/TrieDateField.java
lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java
lucene/solr/trunk/src/java/org/apache/solr/search/ValueSourceParser.java
lucene/solr/trunk/src/java/org/apache/solr/search/function/PowFloatFunction.java
lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Thu Aug 20 22:05:09 2009
@@ -276,6 +276,10 @@
71. SOLR-1373: Add Filter query to admin/form.jsp
(Jason Rutherglen via hossman)
+72. SOLR-1368: Add ms() function query for getting milliseconds from dates and for
+ high precision date subtraction, add sub() for subtracting other arguments.
+ (yonik)
+
Optimizations
----------------------
1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/DateField.java Thu Aug 20 22:05:09 2009
@@ -17,31 +17,26 @@
package org.apache.solr.schema;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.request.XMLWriter;
-import org.apache.solr.request.TextResponseWriter;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.search.SortField;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.index.IndexReader;
-import org.apache.solr.search.function.*;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.TextResponseWriter;
+import org.apache.solr.request.XMLWriter;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.function.*;
import org.apache.solr.util.DateMathParser;
-
-import java.util.Map;
+
import java.io.IOException;
+import java.text.*;
import java.util.Date;
-import java.util.TimeZone;
import java.util.Locale;
-import java.text.DecimalFormatSymbols;
-import java.text.SimpleDateFormat;
-import java.text.DateFormat;
-import java.text.NumberFormat;
-import java.text.DecimalFormat;
-import java.text.ParsePosition;
-import java.text.ParseException;
-import java.text.FieldPosition;
+import java.util.Map;
+import java.util.TimeZone;
// TODO: make a FlexibleDateField that can accept dates in multiple
// formats, better for human entered dates.
@@ -261,6 +256,62 @@
protected Date parseDate(String s) throws ParseException {
return fmtThreadLocal.get().parse(s);
}
+
+ /** Parse a date string in the standard format, or any supported by DateUtil.parseDate */
+ public Date parseDateLenient(String s, SolrQueryRequest req) throws ParseException {
+ // request could define timezone in the future
+ try {
+ return fmtThreadLocal.get().parse(s);
+ } catch (Exception e) {
+ return DateUtil.parseDate(s);
+ }
+ }
+
+ /**
+ * Parses a String which may be a date
+ * followed by an optional math expression.
+ * @param now an optional fixed date to use as "NOW" in the DateMathParser
+ * @param val the string to parse
+ */
+ public Date parseMathLenient(Date now, String val, SolrQueryRequest req) {
+ String math = null;
+ final DateMathParser p = new DateMathParser(MATH_TZ, MATH_LOCALE);
+
+ if (null != now) p.setNow(now);
+
+ if (val.startsWith(NOW)) {
+ math = val.substring(NOW.length());
+ } else {
+ final int zz = val.indexOf(Z);
+ if (0 < zz) {
+ math = val.substring(zz+1);
+ try {
+ // p.setNow(toObject(val.substring(0,zz)));
+ p.setNow(parseDateLenient(val.substring(0,zz+1), req));
+ } catch (ParseException e) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid Date in Date Math String:'"
+ +val+'\'',e);
+ }
+ } else {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid Date String:'" +val+'\'');
+ }
+ }
+
+ if (null == math || math.equals("")) {
+ return p.getNow();
+ }
+
+ try {
+ return p.parseMath(math);
+ } catch (ParseException e) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid Date Math String:'" +val+'\'',e);
+ }
+ }
+
+
/**
* Thread safe DateFormat that can <b>format</b> in the canonical
Modified: lucene/solr/trunk/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/schema/TrieDateField.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/schema/TrieDateField.java Thu Aug 20 22:05:09 2009
@@ -42,7 +42,7 @@
public class TrieDateField extends DateField {
protected int precisionStepArg = TrieField.DEFAULT_PRECISION_STEP; // the one passed in or defaulted
- protected int precisionStep; // normalized
+ protected int precisionStep = precisionStepArg; // normalized
@Override
protected void init(IndexSchema schema, Map<String, String> args) {
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/FunctionQParser.java Thu Aug 20 22:05:09 2009
@@ -86,6 +86,44 @@
consumeArgumentDelimiter();
return value;
}
+
+ public String parseArg() throws ParseException {
+ sp.eatws();
+ char ch = sp.peek();
+ String val = null;
+ switch (ch) {
+ case ')': return null;
+ case '$':
+ sp.pos++;
+ String param = sp.getId();
+ val = getParam(param);
+ break;
+ case '\'':
+ case '"':
+ val = sp.getQuotedString();
+ break;
+ default:
+ // read unquoted literal ended by whitespace ',' or ')'
+ // there is no escaping.
+ int valStart = sp.pos;
+ for (;;) {
+ if (sp.pos >= sp.end) {
+ throw new ParseException("Missing end to unquoted value starting at " + valStart + " str='" + sp.val +"'");
+ }
+ char c = sp.val.charAt(sp.pos);
+ if (c==')' || c==',' || Character.isWhitespace(c)) {
+ val = sp.val.substring(valStart, sp.pos);
+ break;
+ }
+ sp.pos++;
+ }
+ }
+
+ sp.eatws();
+ consumeArgumentDelimiter();
+ return val;
+ }
+
/**
* Parse a list of ValueSource. Must be the final set of arguments
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/ValueSourceParser.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/ValueSourceParser.java Thu Aug 20 22:05:09 2009
@@ -19,12 +19,20 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Date;
+import java.io.IOException;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
+import org.apache.lucene.index.IndexReader;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.SolrException;
import org.apache.solr.search.function.*;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
+import org.apache.solr.schema.TrieDateField;
+import org.apache.solr.schema.DateField;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.LegacyDateField;
/**
* A factory that parses user queries to generate ValueSource instances.
@@ -232,6 +240,24 @@
}
});
+ standardValueSourceParsers.put("sub", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ ValueSource a = fp.parseValueSource();
+ ValueSource b = fp.parseValueSource();
+ return new DualFloatFunction(a,b) {
+ protected String name() {
+ return "sub";
+ }
+ protected float func(int doc, DocValues aVals, DocValues bVals) {
+ return aVals.floatVal(doc) - bVals.floatVal(doc);
+ }
+ };
+ }
+
+ public void init(NamedList args) {
+ }
+
+ });
standardValueSourceParsers.put("query", new ValueSourceParser() {
// boost(query($q),rating)
public ValueSource parse(FunctionQParser fp) throws ParseException {
@@ -259,6 +285,149 @@
}
});
+ standardValueSourceParsers.put("ms", new DateValueSourceParser());
}
}
+
+
+
+
+class DateValueSourceParser extends ValueSourceParser {
+ DateField df = new TrieDateField();
+ public void init(NamedList args) {}
+
+ public Date getDate(FunctionQParser fp, String arg) {
+ if (arg==null) return null;
+ if (arg.startsWith("NOW") || (arg.length()>0 && Character.isDigit(arg.charAt(0)))) {
+ return df.parseMathLenient(null, arg, fp.req);
+ }
+ return null;
+ }
+
+ public ValueSource getValueSource(FunctionQParser fp, String arg) {
+ if (arg==null) return null;
+ SchemaField f = fp.req.getSchema().getField(arg);
+ if (f.getType().getClass() == DateField.class || f.getType().getClass() == LegacyDateField.class) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Can't use ms() function on non-numeric legacy date field " + arg);
+ }
+ return f.getType().getValueSource(f, fp);
+ }
+
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ String first = fp.parseArg();
+ String second = fp.parseArg();
+ if (first==null) first="NOW";
+
+ Date d1=getDate(fp,first);
+ ValueSource v1 = d1==null ? getValueSource(fp, first) : null;
+
+ Date d2=getDate(fp,second);
+ ValueSource v2 = d2==null ? getValueSource(fp, second) : null;
+
+ // d constant
+ // v field
+ // dd constant
+ // dv subtract field from constant
+ // vd subtract constant from field
+ // vv subtract fields
+
+ final long ms1 = (d1 == null) ? 0 : d1.getTime();
+ final long ms2 = (d2 == null) ? 0 : d2.getTime();
+
+ // "d,dd" handle both constant cases
+
+ if (d1 != null && v2==null) {
+ return new LongConstValueSource(ms1-ms2);
+ }
+
+ // "v" just the date field
+ if (v1 != null && v2==null && d2==null) {
+ return v1;
+ }
+
+
+ // "dv"
+ if (d1!=null && v2!=null)
+ return new DualFloatFunction(new LongConstValueSource(ms1), v2) {
+ protected String name() {
+ return "ms";
+ }
+ protected float func(int doc, DocValues aVals, DocValues bVals) {
+ return ms1 - bVals.longVal(doc);
+ }
+ };
+
+ // "vd"
+ if (v1!=null && d2!=null)
+ return new DualFloatFunction(v1, new LongConstValueSource(ms2)) {
+ protected String name() {
+ return "ms";
+ }
+ protected float func(int doc, DocValues aVals, DocValues bVals) {
+ return aVals.longVal(doc) - ms2;
+ }
+ };
+
+ // "vv"
+ if (v1!=null && v2!=null)
+ return new DualFloatFunction(v1,v2) {
+ protected String name() {
+ return "ms";
+ }
+ protected float func(int doc, DocValues aVals, DocValues bVals) {
+ return aVals.longVal(doc) - bVals.longVal(doc);
+ }
+ };
+
+ return null; // shouldn't happen
+ }
+
+}
+
+
+// Private for now - we need to revisit how to handle typing in function queries
+class LongConstValueSource extends ValueSource {
+ final long constant;
+
+ public LongConstValueSource(long constant) {
+ this.constant = constant;
+ }
+
+ public String description() {
+ return "const(" + constant + ")";
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return constant;
+ }
+ public int intVal(int doc) {
+ return (int)constant;
+ }
+ public long longVal(int doc) {
+ return constant;
+ }
+ public double doubleVal(int doc) {
+ return constant;
+ }
+ public String strVal(int doc) {
+ return Long.toString(constant);
+ }
+ public String toString(int doc) {
+ return description();
+ }
+ };
+ }
+
+ public int hashCode() {
+ return (int)constant + (int)(constant>>>32);
+ }
+
+ public boolean equals(Object o) {
+ if (LongConstValueSource.class != o.getClass()) return false;
+ LongConstValueSource other = (LongConstValueSource)o;
+ return this.constant == other.constant;
+ }
+}
\ No newline at end of file
Added: lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java?rev=806374&view=auto
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java (added)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java Thu Aug 20 22:05:09 2009
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+
+import java.io.IOException;
+
+public abstract class DualFloatFunction extends ValueSource {
+ protected final ValueSource a;
+ protected final ValueSource b;
+
+ /**
+ * @param a the base.
+ * @param b the exponent.
+ */
+ public DualFloatFunction(ValueSource a, ValueSource b) {
+ this.a = a;
+ this.b = b;
+ }
+
+ protected abstract String name();
+ protected abstract float func(int doc, DocValues aVals, DocValues bVals);
+
+ public String description() {
+ return name() + "(" + a.description() + "," + b.description() + ")";
+ }
+
+ public DocValues getValues(IndexReader reader) throws IOException {
+ final DocValues aVals = a.getValues(reader);
+ final DocValues bVals = b.getValues(reader);
+ return new DocValues() {
+ public float floatVal(int doc) {
+ return func(doc, aVals, bVals);
+ }
+ public int intVal(int doc) {
+ return (int)floatVal(doc);
+ }
+ public long longVal(int doc) {
+ return (long)floatVal(doc);
+ }
+ public double doubleVal(int doc) {
+ return floatVal(doc);
+ }
+ public String strVal(int doc) {
+ return Float.toString(floatVal(doc));
+ }
+ public String toString(int doc) {
+ return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
+ }
+ };
+ }
+
+ public int hashCode() {
+ int h = a.hashCode();
+ h ^= (h << 13) | (h >>> 20);
+ h += b.hashCode();
+ h ^= (h << 23) | (h >>> 10);
+ h += name().hashCode();
+ return h;
+ }
+
+ public boolean equals(Object o) {
+ if (this.getClass() != o.getClass()) return false;
+ DualFloatFunction other = (DualFloatFunction)o;
+ return this.a.equals(other.a)
+ && this.b.equals(other.b);
+ }
+}
Propchange: lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/solr/trunk/src/java/org/apache/solr/search/function/DualFloatFunction.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/solr/trunk/src/java/org/apache/solr/search/function/PowFloatFunction.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/search/function/PowFloatFunction.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/search/function/PowFloatFunction.java (original)
+++ lucene/solr/trunk/src/java/org/apache/solr/search/function/PowFloatFunction.java Thu Aug 20 22:05:09 2009
@@ -42,64 +42,3 @@
}
-abstract class DualFloatFunction extends ValueSource {
- protected final ValueSource a;
- protected final ValueSource b;
-
- /**
- * @param a the base.
- * @param b the exponent.
- */
- public DualFloatFunction(ValueSource a, ValueSource b) {
- this.a = a;
- this.b = b;
- }
-
- protected abstract String name();
- protected abstract float func(int doc, DocValues aVals, DocValues bVals);
-
- public String description() {
- return name() + "(" + a.description() + "," + b.description() + ")";
- }
-
- public DocValues getValues(IndexReader reader) throws IOException {
- final DocValues aVals = a.getValues(reader);
- final DocValues bVals = b.getValues(reader);
- return new DocValues() {
- public float floatVal(int doc) {
- return func(doc, aVals, bVals);
- }
- public int intVal(int doc) {
- return (int)floatVal(doc);
- }
- public long longVal(int doc) {
- return (long)floatVal(doc);
- }
- public double doubleVal(int doc) {
- return floatVal(doc);
- }
- public String strVal(int doc) {
- return Float.toString(floatVal(doc));
- }
- public String toString(int doc) {
- return name() + '(' + aVals.toString(doc) + ',' + bVals.toString(doc) + ')';
- }
- };
- }
-
- public int hashCode() {
- int h = a.hashCode();
- h ^= (h << 13) | (h >>> 20);
- h += b.hashCode();
- h ^= (h << 23) | (h >>> 10);
- h += name().hashCode();
- return h;
- }
-
- public boolean equals(Object o) {
- if (this.getClass() != o.getClass()) return false;
- DualFloatFunction other = (DualFloatFunction)o;
- return this.a.equals(other.a)
- && this.b.equals(other.b);
- }
-}
\ No newline at end of file
Modified: lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=806374&r1=806373&r2=806374&view=diff
==============================================================================
--- lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/solr/trunk/src/test/org/apache/solr/search/function/TestFunctionQuery.java Thu Aug 20 22:05:09 2009
@@ -154,6 +154,8 @@
singleTest(field,"sum(\0,\0)", 10, 20);
singleTest(field,"sum(\0,\0,5)", 10, 25);
+ singleTest(field,"sub(\0,1)", 10, 9);
+
singleTest(field,"product(\0,1)", 10, 10);
singleTest(field,"product(\0,-2,-4)", 10, 80);
@@ -276,7 +278,7 @@
}
public void testGeneral() {
- assertU(adoc("id","1"));
+ assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
assertU(adoc("id","2"));
assertU(commit()); // create more than one segment
assertU(adoc("id","3"));
@@ -292,5 +294,15 @@
assertQ(req("fl","*,score","q", "{!func}top(ord(id))", "fq","id:6"), "//float[@name='score']='6.0'");
assertQ(req("fl","*,score","q", "{!func}rord(id)", "fq","id:1"),"//float[@name='score']='6.0'");
assertQ(req("fl","*,score","q", "{!func}top(rord(id))", "fq","id:1"),"//float[@name='score']='6.0'");
+
+
+ // test that we can subtract dates to millisecond precision
+ assertQ(req("fl","*,score","q", "{!func}ms(a_tdt,b_tdt)", "fq","id:1"), "//float[@name='score']='-1.0'");
+ assertQ(req("fl","*,score","q", "{!func}ms(b_tdt,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+ assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,2009-08-31T12:10:10.124Z)", "fq","id:1"), "//float[@name='score']='1.0'");
+ assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.124Z,a_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+ assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z,b_tdt)", "fq","id:1"), "//float[@name='score']='1.0'");
+
+ assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z/SECOND,2009-08-31T12:10:10.124Z/SECOND)", "fq","id:1"), "//float[@name='score']='0.0'");
}
}
\ No newline at end of file