You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2010/06/14 22:54:54 UTC
svn commit: r954640 - in /lucene/dev/trunk/solr: ./
src/java/org/apache/solr/search/ src/java/org/apache/solr/search/function/
src/test/org/apache/solr/search/function/
Author: yonik
Date: Mon Jun 14 20:54:54 2010
New Revision: 954640
URL: http://svn.apache.org/viewvc?rev=954640&view=rev
Log:
SOLR-1932: New relevancy function queries: termfreq, tf, docfreq, idf, norm, maxdoc, numdocs
Added:
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrQueryParser.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/ValueSourceParser.java
lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=954640&r1=954639&r2=954640&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon Jun 14 20:54:54 2010
@@ -176,6 +176,11 @@ New Features
* SOLR-1915: DebugComponent now supports using a NamedList to model
Explanation objects in it's responses instead of
Explanation.toString (hossman)
+
+* SOLR-1932: New relevancy function queries: termfreq, tf, docfreq, idf
+ norm, maxdoc, numdocs. (yonik)
+
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrQueryParser.java?rev=954640&r1=954639&r2=954640&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrQueryParser.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrQueryParser.java Mon Jun 14 20:54:54 2010
@@ -54,8 +54,6 @@ import org.apache.solr.schema.TextField;
* </p>
*
* @see QueryParsing#parseFunction
- * @see ConstantScoreRangeQuery
- * @see ConstantScorePrefixQuery
*/
public class SolrQueryParser extends QueryParser {
protected final IndexSchema schema;
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/ValueSourceParser.java?rev=954640&r1=954639&r2=954640&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/ValueSourceParser.java Mon Jun 14 20:54:54 2010
@@ -17,22 +17,24 @@
package org.apache.solr.search;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spell.JaroWinklerDistance;
import org.apache.lucene.search.spell.LevensteinDistance;
import org.apache.lucene.search.spell.NGramDistance;
import org.apache.lucene.search.spell.StringDistance;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.UnicodeUtil;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.schema.DateField;
-import org.apache.solr.schema.LegacyDateField;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.TrieDateField;
+import org.apache.solr.schema.*;
import org.apache.solr.search.function.*;
import org.apache.solr.search.function.distance.*;
+import org.apache.solr.util.ByteUtils;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import java.io.IOException;
@@ -451,6 +453,81 @@ public abstract class ValueSourceParser
return new DoubleConstValueSource(Math.E);
}
});
+
+
+ addParser("docfreq", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ TInfo tinfo = parseTerm(fp);
+ return new DocFreqValueSource(tinfo.field, tinfo.val, tinfo.indexedField, tinfo.indexedBytes);
+ }
+ });
+
+ addParser("idf", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ TInfo tinfo = parseTerm(fp);
+ return new IDFValueSource(tinfo.field, tinfo.val, tinfo.indexedField, tinfo.indexedBytes);
+ }
+ });
+
+ addParser("termfreq", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ TInfo tinfo = parseTerm(fp);
+ return new TermFreqValueSource(tinfo.field, tinfo.val, tinfo.indexedField, tinfo.indexedBytes);
+ }
+ });
+
+ addParser("tf", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ TInfo tinfo = parseTerm(fp);
+ return new TFValueSource(tinfo.field, tinfo.val, tinfo.indexedField, tinfo.indexedBytes);
+ }
+ });
+
+ addParser("norm", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ String field = fp.parseArg();
+ return new NormValueSource(field);
+ }
+ });
+
+ addParser("maxdoc", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ return new MaxDocValueSource();
+ }
+ });
+
+ addParser("numdocs", new ValueSourceParser() {
+ public ValueSource parse(FunctionQParser fp) throws ParseException {
+ return new NumDocsValueSource();
+ }
+ });
+ }
+
+ private static TInfo parseTerm(FunctionQParser fp) throws ParseException {
+ TInfo tinfo = new TInfo();
+
+ tinfo.indexedField = tinfo.field = fp.parseArg();
+ tinfo.val = fp.parseArg();
+ tinfo.indexedBytes = new BytesRef();
+
+ FieldType ft = fp.getReq().getSchema().getFieldTypeNoEx(tinfo.field);
+ if (ft == null) ft = new StrField();
+
+ if (ft instanceof TextField) {
+ // need to do analyisis on the term
+ String indexedVal = tinfo.val;
+ Query q = ft.getFieldQuery(fp, fp.getReq().getSchema().getFieldOrNull(tinfo.field), tinfo.val);
+ if (q instanceof TermQuery) {
+ Term term = ((TermQuery)q).getTerm();
+ tinfo.indexedField = term.field();
+ indexedVal = term.text();
+ }
+ UnicodeUtil.UTF16toUTF8(indexedVal, 0, indexedVal.length(), tinfo.indexedBytes);
+ } else {
+ ft.readableToIndexed(tinfo.val, tinfo.indexedBytes);
+ }
+
+ return tinfo;
}
private static void splitSources(int dim, List<ValueSource> sources, List<ValueSource> dest1, List<ValueSource> dest2) {
@@ -502,6 +579,14 @@ public abstract class ValueSourceParser
MultiValueSource mv1;
MultiValueSource mv2;
}
+
+ private static class TInfo {
+ String field;
+ String val;
+ String indexedField;
+ BytesRef indexedBytes;
+ }
+
}
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.util.ByteUtils;
+
+import java.io.IOException;
+import java.util.Map;
+
+
+class ConstIntDocValues extends DocValues {
+ final int ival;
+ final float fval;
+ final double dval;
+ final long lval;
+ final String sval;
+ final ValueSource parent;
+
+ ConstIntDocValues(int val, ValueSource parent) {
+ ival = val;
+ fval = val;
+ dval = val;
+ lval = val;
+ sval = Integer.toString(val);
+ this.parent = parent;
+ }
+
+ public float floatVal(int doc) {
+ return fval;
+ }
+ public int intVal(int doc) {
+ return ival;
+ }
+ public long longVal(int doc) {
+ return lval;
+ }
+ public double doubleVal(int doc) {
+ return dval;
+ }
+ public String strVal(int doc) {
+ return sval;
+ }
+ public String toString(int doc) {
+ return parent.description() + '=' + sval;
+ }
+}
+
+class ConstDoubleDocValues extends DocValues {
+ final int ival;
+ final float fval;
+ final double dval;
+ final long lval;
+ final String sval;
+ final ValueSource parent;
+
+ ConstDoubleDocValues(double val, ValueSource parent) {
+ ival = (int)val;
+ fval = (float)val;
+ dval = val;
+ lval = (long)val;
+ sval = Double.toString(val);
+ this.parent = parent;
+ }
+
+ public float floatVal(int doc) {
+ return fval;
+ }
+ public int intVal(int doc) {
+ return ival;
+ }
+ public long longVal(int doc) {
+ return lval;
+ }
+ public double doubleVal(int doc) {
+ return dval;
+ }
+ public String strVal(int doc) {
+ return sval;
+ }
+ public String toString(int doc) {
+ return parent.description() + '=' + sval;
+ }
+}
+
+abstract class FloatDocValues extends DocValues {
+ protected final ValueSource vs;
+
+ public FloatDocValues(ValueSource vs) {
+ this.vs = vs;
+ }
+
+ @Override
+ public byte byteVal(int doc) {
+ return (byte)floatVal(doc);
+ }
+
+ @Override
+ public short shortVal(int doc) {
+ return (short)floatVal(doc);
+ }
+
+ @Override
+ public abstract float floatVal(int doc);
+
+ @Override
+ public int intVal(int doc) {
+ return (int)floatVal(doc);
+ }
+
+ @Override
+ public long longVal(int doc) {
+ return (long)floatVal(doc);
+ }
+
+ @Override
+ public double doubleVal(int doc) {
+ return (double)floatVal(doc);
+ }
+
+ @Override
+ public String strVal(int doc) {
+ return Float.toString(floatVal(doc));
+ }
+
+ @Override
+ public String toString(int doc) {
+ return vs.description() + '=' + strVal(doc);
+ }
+}
+
+abstract class IntDocValues extends DocValues {
+ protected final ValueSource vs;
+
+ public IntDocValues(ValueSource vs) {
+ this.vs = vs;
+ }
+
+ @Override
+ public byte byteVal(int doc) {
+ return (byte)intVal(doc);
+ }
+
+ @Override
+ public short shortVal(int doc) {
+ return (short)intVal(doc);
+ }
+
+ @Override
+ public float floatVal(int doc) {
+ return (float)intVal(doc);
+ }
+
+ @Override
+ public abstract int intVal(int doc);
+
+ @Override
+ public long longVal(int doc) {
+ return (long)intVal(doc);
+ }
+
+ @Override
+ public double doubleVal(int doc) {
+ return (double)intVal(doc);
+ }
+
+ @Override
+ public String strVal(int doc) {
+ return Integer.toString(intVal(doc));
+ }
+
+ @Override
+ public String toString(int doc) {
+ return vs.description() + '=' + strVal(doc);
+ }
+}
+
+
+/**
+ * <code>DocFreqValueSource</code> returns the number of documents containing the term.
+ * @internal
+ */
+public class DocFreqValueSource extends ValueSource {
+ protected String field;
+ protected String indexedField;
+ protected String val;
+ protected BytesRef indexedBytes;
+
+ public DocFreqValueSource(String field, String val, String indexedField, BytesRef indexedBytes) {
+ this.field = field;
+ this.val = val;
+ this.indexedField = indexedField;
+ this.indexedBytes = indexedBytes;
+ }
+
+ public String name() {
+ return "docfreq";
+ }
+
+ @Override
+ public String description() {
+ return name() + '(' + field + ',' + val + ')';
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ Searcher searcher = (Searcher)context.get("searcher");
+ // todo: we need docFreq that takes a BytesRef
+ String strVal = ByteUtils.UTF8toUTF16(indexedBytes);
+ int docfreq = searcher.docFreq(new Term(indexedField, strVal));
+ return new ConstIntDocValues(docfreq, this);
+ }
+
+ @Override
+ public void createWeight(Map context, Searcher searcher) throws IOException {
+ context.put("searcher",searcher);
+ }
+
+ public int hashCode() {
+ return getClass().hashCode() + indexedField.hashCode()*29 + indexedBytes.hashCode();
+ }
+
+ public boolean equals(Object o) {
+ if (this.getClass() != o.getClass()) return false;
+ DocFreqValueSource other = (DocFreqValueSource)o;
+ return this.indexedField.equals(other.indexedField) && this.indexedBytes.equals(other.indexedBytes);
+ }
+}
+
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/DocFreqValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.search.SolrIndexReader;
+import org.apache.solr.util.ByteUtils;
+
+import java.io.IOException;
+import java.util.Map;
+
+/** @internal */
+public class IDFValueSource extends DocFreqValueSource {
+ public IDFValueSource(String field, String val, String indexedField, BytesRef indexedBytes) {
+ super(field, val, indexedField, indexedBytes);
+ }
+
+ @Override
+ public String name() {
+ return "idf";
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ Searcher searcher = (Searcher)context.get("searcher");
+ Similarity sim = searcher.getSimilarity();
+ // todo: we need docFreq that takes a BytesRef
+ String strVal = ByteUtils.UTF8toUTF16(indexedBytes);
+ int docfreq = searcher.docFreq(new Term(indexedField, strVal));
+ float idf = sim.idf(docfreq, searcher.maxDoc());
+ return new ConstDoubleDocValues(idf, this);
+ }
+}
+
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/IDFValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Searcher;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class MaxDocValueSource extends ValueSource {
+ public String name() {
+ return "maxdoc";
+ }
+
+ @Override
+ public String description() {
+ return name() + "()";
+ }
+
+ @Override
+ public void createWeight(Map context, Searcher searcher) throws IOException {
+ context.put("searcher",searcher);
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ Searcher searcher = (Searcher)context.get("searcher");
+ return new ConstIntDocValues(searcher.maxDoc(), this);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return this.getClass() == o.getClass();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getClass().hashCode();
+ }
+}
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/MaxDocValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.solr.search.SolrIndexReader;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class NormValueSource extends ValueSource {
+ protected String field;
+ public NormValueSource(String field) {
+ this.field = field;
+ }
+
+ public String name() {
+ return "norm";
+ }
+
+ @Override
+ public String description() {
+ return name() + '(' + field + ')';
+ }
+
+ @Override
+ public void createWeight(Map context, Searcher searcher) throws IOException {
+ context.put("searcher",searcher);
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ Searcher searcher = (Searcher)context.get("searcher");
+ final Similarity similarity = searcher.getSimilarity();
+ final byte[] norms = reader.norms(field);
+ if (norms == null) {
+ return new ConstDoubleDocValues(0.0, this);
+ }
+
+ return new FloatDocValues(this) {
+ @Override
+ public float floatVal(int doc) {
+ return similarity.decodeNormValue(norms[doc]);
+ }
+ };
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this.getClass() != o.getClass()) return false;
+ return this.field.equals(((NormValueSource)o).field);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getClass().hashCode() + field.hashCode();
+ }
+}
+
+
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NormValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.solr.search.SolrIndexReader;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class NumDocsValueSource extends ValueSource {
+ public String name() {
+ return "numdocs";
+ }
+
+ @Override
+ public String description() {
+ return name() + "()";
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ // Searcher has no numdocs so we must use the reader instead
+ SolrIndexReader topReader = (SolrIndexReader)reader;
+ while (topReader.getParent() != null) topReader = topReader.getParent();
+ return new ConstIntDocValues(topReader.numDocs(), this);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return this.getClass() == o.getClass();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getClass().hashCode();
+ }
+}
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/NumDocsValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,93 @@
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TFValueSource extends TermFreqValueSource {
+ public TFValueSource(String field, String val, String indexedField, BytesRef indexedBytes) {
+ super(field, val, indexedField, indexedBytes);
+ }
+
+ @Override
+ public String name() {
+ return "tf";
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ // use MultiFields, just in case someone did a top() function
+ Fields fields = MultiFields.getFields(reader);
+ final Terms terms = fields.terms(field);
+ final Similarity similarity = ((Searcher)context.get("searcher")).getSimilarity();
+
+ return new FloatDocValues(this) {
+ DocsEnum docs ;
+ int atDoc;
+ int lastDocRequested = -1;
+
+ { reset(); }
+
+ public void reset() throws IOException {
+ // no one should call us for deleted docs?
+ docs = terms.docs(null, indexedBytes, null);
+ if (docs == null) {
+ docs = new DocsEnum() {
+ @Override
+ public int freq() {
+ return 0;
+ }
+
+ @Override
+ public int docID() {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ };
+ }
+ atDoc = -1;
+ }
+
+ @Override
+ public float floatVal(int doc) {
+ try {
+ if (doc < lastDocRequested) {
+ // out-of-order access.... reset
+ reset();
+ }
+ lastDocRequested = doc;
+
+ if (atDoc < doc) {
+ atDoc = docs.advance(doc);
+ }
+
+ if (atDoc > doc) {
+ // term doesn't match this document... either because we hit the
+ // end, or because the next doc is after this doc.
+ return similarity.tf(0);
+ }
+
+ // a match!
+ return similarity.tf(docs.freq());
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "caught exception in function "+description()+" : doc="+doc, e);
+ }
+ }
+ };
+ }
+}
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TFValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java?rev=954640&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java Mon Jun 14 20:54:54 2010
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search.function;
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Searcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrException;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TermFreqValueSource extends DocFreqValueSource {
+ public TermFreqValueSource(String field, String val, String indexedField, BytesRef indexedBytes) {
+ super(field, val, indexedField, indexedBytes);
+ }
+
+ @Override
+ public String name() {
+ return "termfreq";
+ }
+
+ @Override
+ public DocValues getValues(Map context, IndexReader reader) throws IOException {
+ // use MultiFields, just in case someone did a top() function
+ Fields fields = MultiFields.getFields(reader);
+ final Terms terms = fields.terms(field);
+
+ return new IntDocValues(this) {
+ DocsEnum docs ;
+ int atDoc;
+ int lastDocRequested = -1;
+
+ { reset(); }
+
+ public void reset() throws IOException {
+ // no one should call us for deleted docs?
+ docs = terms.docs(null, indexedBytes, null);
+ if (docs == null) {
+ docs = new DocsEnum() {
+ @Override
+ public int freq() {
+ return 0;
+ }
+
+ @Override
+ public int docID() {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return DocIdSetIterator.NO_MORE_DOCS;
+ }
+ };
+ }
+ atDoc = -1;
+ }
+
+ @Override
+ public int intVal(int doc) {
+ try {
+ if (doc < lastDocRequested) {
+ // out-of-order access.... reset
+ reset();
+ }
+ lastDocRequested = doc;
+
+ if (atDoc < doc) {
+ atDoc = docs.advance(doc);
+ }
+
+ if (atDoc > doc) {
+ // term doesn't match this document... either because we hit the
+ // end, or because the next doc is after this doc.
+ return 0;
+ }
+
+ // a match!
+ return docs.freq();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "caught exception in function "+description()+" : doc="+doc, e);
+ }
+ }
+ };
+ }
+}
+
+
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/search/function/TermFreqValueSource.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=954640&r1=954639&r2=954640&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java Mon Jun 14 20:54:54 2010
@@ -17,7 +17,9 @@
package org.apache.solr.search.function;
+import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.Similarity;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -274,15 +276,30 @@ public class TestFunctionQuery extends S
clearIndex();
assertU(adoc("id","1", "a_tdt","2009-08-31T12:10:10.123Z", "b_tdt","2009-08-31T12:10:10.124Z"));
- assertU(adoc("id","2"));
+ assertU(adoc("id","2", "a_t","how now brown cow"));
assertU(commit()); // create more than one segment
- assertU(adoc("id","3"));
+ assertU(adoc("id","3", "a_t","brown cow"));
assertU(adoc("id","4"));
assertU(commit()); // create more than one segment
assertU(adoc("id","5"));
- assertU(adoc("id","6"));
+ assertU(adoc("id","6", "a_t","cow cow cow cow cow"));
assertU(commit());
+ // test relevancy functions
+ assertQ(req("fl","*,score","q", "{!func}numdocs()", "fq","id:6"), "//float[@name='score']='6.0'");
+ assertQ(req("fl","*,score","q", "{!func}maxdoc()", "fq","id:6"), "//float[@name='score']='6.0'");
+ assertQ(req("fl","*,score","q", "{!func}docfreq(a_t,cow)", "fq","id:6"), "//float[@name='score']='3.0'");
+ assertQ(req("fl","*,score","q", "{!func}docfreq('a_t','cow')", "fq","id:6"), "//float[@name='score']='3.0'");
+ assertQ(req("fl","*,score","q", "{!func}docfreq($field,$value)", "fq","id:6", "field","a_t", "value","cow"), "//float[@name='score']='3.0'");
+ assertQ(req("fl","*,score","q", "{!func}termfreq(a_t,cow)", "fq","id:6"), "//float[@name='score']='5.0'");
+ Similarity similarity = new DefaultSimilarity();
+ assertQ(req("fl","*,score","q", "{!func}idf(a_t,cow)", "fq","id:6"),
+ "//float[@name='score']='" + similarity.idf(3,6) + "'");
+ assertQ(req("fl","*,score","q", "{!func}tf(a_t,cow)", "fq","id:6"),
+ "//float[@name='score']='" + similarity.tf(5) + "'");
+ assertQ(req("fl","*,score","q", "{!func}norm(a_t)", "fq","id:2"),
+ "//float[@name='score']='" + similarity.lengthNorm("a_t",4) + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
+
// test that ord and rord are working on a global index basis, not just
// at the segment level (since Lucene 2.9 has switched to per-segment searching)
assertQ(req("fl","*,score","q", "{!func}ord(id)", "fq","id:6"), "//float[@name='score']='6.0'");