You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Yonik Seeley <ys...@gmail.com> on 2005/09/24 07:02:43 UTC
score based on field value
There has been a lot of interest in generating a score or boosting based on
the value of a particular field.
Here is my first prototype that can handle int and float field values.
I'm not particularly happy with the form of this solution yet, which is why
I'm throwing it out to dev to see if anyone has any good ideas about the
direction to take this.
-Yonik
Now hiring -- http://tinyurl.com/7m67g
package org.apache.lucene.search;
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
/**
* @author yonik
*/
public class FieldValueQuery extends Query {
// have a common base class?
public interface FloatFunc {
public float getScore(float fieldVal);
}
public interface IntFunc {
public float getScore(int fieldVal);
}
public static class FloatLine implements FloatFunc {
final float slope,intercept;
public FloatLine(float slope, float intercept) {
this.slope=slope;
this.intercept=intercept;
}
public float getScore(float fieldVal) {
return fieldVal*slope + intercept;
}
}
public static class IntLine implements IntFunc {
final float slope,intercept;
public IntLine(float slope, float intercept) {
this.slope=slope;
this.intercept=intercept;
}
public float getScore(int fieldVal) {
return fieldVal*slope + intercept;
}
}
String fname;
FloatFunc ffunc;
IntFunc ifunc;
public FieldValueQuery(String field, FloatFunc func) {
this.fname = field;
this.ffunc=func;
}
public FieldValueQuery(String field, IntFunc func) {
this.fname = field;
this.ifunc=func;
}
public Query rewrite(IndexReader reader) throws IOException {
return this;
}
protected class FunctionWeight implements Weight {
private Searcher searcher;
private float queryNorm;
private float queryWeight;
public FunctionWeight(Searcher searcher) {
this.searcher = searcher;
}
public Query getQuery() {
return FieldValueQuery.this;
}
public float getValue() {
return queryWeight;
}
public float sumOfSquaredWeights() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
public void normalize(float norm) {
this.queryNorm = norm;
queryWeight *= this.queryNorm;
}
public Scorer scorer(IndexReader reader) throws IOException {
if (ifunc != null) {
return new AllScorer(getSimilarity(searcher), reader, this) {
// hmmm, what if user needs a custom float reader?
// should the FieldCache access be encapsulated
// in the Func object instead? It kind of exposes more
// lucene internals though...
final int[] arr = FieldCache.DEFAULT.getInts(reader, fname);
public float score() throws IOException {
return ifunc.getScore(arr[doc]) * qWeight;
};
};
} else if (ffunc != null) {
return new AllScorer(getSimilarity(searcher), reader, this) {
final float[] arr = FieldCache.DEFAULT.getFloats(reader, fname);
public float score() throws IOException {
return ffunc.getScore(arr[doc]) * qWeight;
};
};
} else {
throw new RuntimeException(); // impossible
}
}
public Explanation explain(IndexReader reader, int doc) throws IOException {
return scorer(reader).explain(doc);
}
}
protected class AllScorer extends Scorer {
final IndexReader reader;
final int maxDoc;
final float qWeight;
int doc=-1;
public AllScorer(Similarity similarity, IndexReader reader, Weight w) throws
IOException {
super(similarity);
this.qWeight = w.getValue();
this.reader = reader;
this.maxDoc = reader.maxDoc();
}
// instead of matching all docs, we could also embed a query.
// the score could either ignore the subscore, or boost it.
// Containment: floatline(foo:myTerm, "myFloatField", 1.0, 0.0f)
// Boost: foo:myTerm^floatline("myFloatField",1.0,0.0f)
public boolean next() throws IOException {
for(;;) {
++doc;
if (doc>=maxDoc) {
return false;
}
if (reader.isDeleted(doc)) continue;
return true;
}
}
public int doc() {
return doc;
}
public float score() throws IOException {
return qWeight;
}
public boolean skipTo(int target) throws IOException {
doc=target-1;
return next();
}
public Explanation explain(int doc) throws IOException {
// TODO: do it right when design is settled
this.doc = doc;
float sc = score();
return new Explanation(sc,"Function " + sc/qWeight + " * queryWeight("
+qWeight + ")");
}
}
protected Weight createWeight(Searcher searcher) {
return new FieldValueQuery.FunctionWeight(searcher);
}
/** Prints a user-readable version of this query. */
public String toString(String field)
{
if (ifunc != null) return fname + ':' + ifunc;
else if (ffunc != null) return fname + ':' + ffunc;
else return fname + ':' + "unknown function";
}
/** Returns true if <code>o</code> is equal to this. */
public boolean equals(Object o) {
return this==o;
// TODO
}
/** Returns a hash code value for this object. */
public int hashCode() {
int h = Float.floatToIntBits(getBoost());
h ^= fname.hashCode();
// TODO
return h;
}
}
Re: score based on field value
Posted by Yonik Seeley <ys...@gmail.com>.
Here is another prototype.
Field value retrieval has been decoupled from function calculation, so one
will be able to use a field of any type with the same function class.
Comments?
If not, I think this may be the version I clean up, implement sources for
Integer and Ordinal fields, and submit to JIRA.
-Yonik
Now hiring -- http://tinyurl.com/7m67g
/**
* @author yonik
*/
public class LinearFloatFunction implements FunctionFactory {
protected final String field;
protected final FieldValueSourceFactory sourceFac;
protected final float slope;
protected final float intercept;
public LinearFloatFunction(String field, float slope, float intercept,
FieldValueSourceFactory sourceFac) {
this.field = field;
this.sourceFac=sourceFac==null ? FloatFieldSource.DEFAULT : sourceFac;
this.slope=slope;
this.intercept = intercept;
}
public FunctionInst createFunction(IndexReader reader) throws IOException {
final FieldValueSource source = sourceFac.getSource(field,reader);
return new FunctionInst() {
public float score(int doc) {
return source.floatVal(doc) * slope + intercept;
}
public String toString(int doc) {
return "f(" + source.floatVal(doc) + ")=" + score(doc);
}
};
}
public String toString() {
return slope + "*float(" + field + ")+" + intercept;
}
public int hashCode() {
return field.hashCode()
^ Float.floatToIntBits(slope)
^ (Float.floatToIntBits(intercept)*29)
^ sourceFac.hashCode();
}
public boolean equals(Object o) {
if (!(o instanceof LinearFloatFunction)) return false;
LinearFloatFunction other = (LinearFloatFunction)o;
return this.field.equals(other.field)
&& this.slope == other.slope
&& this.intercept == other.intercept
&& this.sourceFac.equals(other.sourceFac);
}
}
// there needs to be an object created at query evaluation time that
// is not referenced by the query itself. (you don't want a query
// object used as a key, carrying around big objects
abstract class FieldValueSource {
public float floatVal(int doc) { throw new UnsupportedOperationException();
}
public int intVal(int doc) { throw new UnsupportedOperationException(); }
public long longVal(int doc) { throw new UnsupportedOperationException(); }
public double doubleVal(int doc) { throw new
UnsupportedOperationException(); }
public String strVal(int doc) { throw new UnsupportedOperationException(); }
}
interface FieldValueSourceFactory {
public FieldValueSource getSource(String field, IndexReader reader) throws
IOException;
public boolean equals(Object o);
public int hashCode();
}
class FloatFieldSource implements FieldValueSourceFactory {
public static FloatFieldSource DEFAULT = new FloatFieldSource();
FieldCache.FloatParser parser;
public FloatFieldSource() {
}
public FloatFieldSource(FieldCache.FloatParser parser) {
this.parser = parser;
}
public FieldValueSource getSource(String field, IndexReader reader) throws
IOException {
final float[] arr = (parser==null) ?
FieldCache.DEFAULT.getFloats(reader, field) :
FieldCache.DEFAULT.getFloats(reader, field, parser);
return new FieldValueSource() {
public float floatVal(int doc) {
return arr[doc];
}
public int intVal(int doc) {
return (int)arr[doc];
}
public long longVal(int doc) {
return (long)arr[doc];
}
public double doubleVal(int doc) {
return (double)arr[doc];
}
public String strVal(int doc) {
return Float.toString(doc);
}
};
}
public boolean equals(Object o) {
if (!this.getClass().equals(o.getClass())) return false;
FloatFieldSource other = (FloatFieldSource)o;
return this.parser==null ? other.parser==null :
this.parser.getClass().equals(other.parser.getClass());
}
public int hashCode() {
return parser==null ? Float.class.hashCode() : parser.getClass().hashCode();
};
}
interface FunctionFactory {
public FunctionInst createFunction(IndexReader reader) throws IOException;
}
abstract class FunctionInst {
public abstract float score(int doc);
}
public class FunctionQuery extends Query {
FunctionFactory func;
public FunctionQuery(FunctionFactory func) {
this.func=func;
}
public Query rewrite(IndexReader reader) throws IOException {
return this;
}
protected class FunctionWeight implements Weight {
private Searcher searcher;
private float queryNorm;
private float queryWeight;
public FunctionWeight(Searcher searcher) {
this.searcher = searcher;
}
public Query getQuery() {
return FunctionQuery.this;
}
public float getValue() {
return queryWeight;
}
public float sumOfSquaredWeights() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
public void normalize(float norm) {
this.queryNorm = norm;
queryWeight *= this.queryNorm;
}
public Scorer scorer(IndexReader reader) throws IOException {
return new AllScorer(getSimilarity(searcher), reader, this);
}
public Explanation explain(IndexReader reader, int doc) throws IOException {
return scorer(reader).explain(doc);
}
}
protected class AllScorer extends Scorer {
final IndexReader reader;
final int maxDoc;
final float qWeight;
int doc=-1;
final FunctionInst ff;
public AllScorer(Similarity similarity, IndexReader reader, Weight w) throws
IOException {
super(similarity);
this.qWeight = w.getValue();
this.reader = reader;
this.maxDoc = reader.maxDoc();
ff = func.createFunction(reader);
}
// instead of matching all docs, we could also embed a query.
// the score could either ignore the subscore, or boost it.
// Containment: floatline(foo:myTerm, "myFloatField", 1.0, 0.0f)
// Boost: foo:myTerm^floatline("myFloatField",1.0,0.0f)
public boolean next() throws IOException {
for(;;) {
++doc;
if (doc>=maxDoc) {
return false;
}
if (reader.isDeleted(doc)) continue;
// todo: allow score() to throw a specific exception
// and continue on to the next document if it is thrown...
// that may be useful...
return true;
}
}
public int doc() {
return doc;
}
public float score() throws IOException {
return qWeight * ff.score(doc);
}
public boolean skipTo(int target) throws IOException {
doc=target-1;
return next();
}
public Explanation explain(int doc) throws IOException {
float sc = qWeight * ff.score(doc);
return new Explanation(sc,"Function " + ff.score(doc) + " * queryWeight("
+qWeight + ")");
// TODO: add hierarchy
}
}
protected Weight createWeight(Searcher searcher) {
return new FunctionQuery.FunctionWeight(searcher);
}
/** Prints a user-readable version of this query. */
public String toString(String field)
{
return func.toString() + (getBoost()==0 ? "" : "^"+getBoost());
}
/** Returns true if <code>o</code> is equal to this. */
public boolean equals(Object o) {
if (!(o instanceof FunctionQuery)) return false;
FunctionQuery other = (FunctionQuery)o;
return this.getBoost() == other.getBoost()
&& this.func.equals(other.func);
}
/** Returns a hash code value for this object. */
public int hashCode() {
int h = Float.floatToIntBits(getBoost());
h ^= func.hashCode();
return h;
}
}