You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/15 14:34:23 UTC
svn commit: r1147120 - in
/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search:
./ similarities/
Author: rmuir
Date: Fri Jul 15 12:34:21 2011
New Revision: 1147120
URL: http://svn.apache.org/viewvc?rev=1147120&view=rev
Log:
LUCENE-3220: implement various ranking models as similarities
Added:
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Distribution.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Lambda.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Normalization.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java (with props)
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java (with props)
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,205 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
+import org.apache.lucene.util.SmallFloat;
+
+/**
+ * BM25 Similarity.
+ */
+public class MockBM25Similarity extends Similarity {
+ // TODO: the norm table can probably be per-sim so you can configure these
+ // its also pretty nice that we don't bake the parameter into the index... you can tune it at runtime.
+ private final float k1;
+ private final float b;
+
+ public MockBM25Similarity(float k1, float b) {
+ this.k1 = k1;
+ this.b = b;
+ }
+
+ /** Sets the default values for BM25:
+ * <ul>
+ * <li>{@code k1 = 2.0},
+ * <li>{@code b = 0.75}.</li>
+ * </ul>
+ */
+ public MockBM25Similarity() {
+ this.k1 = 2.0f;
+ this.b = 0.75f;
+ }
+
+ /**
+ * Our normalization is k1 * ((1 - b) + b * numTerms / avgNumTerms)
+ * currently we put doclen into the boost byte (divided by boost) for simple quantization
+ * our decoder precomputes the full formula into the norm table
+ *
+ * this is pretty crappy for doc/field boosting, but with a static schema you can boost per-field
+ * in your sim anyway (sorta dumb to bake into the index)
+ */
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ final int numTerms = state.getLength() - state.getNumOverlap();
+ return encodeNormValue(numTerms / state.getBoost());
+ }
+
+ /** Cache of decoded bytes. */
+ private static final float[] NORM_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i);
+ }
+ }
+
+ public float decodeNormValue(byte b) {
+ return NORM_TABLE[b & 0xFF];
+ }
+
+ public byte encodeNormValue(float f) {
+ return SmallFloat.floatToByte315(f);
+ }
+
+ // weight for a term as log(1 + ((n - dfj + 0.5F)/(dfj + 0.5F)))
+ // TODO: are we summing this in the right place for phrase estimation????
+ @Override
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
+ float value = 0.0f;
+ final StringBuilder exp = new StringBuilder();
+
+ final int max = searcher.maxDoc();
+
+ for (final TermContext stat : termStats ) {
+ final int dfj = stat.docFreq();
+ value += Math.log(1 + ((max - dfj + 0.5F)/(dfj + 0.5F)));
+ exp.append(" ");
+ exp.append(dfj);
+ }
+
+ return new BM25Stats(value, queryBoost, avgDocumentLength(searcher, fieldName));
+ }
+
+ @Override
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ }
+
+ @Override
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ }
+
+ /** return avg doc length across the field (or 1 if the codec does not store sumTotalTermFreq) */
+ private float avgDocumentLength(IndexSearcher searcher, String field) throws IOException {
+ if (!searcher.reader.hasNorms(field)) {
+ return 0f;
+ } else {
+ long sumTotalTermFreq = MultiFields.getTerms(searcher.reader, field).getSumTotalTermFreq();
+ long maxdoc = searcher.reader.maxDoc();
+ return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc);
+ }
+ }
+
+ private class ExactBM25DocScorer extends ExactDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+ private final float avgdl;
+
+ ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
+ // we incorporate boost here up front... maybe we should multiply by tf instead?
+ this.weightValue = stats.idf * stats.queryBoost * stats.topLevelBoost;
+ this.avgdl = stats.avgdl;
+ this.norms = norms;
+ }
+
+ // todo: optimize
+ @Override
+ public float score(int doc, int freq) {
+ float norm = norms == null ? 0 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
+ return weightValue * (freq * (k1 + 1)) / (freq + norm);
+ }
+ }
+
+ private class SloppyBM25DocScorer extends SloppyDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+ private final float avgdl;
+
+ SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
+ // we incorporate boost here up front... maybe we should multiply by tf instead?
+ this.weightValue = stats.idf * stats.queryBoost * stats.topLevelBoost;
+ this.avgdl = stats.avgdl;
+ this.norms = norms;
+ }
+
+ // todo: optimize
+ @Override
+ public float score(int doc, float freq) {
+ float norm = norms == null ? 0 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
+ return weightValue * (freq * (k1 + 1)) / (freq + norm);
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+ }
+
+ /** Collection statistics for the BM25 model. */
+ public static class BM25Stats extends Stats {
+ /** BM25's idf */
+ private final float idf;
+ /** The average document length. */
+ private final float avgdl;
+ /** query's inner boost */
+ private final float queryBoost;
+ /** any outer query's boost */
+ private float topLevelBoost;
+
+ public BM25Stats(float idf, float queryBoost, float avgdl) {
+ this.idf = idf;
+ this.queryBoost = queryBoost;
+ this.avgdl = avgdl;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = idf * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // we don't normalize with queryNorm at all, we just capture the top-level boost
+ this.topLevelBoost = topLevelBoost;
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,32 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class MockBM25SimilarityProvider extends DefaultSimilarityProvider {
+
+ private MockBM25Similarity impl = new MockBM25Similarity();
+
+ public MockBM25Similarity get(String field) {
+ return impl;
+ }
+
+ /** Sets the {@code Similarity} returned by {@link #get}. */
+ public void set(MockBM25Similarity sim) {
+ impl = sim;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,191 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
+import org.apache.lucene.util.SmallFloat;
+
+/**
+ * Dirichlet LM Similarity.
+ * <p>
+ * This uses Terrier's modified formula "Bayesian smoothing with Dirichlet Prior" (which ensures only positive scores)
+ * from Zhai & Lafferty's A Study of Smoothing Methods for Language Models Applied to Information Retrieval.
+ * The formula has been modified in several ways:
+ * <ul>
+ * <li>Supports doc/field/query boosting
+ * <li>Uses natural log instead of base2 for simplicity.
+ * <li>The formula has been re-arranged: as part is computed in the weight, we use totalTermFrequency+1/sumOfTotalTermFrequency+1
+ * to prevent any divide by zero, other parts of the formula re-arranged for performance
+ * </ul>
+ * </p>
+ * <code>
+ * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency)))) + log(mu / (numTerms + mu))
+ * </code>
+ * <p>
+ * NOTE: to use this Similarity, use MockLMSimilarityProvider (as this formula already incorporates coord()
+ * and currently depends upon a disabled queryNorm)
+ * WARNING: doesn't work with preflex codec
+ */
+public class MockLMSimilarity extends Similarity {
+ // TODO: the norm table can probably be per-sim so you can configure this.
+ // its also pretty nice that we don't bake the parameter into the index... you can tune it at runtime.
+ private static final float mu = 2000f;
+
+ /**
+ * Our normalization is log(mu / (doclen + mu))
+ * currently we put doclen into the boost byte (divided by boost) for simple quantization
+ * our decoder precomputes the full formula into the norm table
+ */
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ final int numTerms = state.getLength() - state.getNumOverlap();
+ return encodeNormValue(numTerms / state.getBoost());
+ }
+
+ /** Cache of decoded bytes. */
+ private static final float[] NORM_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ float doclen = SmallFloat.byte315ToFloat((byte)i);
+ NORM_TABLE[i] = (float) Math.log(mu / (doclen + mu));
+ }
+ }
+
+ public float decodeNormValue(byte b) {
+ return NORM_TABLE[b & 0xFF];
+ }
+
+ public byte encodeNormValue(float f) {
+ return SmallFloat.floatToByte315(f);
+ }
+
+ // weight for a term as 1 / (mu * (totalTermFrequency / sumOfTotalTermFrequency))
+ @Override
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
+ float value = 0.0f;
+ final StringBuilder exp = new StringBuilder();
+ final long sumOfTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq();
+
+ for (final TermContext stat : termStats ) {
+ final long totalTermFrequency = stat.totalTermFreq();
+ value += 1 / (mu * ((totalTermFrequency+1L)/(double)(sumOfTotalTermFreq+1L)));
+ exp.append(" ");
+ exp.append(totalTermFrequency);
+ }
+
+ return new LMStats(value, queryBoost);
+ }
+
+ @Override
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName));
+ }
+
+ @Override
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName));
+ }
+
+ /**
+ * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency))) ) + log(mu / (numTerms + mu))
+ */
+ private class ExactMockLMDocScorer extends ExactDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+ private static final int SCORE_CACHE_SIZE = 32;
+ private float[] scoreCache = new float[SCORE_CACHE_SIZE];
+
+ ExactMockLMDocScorer(float weightValue, byte norms[]) {
+ this.weightValue = weightValue;
+ this.norms = norms;
+ for (int i = 0; i < SCORE_CACHE_SIZE; i++)
+ scoreCache[i] = (float)Math.log(1 + (i*weightValue));
+ }
+
+ @Override
+ public float score(int doc, int freq) {
+ final float raw = freq < SCORE_CACHE_SIZE // check cache
+ ? scoreCache[freq] // cache hit
+ : (float)Math.log(1 + (freq*weightValue)); // cache miss
+
+ return norms == null ? raw : raw + decodeNormValue(norms[doc]);
+ }
+ }
+
+ private class SloppyMockLMDocScorer extends SloppyDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+
+ SloppyMockLMDocScorer(float weightValue, byte norms[]) {
+ this.weightValue = weightValue;
+ this.norms = norms;
+ }
+
+ @Override
+ public float score(int doc, float freq) {
+ final float raw = (float)Math.log(1 + (freq*weightValue));
+ return norms == null ? raw : raw + decodeNormValue(norms[doc]);
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+ }
+
+ public static class LMStats extends Stats {
+ // dunno if this idf-like thing has a real name, its part1 of the formula to me.
+ private final float part1;
+ private final float queryBoost;
+ private float value;
+
+ public LMStats(float part1, float queryBoost) {
+ this.part1 = part1;
+ this.queryBoost = queryBoost;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = part1 * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // set our value here
+ this.value = part1 * queryBoost * topLevelBoost;
+ }
+
+ public float getValue() {
+ return value;
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,42 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * SimilarityProvider for {@link MockLMSimilarity}
+ * <ul>
+ * <li> disables coord, because its already factored into the formula
+ * <li> disables queryNorm, because we (currently) shove part of the formula in there as "idf"
+ * </ul>
+ * WARNING: doesn't work with preflex codec
+ */
+public class MockLMSimilarityProvider implements SimilarityProvider {
+ private static final Similarity impl = new MockLMSimilarity();
+
+ public float coord(int overlap, int maxOverlap) {
+ return 1f;
+ }
+
+ public float queryNorm(float sumOfSquaredWeights) {
+ return 1f;
+ }
+
+ public Similarity get(String field) {
+ return impl;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,36 @@
+package org.apache.lucene.search.similarities;
+
+// nocommit: license header (for all these files)
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * This class acts as the base class for the implementations of the <em>first
+ * normalization of the informative content</em> in the DFR framework. This
+ * component is also called the <em>after effect</em> and is defined by the
+ * formula <em>Inf<sub>2</sub> = 1 - Prob<sub>2</sub></em>, where
+ * <em>Prob<sub>2</sub></em> measures the <em>information gain</em>.
+ *
+ * @see DFRSimilarity
+ * @lucene.experimental
+ */
+public abstract class AfterEffect {
+ /** Returns the aftereffect score. */
+ public abstract float score(EasyStats stats, float tfn);
+
+ /** Returns an explanation for the score. */
+ public abstract Explanation explain(EasyStats stats, float tfn);
+
+ /** Implementation used when there is no aftereffect. */
+ public static final class NoAfterEffect extends AfterEffect {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ return 1f;
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats, float tfn) {
+ return new Explanation(1, "no aftereffect");
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,27 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Model of the information gain based on the ration of two Bernoulli processes.
+ * @lucene.experimental
+ */
+public class AfterEffectB extends AfterEffect {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ long F = stats.getTotalTermFreq();
+ int n = stats.getDocFreq();
+ return (F + 1) / (n * (tfn + 1));
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats, float tfn) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(score(stats, tfn));
+ result.addDetail(new Explanation(tfn, "tfn"));
+ result.addDetail(new Explanation(stats.getTotalTermFreq(), "totalTermFreq"));
+ result.addDetail(new Explanation(stats.getDocFreq(), "docFreq"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,23 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Model of the information gain based on Laplace's law of succession.
+ * @lucene.experimental
+ */
+public class AfterEffectL extends AfterEffect {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ return 1 / (tfn + 1);
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats, float tfn) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(score(stats, tfn));
+ result.addDetail(new Explanation(tfn, "tfn"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,36 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * This class acts as the base class for the specific <em>basic model</em>
+ * implementations in the DFR framework. Basic models compute the
+ * <em>informative content Inf<sub>1</sub> = -log<sub>2</sub>Prob<sub>1</sub>
+ * </em>.
+ *
+ * @see DFRSimilarity
+ * @lucene.experimental
+ */
+public abstract class BasicModel {
+ /** Returns the informative content score. */
+ public abstract float score(EasyStats stats, float tfn);
+
+ /**
+ * Returns an explanation for the score.
+ * <p>Most basic models use the number of documents and the total term
+ * frequency to compute Inf<sub>1</sub>. This method provides a generic
+ * explanation for such models. Subclasses that use other statistics must
+ * override this method.</p>
+ */
+ public Explanation explain(EasyStats stats, float tfn) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(score(stats, tfn));
+ result.addDetail(new Explanation(tfn, "tfn"));
+ result.addDetail(
+ new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments"));
+ result.addDetail(
+ new Explanation(stats.getTotalTermFreq(), "totalTermFreq"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,22 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Limiting form of the Bose-Einstein model.
+ * @lucene.experimental
+ */
+public class BasicModelBE extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ long N = stats.getNumberOfDocuments();
+ long F = stats.getTotalTermFreq();
+ return (float)(-log2((N - 1) * Math.E)
+ + f(N + F -1, N + F - tfn - 2) - f(F, F - tfn));
+ }
+
+ /** The <em>f</em> helper function defined for <em>B<sub>E</sub></em>. */
+ private final double f(long n, float m) {
+ return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n);
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,20 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Implements the approximation of the binomial model with the divergence
+ * for DFR.
+ * @lucene.experimental
+ */
+public class BasicModelD extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ long F = stats.getTotalTermFreq();
+ double phi = (double)tfn / F;
+ double nphi = 1 - phi;
+ double p = 1.0 / stats.getNumberOfDocuments();
+ double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p));
+ return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,16 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Geometric as limiting form of the Bose-Einstein model.
+ * @lucene.experimental
+ */
+public class BasicModelG extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ double lambda = stats.getTotalTermFreq() / stats.getNumberOfDocuments();
+ // -log(1 / (lambda + 1)) -> log(lambda + 1)
+ return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,17 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * An approximation of the <em>I(n<sub>e</sub>)</em> model.
+ * @lucene.experimental
+ */
+public class BasicModelIF extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ // TODO: Refactor this method to a parent class? See the other two Ix model.
+ int N = stats.getNumberOfDocuments();
+ long F = stats.getTotalTermFreq();
+ return tfn * (float)(log2((N + 1) / (F + 0.5)));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,30 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * The basic tf-idf model of randomness.
+ * @lucene.experimental
+ */
+public class BasicModelIn extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ int N = stats.getNumberOfDocuments();
+ int n = stats.getDocFreq();
+ return tfn * (float)(log2((N + 1) / (n + 0.5)));
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats, float tfn) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(score(stats, tfn));
+ result.addDetail(new Explanation(tfn, "tfn"));
+ result.addDetail(
+ new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments"));
+ result.addDetail(
+ new Explanation(stats.getDocFreq(), "docFreq"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,18 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Tf-idf model of randomness, based on a mixture of Poisson and inverse
+ * document frequency.
+ * @lucene.experimental
+ */
+public class BasicModelIne extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ int N = stats.getNumberOfDocuments();
+ long F = stats.getTotalTermFreq();
+ double ne = N * (1 - Math.pow((N - 1) / N, F));
+ return tfn * (float)(log2((N + 1) / (ne + 0.5)));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,17 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Implements the Poisson approximation for the binomial model for DFR.
+ * @lucene.experimental
+ */
+public class BasicModelP extends BasicModel {
+ @Override
+ public final float score(EasyStats stats, float tfn) {
+ float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments();
+ return (float)(tfn * log2(tfn / lambda)
+ + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E)
+ + 0.5 * log2(2 * Math.PI * tfn));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,64 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Implements the <em>divergence from randomness (DFR)</em> framework
+ * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002.
+ * Probabilistic models of information retrieval based on measuring the
+ * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002),
+ * 357-389.
+ * <p>The DFR scoring formula is composed of three separate components: the
+ * <em>basic model</em>, the <em>aftereffect</em> and an additional
+ * <em>normalization</em> component, represented by the classes
+ * {@code BasicModel}, {@code AfterEffect} and {@code Normalization},
+ * respectively. The names of these classes were chosen to match the names of
+ * their counterparts in the Terrier IR engine.</p>
+ * <p>Note that <em>qtf</em>, the multiplicity of term-occurrence in the query,
+ * is not handled by this implementation.</p>
+ *
+ * @see BasicModel
+ * @see AfterEffect
+ * @see Normalization
+ * @lucene.experimental
+ */
+public class DFRSimilarity extends EasySimilarity {
+ /** The basic model for information content. */
+ protected final BasicModel basicModel;
+ /** The first normalization of the information content. */
+ protected final AfterEffect afterEffect;
+ /** The term frequency normalization. */
+ protected final Normalization normalization;
+
+ public DFRSimilarity(BasicModel basicModel,
+ AfterEffect afterEffect,
+ Normalization normalization) {
+ this.basicModel = basicModel;
+ this.afterEffect = afterEffect != null
+ ? afterEffect : new AfterEffect.NoAfterEffect();
+ this.normalization = normalization != null
+ ? normalization : new Normalization.NoNormalization();
+ }
+
+ @Override
+ protected float score(EasyStats stats, float freq, byte norm) {
+ float tfn = normalization.tfn(stats, freq, decodeNormValue(norm));
+ return stats.getTotalBoost() *
+ basicModel.score(stats, tfn) * afterEffect.score(stats, tfn);
+ }
+
+ @Override
+ protected void explain(Explanation expl,
+ EasyStats stats, int doc, float freq, byte norm) {
+ if (stats.getTotalBoost() != 1.0f) {
+ expl.addDetail(new Explanation(stats.getTotalBoost(), "boost"));
+ }
+
+ int len = decodeNormValue(norm);
+ Explanation normExpl = normalization.explain(stats, freq, len);
+ float tfn = normExpl.getValue();
+ expl.addDetail(normExpl);
+ expl.addDetail(basicModel.explain(stats, tfn));
+ expl.addDetail(afterEffect.explain(stats, tfn));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Distribution.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Distribution.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Distribution.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Distribution.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,21 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * The probabilistic distribution used to model term occurrence
+ * in information-based models.
+ * @see IBSimilarity
+ * @lucene.experimental
+ */
+public abstract class Distribution {
+ /** Computes the score. */
+ public abstract float score(EasyStats stats, float tfn, float lambda);
+
+ /** Explains the score. Returns the name of the model only, since
+ * both {@code tfn} and {@code lambda} are explained elsewhere. */
+ public Explanation explain(EasyStats stats, float tfn, float lambda) {
+ return new Explanation(
+ score(stats, tfn, lambda), getClass().getSimpleName());
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,15 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * Log-logistic distribution.
+ * <p>Unlike for DFR, the natural logarithm is used, as
+ * it is faster to compute and the original paper does not express any
+ * preference to a specific base.</p>
+ * @lucene.experimental
+ */
+public abstract class DistributionLL extends Distribution {
+ @Override
+ public final float score(EasyStats stats, float tfn, float lambda) {
+ return (float)-Math.log(lambda / (tfn + lambda));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,17 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * The smoothed power-law (SPL) distribution for the information-based framework
+ * that is described in the original paper.
+ * <p>Unlike for DFR, the natural logarithm is used, as
+ * it is faster to compute and the original paper does not express any
+ * preference to a specific base.</p>
+ * @lucene.experimental
+ */
+public abstract class DistributionSPL extends Distribution {
+ @Override
+ public final float score(EasyStats stats, float tfn, float lambda) {
+ return (float)-Math.log(
+ (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,229 @@
+package org.apache.lucene.search.similarities;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Similarity;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.SmallFloat;
+import org.apache.lucene.util.TermContext;
+
+/**
+ * A subclass of {@code Similarity} that provides a simplified API for its
+ * descendants.
+ * @lucene.experimental
+ */
+public abstract class EasySimilarity extends Similarity {
+ /** For {@link #log2(double)}. Precomputed for efficiency reasons. */
+ private static final double LOG_2 = Math.log(2);
+
+ /**
+ * Calls {@link #fillEasyStats(EasyStats, IndexSearcher, String, TermContext...)}.
+ * Subclasses that override this method may invoke {@code fillStats} with any
+ * subclass of {@code EasyStats}.
+ */
+ @Override
+ public EasyStats computeStats(IndexSearcher searcher, String fieldName,
+ float queryBoost, TermContext... termContexts) throws IOException {
+ EasyStats stats = new EasyStats(queryBoost);
+ fillEasyStats(stats, searcher, fieldName, termContexts);
+ return stats;
+ }
+
+ /** Fills all member fields defined in {@code EasyStats} in {@code stats}. */
+ protected final void fillEasyStats(EasyStats stats, IndexSearcher searcher,
+ String fieldName, TermContext... termContexts) throws IOException {
+ IndexReader reader = searcher.getIndexReader();
+ int numberOfDocuments = reader.maxDoc();
+ long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(),
+ fieldName).getSumTotalTermFreq();
+ long numberOfFieldTokens = sumTotalTermFreq; // nocommit: these are the same stat?
+ float avgFieldLength = (float)sumTotalTermFreq / numberOfDocuments;
+
+ // nocommit This is for phrases, and it doesn't really work... have to
+ // find a method that makes sense
+ int docFreq = 0;
+ long totalTermFreq = 0;
+ for (final TermContext context : termContexts) {
+ docFreq += context.docFreq();
+ totalTermFreq += context.totalTermFreq();
+ }
+
+ stats.setNumberOfDocuments(numberOfDocuments);
+ stats.setNumberOfFieldTokens(numberOfFieldTokens);
+ stats.setAvgFieldLength(avgFieldLength);
+ stats.setDocFreq(docFreq);
+ stats.setTotalTermFreq(totalTermFreq);
+ stats.setSumTotalTermFreq(sumTotalTermFreq);
+ // nocommit uniqueTermCount? (LUCENE-3290)
+ }
+
+ /** Encodes the document length. */
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getLength());
+ }
+
+ /** Decodes a normalization factor stored in an index.
+ * @see #encodeNormValue(float)
+ */
+ // nocommit to protected?
+ // nocommit is int OK?
+ public int decodeNormValue(byte norm) {
+ // SmallFloat seems OK, because tf is smoothed anyway.
+ return (int)SmallFloat.byte315ToFloat(norm);
+ }
+
+ /** Encodes the length to a byte via SmallInt. */
+ // nocommit to protected?
+ public byte encodeNormValue(int length) {
+ // SmallFloat seems OK, because tf is smoothed anyway.
+ return SmallFloat.floatToByte315(length);
+ }
+
+ /**
+ * Scores the document {@code doc}.
+ * <p>Subclasses must apply their scoring formula in this class.</p>
+ * @param stats the corpus level statistics.
+ * @param freq the term frequency.
+ * @param norm the current document's field norm.
+ * @return the score.
+ */
+ protected abstract float score(EasyStats stats, float freq, byte norm);
+
+ /**
+ * Subclasses should implement this method to explain the score. {@code expl}
+ * already contains the score, the name of the class and the doc id, as well
+ * as the term frequency and its explanation; subclasses can add additional
+ * clauses to explain details of their scoring formulae.
+ * <p>The default implementation does nothing.</p>
+ *
+ * @param expl the explanation to extend with details.
+ * @param stats the corpus level statistics.
+ * @param doc the document id.
+ * @param freq the term frequency.
+ * @param norm the current document's field norm.
+ */
+ protected void explain(
+ Explanation expl, EasyStats stats, int doc, float freq, byte norm) {}
+
+ /**
+ * Explains the score. The implementation here provides a basic explanation
+ * in the format <em>score(name-of-similarity, doc=doc-id,
+ * freq=term-frequency), computed from:</em>, and
+ * attaches the score (computed via the {@link #score(EasyStats, float, byte)}
+ * method) and the explanation for the term frequency. Subclasses content with
+ * this format may add additional details in
+ * {@link #explain(Explanation, EasyStats, int, float, byte)}.
+ *
+ * @param stats the corpus level statistics.
+ * @param doc the document id.
+ * @param freq the term frequency and its explanation.
+ * @param norm the current document's field norm.
+ * @return the explanation.
+ */
+ protected Explanation explain(
+ EasyStats stats, int doc, Explanation freq, byte norm) {
+ Explanation result = new Explanation();
+ result.setValue(score(stats, freq.getValue(), norm));
+ result.setDescription("score(" + getClass().getSimpleName() +
+ ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:");
+ result.addDetail(freq);
+
+ explain(result, stats, doc, freq.getValue(), norm);
+
+ return result;
+ }
+
+ @Override
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName,
+ AtomicReaderContext context) throws IOException {
+ return new EasyExactDocScorer((EasyStats) stats,
+ context.reader.norms(fieldName));
+ }
+
+ @Override
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName,
+ AtomicReaderContext context) throws IOException {
+ return new EasySloppyDocScorer((EasyStats) stats,
+ context.reader.norms(fieldName));
+ }
+
+ // ----------------------------- Static methods ------------------------------
+
+ /** Returns the base two logarithm of {@code x}. */
+ public static double log2(double x) {
+ // Put this to a 'util' class?
+ return Math.log(x) / LOG_2;
+ }
+
+ // --------------------------------- Classes ---------------------------------
+
+ /** Delegates the {@link #score(int, int)} and
+ * {@link #explain(int, Explanation)} methods to
+ * {@link EasySimilarity#score(EasyStats, float, byte)} and
+ * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)},
+ * respectively.
+ */
+ private class EasyExactDocScorer extends ExactDocScorer {
+ private final EasyStats stats;
+ private final byte[] norms;
+
+ EasyExactDocScorer(EasyStats stats, byte norms[]) {
+ this.stats = stats;
+ this.norms = norms;
+ }
+
+ @Override
+ public float score(int doc, int freq) {
+ return EasySimilarity.this.score(stats, freq, norms[doc]);
+ }
+
+ @Override
+ public Explanation explain(int doc, Explanation freq) {
+ return EasySimilarity.this.explain(stats, doc, freq, norms[doc]);
+ }
+ }
+
+ /** Delegates the {@link #score(int, int)} and
+ * {@link #explain(int, Explanation)} methods to
+ * {@link EasySimilarity#score(EasyStats, float, byte)} and
+ * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)},
+ * respectively.
+ */
+ private class EasySloppyDocScorer extends SloppyDocScorer {
+ private final EasyStats stats;
+ private final byte[] norms;
+
+ EasySloppyDocScorer(EasyStats stats, byte norms[]) {
+ this.stats = stats;
+ this.norms = norms;
+ }
+
+ // todo: optimize
+ @Override
+ public float score(int doc, float freq) {
+ return EasySimilarity.this.score(stats, freq, norms[doc]);
+ }
+ @Override
+ public Explanation explain(int doc, Explanation freq) {
+ return EasySimilarity.this.explain(stats, doc, freq, norms[doc]);
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ // nocommit: do we care about exposing this?
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return 1f;
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,147 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Similarity;
+
+/**
+ * Stores all statistics commonly used ranking methods.
+ * @lucene.experimental
+ */
+public class EasyStats extends Similarity.Stats {
+ /** The number of documents. */
+ protected int numberOfDocuments; // TODO: to long?
+ /** The total number of tokens in the field. */
+ protected long numberOfFieldTokens;
+ /** The average field length. */
+ protected float avgFieldLength;
+ /** The document frequency. */
+ protected int docFreq;
+ /** The total number of occurrences of this term across all documents. */
+ // TODO: same field?
+ protected long totalTermFreq;
+ /** The total number of terms across all documents. */
+ // TODO: same field?
+ protected long sumTotalTermFreq;
+ /** The number of unique terms. */
+ // nocommit might be per-segment only
+ protected long uniqueTermCount;
+
+ // -------------------------- Boost-related stuff --------------------------
+
+ /** Query's inner boost. */
+ protected final float queryBoost;
+ /** Any outer query's boost. */
+ protected float topLevelBoost;
+ /** For most Similarities, the immediate and the top level query boosts are
+ * not handled differently. Hence, this field is just the product of the
+ * other two. */
+ protected float totalBoost;
+
+ /** Constructor. Sets the query boost. */
+ public EasyStats(float queryBoost) {
+ this.queryBoost = queryBoost;
+ }
+
+ // ------------------------- Getter/setter methods -------------------------
+
+ /** Returns the number of documents. */
+ public int getNumberOfDocuments() {
+ return numberOfDocuments;
+ }
+
+ /** Sets the number of documents. */
+ public void setNumberOfDocuments(int numberOfDocuments) {
+ this.numberOfDocuments = numberOfDocuments;
+ }
+
+ /** Returns the total number of tokens in the field. */
+ public long getNumberOfFieldTokens() {
+ return numberOfFieldTokens;
+ }
+
+ /** Sets the total number of tokens in the field. */
+ public void setNumberOfFieldTokens(long numberOfFieldTokens) {
+ this.numberOfFieldTokens = numberOfFieldTokens;
+ }
+
+ /** Returns the average field length. */
+ public float getAvgFieldLength() {
+ return avgFieldLength;
+ }
+
+ /** Sets the average field length. */
+ public void setAvgFieldLength(float avgFieldLength) {
+ this.avgFieldLength = avgFieldLength;
+ }
+
+ /** Returns the document frequency. */
+ public int getDocFreq() {
+ return docFreq;
+ }
+
+ /** Sets the document frequency. */
+ public void setDocFreq(int docFreq) {
+ this.docFreq = docFreq;
+ }
+
+ /** Returns the total number of occurrences of this term across all documents. */
+ public long getTotalTermFreq() {
+ return totalTermFreq;
+ }
+
+ /** Sets the total number of occurrences of this term across all documents. */
+ public void setTotalTermFreq(long totalTermFreq) {
+ this.totalTermFreq = totalTermFreq;
+ }
+
+ /** Returns the total number of terms across all documents. */
+ public long getSumTotalTermFreq() {
+ return sumTotalTermFreq;
+ }
+
+ /** Sets the total number of terms across all documents. */
+ public void setSumTotalTermFreq(long sumTotalTermFreq) {
+ this.sumTotalTermFreq = sumTotalTermFreq;
+ }
+
+ /** Returns the number of unique terms. */
+ public long getUniqueTermCount() {
+ return uniqueTermCount;
+ }
+
+ /** Sets the number of unique terms. */
+ public void setUniqueTermCount(long uniqueTermCount) {
+ this.uniqueTermCount = uniqueTermCount;
+ }
+
+ // -------------------------- Boost-related stuff --------------------------
+
+ /** The square of the raw normalization value.
+ * @see #rawNormalizationValue() */
+ @Override
+ public float getValueForNormalization() {
+ float rawValue = rawNormalizationValue();
+ return rawValue * rawValue;
+ }
+
+ /** Computes the raw normalization value. This basic implementation returns
+ * the query boost. Subclasses may override this method to include other
+ * factors (such as idf), or to save the value for inclusion in
+ * {@link #normalize(float, float)}, etc.
+ */
+ protected float rawNormalizationValue() {
+ return queryBoost;
+ }
+
+ /** No normalization is done. {@code topLevelBoost} is saved in the object,
+ * however. */
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ this.topLevelBoost = topLevelBoost;
+ totalBoost = queryBoost * topLevelBoost;
+ }
+
+ /** Returns the total boost. */
+ public float getTotalBoost() {
+ return totalBoost;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,69 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Provides a framework for the family of information-based models, as described
+ * in Stéphane Clinchant and Eric Gaussier. 2010. Information-based
+ * models for ad hoc IR. In Proceeding of the 33rd international ACM SIGIR
+ * conference on Research and development in information retrieval (SIGIR '10).
+ * ACM, New York, NY, USA, 234-241.
+ * <p>The retrieval function is of the form <em>RSV(q, d) = ∑
+ * -x<sup>q</sup><sub>w</sub> log Prob(X<sub>w</sub> ≥
+ * t<sup>d</sup><sub>w</sub> | λ<sub>w</sub>)</em>, where
+ * <ul>
+ * <li><em>x<sup>q</sup><sub>w</sub></em> is the query boost;</li>
+ * <li><em>X<sub>w</sub></em> is a random variable that counts the occurrences
+ * of word <em>w</em>;</li>
+ * <li><em>t<sup>d</sup><sub>w</sub></em> is the normalized term frequency;</li>
+ * <li><em>λ<sub>w</sub></em> is a parameter.</li>
+ * </ul>
+ * </p>
+ * <p>The framework described in the paper has many similarities to the DFR
+ * framework (see {@link DFRSimilarity}). It is possible that the two
+ * Similarities will be merged at one point.</p>
+ * @lucene.experimental
+ */
+public class IBSimilarity extends EasySimilarity {
+ /** The probabilistic distribution used to model term occurrence. */
+ protected final Distribution distribution;
+ /** The <em>lambda (λ<sub>w</sub>)</em> parameter. */
+ protected final Lambda lambda;
+ /** The term frequency normalization. */
+ protected final Normalization normalization;
+
+ public IBSimilarity(Class<Distribution> distributionClass,
+ Class<Lambda> lambdaClass,
+ Class<Normalization> normalizationClass)
+ throws InstantiationException, IllegalAccessException {
+ distribution = distributionClass.newInstance();
+ lambda = lambdaClass.newInstance();
+ normalization = (normalizationClass != null)
+ ? normalizationClass.newInstance()
+ : new Normalization.NoNormalization();
+ }
+
+ @Override
+ protected float score(EasyStats stats, float freq, byte norm) {
+ return stats.getTotalBoost() *
+ distribution.score(
+ stats,
+ normalization.tfn(stats, freq, decodeNormValue(norm)),
+ lambda.lambda(stats));
+ }
+
+ @Override
+ protected void explain(
+ Explanation expl, EasyStats stats, int doc, float freq, byte norm) {
+ if (stats.getTotalBoost() != 1.0f) {
+ expl.addDetail(new Explanation(stats.getTotalBoost(), "boost"));
+ }
+ int len = decodeNormValue(norm);
+ Explanation normExpl = normalization.explain(stats, freq, len);
+ Explanation lambdaExpl = lambda.explain(stats);
+ expl.addDetail(normExpl);
+ expl.addDetail(lambdaExpl);
+ expl.addDetail(distribution.explain(
+ stats, normExpl.getValue(), lambdaExpl.getValue()));
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,69 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Bayesian smoothing using Dirichlet priors. From Chengxiang Zhai and John
+ * Lafferty. 2001. A study of smoothing methods for language models applied to
+ * Ad Hoc information retrieval. In Proceedings of the 24th annual international
+ * ACM SIGIR conference on Research and development in information retrieval
+ * (SIGIR '01). ACM, New York, NY, USA, 334-342.
+ *
+ * @lucene.experimental
+ */
+public class LMDirichletSimilarity extends LMSimilarity {
+ /** The μ parameter. */
+ private final float mu;
+
+ /** @param mu the μ parameter. */
+ public LMDirichletSimilarity(CollectionModel collectionModel, float mu) {
+ super(collectionModel);
+ this.mu = mu;
+ }
+
+ /** @param mu the μ parameter. */
+ public LMDirichletSimilarity(float mu) {
+ this.mu = mu;
+ }
+
+ /** Instantiates the similarity with the default μ value of 2000. */
+ public LMDirichletSimilarity(CollectionModel collectionModel) {
+ this(collectionModel, 2000);
+ }
+
+ /** Instantiates the similarity with the default μ value of 2000. */
+ public LMDirichletSimilarity() {
+ this(2000);
+ }
+
+ @Override
+ protected float score(EasyStats stats, float freq, byte norm) {
+ return stats.getTotalBoost() *
+ (float)(Math.log(1 + freq /
+ (mu * ((LMStats)stats).getCollectionProbability())) +
+ Math.log(mu / (decodeNormValue(norm) + mu)));
+ }
+
+ @Override
+ protected void explain(Explanation expl, EasyStats stats, int doc,
+ float freq, byte norm) {
+ if (stats.getTotalBoost() != 1.0f) {
+ expl.addDetail(new Explanation(stats.getTotalBoost(), "boost"));
+ }
+
+ // nocommit: mu?
+ Explanation weightExpl = new Explanation();
+ weightExpl.setValue((float)Math.log(1 + freq /
+ (mu * ((LMStats)stats).getCollectionProbability())));
+ weightExpl.setDescription("term weight");
+ expl.addDetail(weightExpl);
+ expl.addDetail(new Explanation(
+ (float)Math.log(mu / (decodeNormValue(norm) + mu)), "document norm"));
+ super.explain(expl, stats, doc, freq, norm);
+ }
+
+ /** Returns the μ parameter. */
+ public float getMu() {
+ return mu;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,52 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.similarities.LMSimilarity.LMStats;
+
+/**
+ * Language model based on the Jelinek-Mercer smoothing method. From Chengxiang
+ * Zhai and John Lafferty. 2001. A study of smoothing methods for language
+ * models applied to Ad Hoc information retrieval. In Proceedings of the 24th
+ * annual international ACM SIGIR conference on Research and development in
+ * information retrieval (SIGIR '01). ACM, New York, NY, USA, 334-342.
+ *
+ * @lucene.experimental
+ */
+public class LMJelinekMercerSimilarity extends LMSimilarity {
+ /** The λ parameter. */
+ private final float lambda;
+
+ /** @param lambda the λ parameter. */
+ public LMJelinekMercerSimilarity(
+ CollectionModel collectionModel, float lambda) {
+ super(collectionModel);
+ this.lambda = lambda;
+ }
+
+ /** @param lambda the λ parameter. */
+ public LMJelinekMercerSimilarity(float lambda) {
+ this.lambda = lambda;
+ }
+
+ @Override
+ protected float score(EasyStats stats, float freq, byte norm) {
+ return stats.getTotalBoost() *
+ (float)Math.log(1 +
+ ((1 - lambda) * freq / decodeNormValue(norm)) /
+ (lambda * ((LMStats)stats).getCollectionProbability()));
+ }
+
+ @Override
+ protected void explain(Explanation expl, EasyStats stats, int doc,
+ float freq, byte norm) {
+ if (stats.getTotalBoost() != 1.0f) {
+ expl.addDetail(new Explanation(stats.getTotalBoost(), "boost"));
+ }
+ super.explain(expl, stats, doc, freq, norm);
+ }
+
+ /** Returns the λ parameter. */
+ public float getLambda() {
+ return lambda;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,103 @@
+package org.apache.lucene.search.similarities;
+
+import java.io.IOException;
+
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.TermContext;
+
+/**
+ * Abstract superclass for language modeling Similarities. The following inner
+ * types are introduced:
+ * <ul>
+ * <li>{@link LMStats}, which defines a new statistic, the probability that
+ * the collection language model generates the current term;</li>
+ * <li>{@link CollectionModel}, which is a strategy interface for object that
+ * compute the collection language model {@code p(w|C)};</li>
+ * <li>{@link DefaultCollectionModel}, an implementation of the former, that
+ * computes the term probability as the number of occurrences of the term in the
+ * collection, divided by the total number of tokens.</li>
+ * </ul>
+ *
+ * @lucene.experimental
+ */
+public abstract class LMSimilarity extends EasySimilarity {
+ /** The collection model. */
+ protected final CollectionModel collectionModel;
+
+ /** Creates a new instance with the specified collection language model. */
+ public LMSimilarity(CollectionModel collectionModel) {
+ this.collectionModel = collectionModel;
+ }
+
+ /** Creates a new instance with the default collection language model. */
+ public LMSimilarity() {
+ this(new DefaultCollectionModel());
+ }
+
+ /**
+ * Computes the collection probability of the current term in addition to the
+ * usual statistics.
+ */
+ @Override
+ public EasyStats computeStats(IndexSearcher searcher, String fieldName,
+ float queryBoost, TermContext... termContexts) throws IOException {
+ LMStats stats = new LMStats(queryBoost);
+ fillEasyStats(stats, searcher, fieldName, termContexts);
+ stats.setCollectionProbability(collectionModel.computeProbability(stats));
+ return stats;
+ }
+
+ @Override
+ protected void explain(Explanation expl, EasyStats stats, int doc,
+ float freq, byte norm) {
+ expl.addDetail(new Explanation(collectionModel.computeProbability(stats),
+ "collection probability"));
+ }
+
+ /** Stores the collection distribution of the current term. */
+ public static class LMStats extends EasyStats {
+ /** The probability that the current term is generated by the collection. */
+ private float collectionProbability;
+
+ public LMStats(float queryBoost) {
+ super(queryBoost);
+ }
+
+ /**
+ * Returns the probability that the current term is generated by the
+ * collection.
+ */
+ public final float getCollectionProbability() {
+ return collectionProbability;
+ }
+
+ /**
+ * Sets the probability that the current term is generated by the
+ * collection.
+ */
+ public final void setCollectionProbability(float collectionProbability) {
+ this.collectionProbability = collectionProbability;
+ }
+ }
+
+ /** A strategy for computing the collection language model. */
+ public static interface CollectionModel {
+ /**
+ * Computes the probability {@code p(w|C)} according to the language model
+ * strategy for the current term.
+ */
+ public float computeProbability(EasyStats stats);
+ }
+
+ /**
+ * Models {@code p(w|C)} as the number of occurrences of the term in the
+ * collection, divided by the total number of tokens.
+ */
+ public static class DefaultCollectionModel implements CollectionModel {
+ @Override
+ public float computeProbability(EasyStats stats) {
+ return (float)stats.getTotalTermFreq() / stats.getSumTotalTermFreq();
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Lambda.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Lambda.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Lambda.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Lambda.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,16 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * The <em>lambda (λ<sub>w</sub>)</em> parameter in information-based
+ * models.
+ * @see IBSimilarity
+ * @lucene.experimental
+ */
+public abstract class Lambda {
+ /** Computes the lambda parameter. */
+ public abstract float lambda(EasyStats stats);
+ /** Explains the lambda parameter. */
+ public abstract Explanation explain(EasyStats stats);
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,26 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Computes lambda as {@code totalTermFreq / numberOfDocuments}.
+ * @lucene.experimental
+ */
+public class LambdaDF extends Lambda {
+ @Override
+ public final float lambda(EasyStats stats) {
+ return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments();
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(lambda(stats));
+ result.addDetail(
+ new Explanation(stats.getTotalTermFreq(), "totalTermFreq"));
+ result.addDetail(
+ new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,26 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Computes lambda as {@code docFreq / numberOfDocuments}.
+ * @lucene.experimental
+ */
+public class LambdaTTF extends Lambda {
+ @Override
+ public final float lambda(EasyStats stats) {
+ return (float)stats.getDocFreq() / stats.getNumberOfDocuments();
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(lambda(stats));
+ result.addDetail(
+ new Explanation(stats.getDocFreq(), "docFreq"));
+ result.addDetail(
+ new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments"));
+ return result;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Normalization.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Normalization.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Normalization.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/Normalization.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,46 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * This class acts as the base class for the implementations of the term
+ * frequency normalization methods in the DFR framework.
+ *
+ * @see DFRSimilarity
+ * @lucene.experimental
+ */
+public abstract class Normalization {
+ /** Returns the normalized term frequency.
+ * @param len the field length. */
+ public abstract float tfn(EasyStats stats, float tf, int len);
+
+ /** Returns an explanation for the normalized term frequency.
+ * <p>The default normalization methods use the field length of the document
+ * and the average field length to compute the normalized term frequency.
+ * This method provides a generic explanation for such methods.
+ * Subclasses that use other statistics must override this method.</p>
+ */
+ public Explanation explain(EasyStats stats, float tf, int len) {
+ Explanation result = new Explanation();
+ result.setDescription(getClass().getSimpleName() + ", computed from: ");
+ result.setValue(tfn(stats, tf, len));
+ result.addDetail(new Explanation(tf, "tf"));
+ result.addDetail(
+ new Explanation(stats.getAvgFieldLength(), "avgFieldLength"));
+ result.addDetail(new Explanation(len, "len"));
+ return result;
+ }
+
+ /** Implementation used when there is no normalization. */
+ public static final class NoNormalization extends Normalization {
+ @Override
+ public final float tfn(EasyStats stats, float tf, int len) {
+ return tf;
+ }
+
+ @Override
+ public final Explanation explain(EasyStats stats, float tf, int len) {
+ return new Explanation(1, "no normalization");
+ }
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,11 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * Normalization model that assumes a uniform distribution of the term frequency.
+ */
+public class NormalizationH1 extends Normalization {
+ @Override
+ public final float tfn(EasyStats stats, float tf, int len) {
+ return tf * stats.getAvgFieldLength() / len;
+ }
+}
Added: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java?rev=1147120&view=auto
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java (added)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java Fri Jul 15 12:34:21 2011
@@ -0,0 +1,14 @@
+package org.apache.lucene.search.similarities;
+
+import static org.apache.lucene.search.similarities.EasySimilarity.log2;
+
+/**
+ * Normalization model in which the term frequency is inversely related to the
+ * length.
+ */
+public class NormalizationH2 extends Normalization {
+ @Override
+ public final float tfn(EasyStats stats, float tf, int len) {
+ return (float)(tf * log2(1 + stats.getAvgFieldLength() / len));
+ }
+}
\ No newline at end of file