You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/18 15:56:23 UTC
svn commit: r1137187 - in /lucene/dev/branches/flexscoring:
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/misc/src/java/org/apache/lucene/index/ ...
Author: rmuir
Date: Sat Jun 18 13:56:22 2011
New Revision: 1137187
URL: http://svn.apache.org/viewvc?rev=1137187&view=rev
Log:
LUCENE-2392: change computeNorm to return a byte, remove encodeNorm/decodeNorm
Modified:
lucene/dev/branches/flexscoring/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/branches/flexscoring/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNormsSum.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSetNorm.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/NormValueSource.java
lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/dev/branches/flexscoring/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Sat Jun 18 13:56:22 2011
@@ -241,8 +241,7 @@ public class InstantiatedIndexWriter imp
final FieldInvertState invertState = new FieldInvertState();
invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
- final float norm = similarityProvider.get(fieldName).computeNorm(invertState);
- normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).encodeNormValue(norm);
+ normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).computeNorm(invertState);
} else {
System.currentTimeMillis();
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sat Jun 18 13:56:22 2011
@@ -51,7 +51,6 @@ import org.apache.lucene.index.TermFreqV
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorMapper;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
@@ -1195,15 +1194,14 @@ public class MemoryIndex {
int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
float boost = info != null ? info.getBoost() : 1.0f;
FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
- float n = fieldSim.computeNorm(invertState);
- byte norm = fieldSim.encodeNormValue(n);
+ byte norm = fieldSim.computeNorm(invertState);
norms = new byte[] {norm};
// cache it for future reuse
cachedNorms = norms;
cachedFieldName = fieldName;
cachedSimilarity = sim;
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" + norm + ":" + numTokens);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + norm + ":" + numTokens);
}
return norms;
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Sat Jun 18 13:56:22 2011
@@ -149,7 +149,7 @@ public class FieldNormModifier {
for (int d = 0; d < termCounts.length; d++) {
if (delDocs == null || !delDocs.get(d)) {
invertState.setLength(termCounts[d]);
- subReader.setNorm(d, fieldName, fieldSim.encodeNormValue(fieldSim.computeNorm(invertState)));
+ subReader.setNorm(d, fieldName, fieldSim.computeNorm(invertState));
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Sat Jun 18 13:56:22 2011
@@ -106,7 +106,7 @@ public class SweetSpotSimilarity extends
* discountOverlaps is true by default or true for this
* specific field. */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTokens;
if (discountOverlaps)
@@ -114,7 +114,7 @@ public class SweetSpotSimilarity extends
else
numTokens = state.getLength();
- return state.getBoost() * computeLengthNorm(numTokens);
+ return encodeNormValue(state.getBoost() * computeLengthNorm(numTokens));
}
/**
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Sat Jun 18 13:56:22 2011
@@ -49,8 +49,8 @@ public class TestFieldNormModifier exten
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Sat Jun 18 13:56:22 2011
@@ -59,15 +59,15 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("3,10: spot i="+i,
1.0f,
- s.computeNorm(invertState),
+ ss.decodeNormValue(s.computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = s.computeNorm(invertState);
+ final byte normS = s.computeNorm(invertState);
assertEquals("3,10: 10<x : i="+i,
normD,
normS,
@@ -105,14 +105,14 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
1.0f,
- sp.get("foo").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("foo").computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("foo").computeNorm(invertState);
+ final byte normS = sp.get("foo").computeNorm(invertState);
assertEquals("f: 3,10: 10<x : i="+i,
normD,
normS,
@@ -122,21 +122,21 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
1.0f,
- sp.get("bar").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("bar").computeNorm(invertState)),
0.0f);
}
for (int i = 6; i <=9; i++) {
invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
1.0f,
- sp.get("yak").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("yak").computeNorm(invertState)),
0.0f);
}
for (int i = 13; i < 1000; i++) {
invertState.setLength(i-12);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("bar").computeNorm(invertState);
+ final byte normS = sp.get("bar").computeNorm(invertState);
assertEquals("f: 8,13: 13<x : i="+i,
normD,
normS,
@@ -144,9 +144,9 @@ public class SweetSpotSimilarityTest ext
}
for (int i = 9; i < 1000; i++) {
invertState.setLength(i-8);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("yak").computeNorm(invertState);
+ final byte normS = sp.get("yak").computeNorm(invertState);
assertEquals("f: 6,9: 9<x : i="+i,
normD,
normS,
@@ -158,8 +158,8 @@ public class SweetSpotSimilarityTest ext
for (int i = 9; i < 1000; i++) {
invertState.setLength(i);
- final float normSS = sp.get("a").computeNorm(invertState);
- final float normS = sp.get("b").computeNorm(invertState);
+ final byte normSS = sp.get("a").computeNorm(invertState);
+ final byte normS = sp.get("b").computeNorm(invertState);
assertTrue("s: i="+i+" : a="+normSS+
" < b="+normS,
normSS < normS);
Modified: lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/flexscoring/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Sat Jun 18 13:56:22 2011
@@ -54,8 +54,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
@@ -175,8 +175,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java Sat Jun 18 13:56:22 2011
@@ -72,8 +72,7 @@ final class NormsWriterPerField extends
assert norms.length == upto;
norms = ArrayUtil.grow(norms, 1+upto);
}
- final float norm = similarity.computeNorm(fieldState);
- norms[upto] = similarity.encodeNormValue(norm);
+ norms[upto] = similarity.computeNorm(fieldState);
docIDs[upto] = docState.docID;
upto++;
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java Sat Jun 18 13:56:22 2011
@@ -31,13 +31,13 @@ public class DefaultSimilarity extends T
*
* @lucene.experimental */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
}
/** Implemented as <code>sqrt(freq)</code>. */
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Sat Jun 18 13:56:22 2011
@@ -32,35 +32,17 @@ import java.io.IOException;
*/
public class MatchAllDocsQuery extends Query {
- public MatchAllDocsQuery() {
- this(null);
- }
-
- private final String normsField;
-
- /**
- * @param normsField Field used for normalization factor (document boost). Null if nothing.
- */
- public MatchAllDocsQuery(String normsField) {
- this.normsField = normsField;
- }
-
private class MatchAllScorer extends Scorer {
final float score;
- final byte[] norms;
private int doc = -1;
private final int maxDoc;
private final Bits delDocs;
- private final Similarity similarity;
-
- MatchAllScorer(IndexReader reader, Similarity similarity, Weight w, float score,
- byte[] norms) throws IOException {
+
+ MatchAllScorer(IndexReader reader, Weight w, float score) throws IOException {
super(w);
- this.similarity = similarity;
delDocs = reader.getDeletedDocs();
this.score = score;
maxDoc = reader.maxDoc();
- this.norms = norms;
}
@Override
@@ -80,11 +62,9 @@ public class MatchAllDocsQuery extends Q
return doc;
}
- // TODO: this is baked-in TF/IDF: should calling sim's docScorer.score() with a freq of 1 on the "norms field"
- // but, by default it uses a "null field", which could pose a problem for someone's SimilarityProvider !!!!
@Override
public float score() {
- return norms == null ? score : score * similarity.decodeNormValue(norms[docID()]);
+ return score;
}
@Override
@@ -95,12 +75,10 @@ public class MatchAllDocsQuery extends Q
}
private class MatchAllDocsWeight extends Weight {
- private Similarity similarity;
private float queryWeight;
private float queryNorm;
public MatchAllDocsWeight(IndexSearcher searcher) {
- this.similarity = normsField == null ? null : searcher.getSimilarityProvider().get(normsField);
}
@Override
@@ -127,8 +105,7 @@ public class MatchAllDocsQuery extends Q
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new MatchAllScorer(context.reader, similarity, this, queryWeight,
- normsField != null ? context.reader.norms(normsField) : null);
+ return new MatchAllScorer(context.reader, this, queryWeight);
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java Sat Jun 18 13:56:22 2011
@@ -40,42 +40,23 @@ public abstract class Similarity {
public static final int NO_DOC_ID_PROVIDED = -1;
- /** Decodes a normalization factor stored in an index.
- * @see #encodeNormValue(float)
- */
- public abstract float decodeNormValue(byte b);
-
/**
* Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
*
- * <p>Implementations should calculate a float value based on the field
+ * <p>Implementations should calculate a byte value based on the field
* state and then return that value.
*
* <p>Matches in longer fields are less precise, so implementations of this
* method usually return smaller values when <code>state.getLength()</code> is large,
* and larger values when <code>state.getLength()</code> is small.
*
- * <p>Note that the return values are computed under
- * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)}
- * and then stored using
- * {@link #encodeNormValue(float)}.
- * Thus they have limited precision, and documents
- * must be re-indexed if this method is altered.
- *
* @lucene.experimental
*
* @param state current processing state for this field
- * @return the calculated float norm
- */
- public abstract float computeNorm(FieldInvertState state);
-
- /** Encodes a normalization factor for storage in an index.
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
- * @see org.apache.lucene.util.SmallFloat
+ * @return the calculated byte norm
*/
- public abstract byte encodeNormValue(float f);
+ public abstract byte computeNorm(FieldInvertState state);
/** Computes the amount of a sloppy phrase match, based on an edit distance.
* This value is summed for each sloppy phrase match in a document to form
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java Sat Jun 18 13:56:22 2011
@@ -645,7 +645,6 @@ public abstract class TFIDFSimilarity ex
/** Decodes a normalization factor stored in an index.
* @see #encodeNormValue(float)
*/
- @Override
public float decodeNormValue(byte b) {
return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127
}
@@ -663,7 +662,6 @@ public abstract class TFIDFSimilarity ex
* @see org.apache.lucene.document.Field#setBoost(float)
* @see org.apache.lucene.util.SmallFloat
*/
- @Override
public byte encodeNormValue(float f) {
return SmallFloat.floatToByte315(f);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sat Jun 18 13:56:22 2011
@@ -38,8 +38,6 @@ import org.apache.lucene.search.FieldCac
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
@@ -375,7 +373,8 @@ public class TestBackwardsCompatibility
Term searchTerm = new Term("id", "6");
int delCount = reader.deleteDocuments(searchTerm);
assertEquals("wrong delete count", 1, delCount);
- reader.setNorm(searcher.search(new TermQuery(new Term("id", "22")), 10).scoreDocs[0].doc, "content", searcher.getSimilarityProvider().get("content").encodeNormValue(2.0f));
+ DefaultSimilarity sim = new DefaultSimilarity();
+ reader.setNorm(searcher.search(new TermQuery(new Term("id", "22")), 10).scoreDocs[0].doc, "content", sim.encodeNormValue(2.0f));
reader.close();
searcher.close();
@@ -421,7 +420,8 @@ public class TestBackwardsCompatibility
Term searchTerm = new Term("id", "6");
int delCount = reader.deleteDocuments(searchTerm);
assertEquals("wrong delete count", 1, delCount);
- reader.setNorm(22, "content", searcher.getSimilarityProvider().get("content").encodeNormValue(2.0f));
+ DefaultSimilarity sim = new DefaultSimilarity();
+ reader.setNorm(22, "content", sim.encodeNormValue(2.0f));
reader.close();
// make sure they "took":
@@ -483,7 +483,8 @@ public class TestBackwardsCompatibility
assertEquals("didn't delete the right number of documents", 1, delCount);
// Set one norm so we get a .s0 file:
- reader.setNorm(21, "content", conf.getSimilarityProvider().get("content").encodeNormValue(1.5f));
+ DefaultSimilarity sim = new DefaultSimilarity();
+ reader.setNorm(21, "content", sim.encodeNormValue(1.5f));
reader.close();
}
@@ -526,7 +527,7 @@ public class TestBackwardsCompatibility
assertEquals("didn't delete the right number of documents", 1, delCount);
// Set one norm so we get a .s0 file:
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
reader.setNorm(21, "content", sim.encodeNormValue(1.5f));
reader.close();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Sat Jun 18 13:56:22 2011
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -652,7 +653,8 @@ public class TestDeletionPolicy extends
writer.close();
IndexReader reader = IndexReader.open(dir, policy, false);
reader.deleteDocument(3*i+1);
- reader.setNorm(4*i+1, "content", conf.getSimilarityProvider().get("content").encodeNormValue(2.0F));
+ DefaultSimilarity sim = new DefaultSimilarity();
+ reader.setNorm(4*i+1, "content", sim.encodeNormValue(2.0F));
IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
assertEquals(16*(1+i), hits.length);
@@ -772,7 +774,8 @@ public class TestDeletionPolicy extends
writer.close();
IndexReader reader = IndexReader.open(dir, policy, false);
reader.deleteDocument(3);
- reader.setNorm(5, "content", conf.getSimilarityProvider().get("content").encodeNormValue(2.0F));
+ DefaultSimilarity sim = new DefaultSimilarity();
+ reader.setNorm(5, "content", sim.encodeNormValue(2.0F));
IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
assertEquals(16, hits.length);
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java Sat Jun 18 13:56:22 2011
@@ -70,7 +70,7 @@ public class TestIndexFileDeleter extend
Term searchTerm = new Term("id", "7");
int delCount = reader.deleteDocuments(searchTerm);
assertEquals("didn't delete the right number of documents", 1, delCount);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
// Set one norm so we get a .s0 file:
reader.setNorm(21, "content", sim.encodeNormValue(1.5f));
reader.close();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Sat Jun 18 13:56:22 2011
@@ -421,7 +421,7 @@ public class TestIndexReader extends Luc
// expected
}
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
try {
reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f));
fail("setNorm after close failed to throw IOException");
@@ -462,7 +462,7 @@ public class TestIndexReader extends Luc
// expected
}
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
try {
reader.setNorm(5, "aaa", sim.encodeNormValue(2.0f));
fail("setNorm should have hit LockObtainFailedException");
@@ -494,7 +494,7 @@ public class TestIndexReader extends Luc
// now open reader & set norm for doc 0
IndexReader reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
reader.setNorm(0, "content", sim.encodeNormValue(2.0f));
// we should be holding the write lock now:
@@ -539,7 +539,7 @@ public class TestIndexReader extends Luc
addDoc(writer, searchTerm.text());
writer.close();
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
// now open reader & set norm for doc 0 (writes to
// _0_1.s0)
reader = IndexReader.open(dir, false);
@@ -738,7 +738,7 @@ public class TestIndexReader extends Luc
}
reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
try {
reader.setNorm(1, "content", sim.encodeNormValue(2.0f));
fail("did not hit exception when calling setNorm on an invalid doc number");
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java Sat Jun 18 13:56:22 2011
@@ -273,7 +273,7 @@ public class TestIndexReaderClone extend
* @throws Exception
*/
private void performDefaultTests(IndexReader r1) throws Exception {
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
float norm1 = sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]);
IndexReader pr1Clone = (IndexReader) r1.clone();
@@ -329,7 +329,7 @@ public class TestIndexReaderClone extend
TestIndexReaderReopen.createIndex(random, dir1, false);
SegmentReader origSegmentReader = getOnlySegmentReader(IndexReader.open(dir1, false));
origSegmentReader.deleteDocument(1);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
origSegmentReader.setNorm(4, "field1", sim.encodeNormValue(0.5f));
SegmentReader clonedSegmentReader = (SegmentReader) origSegmentReader
@@ -429,7 +429,7 @@ public class TestIndexReaderClone extend
final Directory dir1 = newDirectory();
TestIndexReaderReopen.createIndex(random, dir1, false);
IndexReader orig = IndexReader.open(dir1, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
orig.setNorm(1, "field1", sim.encodeNormValue(17.0f));
final byte encoded = sim.encodeNormValue(17.0f);
assertEquals(encoded, MultiNorms.norms(orig, "field1")[1]);
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sat Jun 18 13:56:22 2011
@@ -47,9 +47,9 @@ public class TestIndexReaderCloneNorms e
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
// diable length norm
- return state.getBoost();
+ return encodeNormValue(state.getBoost());
}
};
}
@@ -217,7 +217,7 @@ public class TestIndexReaderCloneNorms e
IndexReader reader4C = (IndexReader) reader3C.clone();
SegmentReader segmentReader4C = getOnlySegmentReader(reader4C);
assertEquals(4, reader3CCNorm.bytesRef().get());
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
reader4C.setNorm(5, "field1", sim.encodeNormValue(0.33f));
// generate a cannot update exception in reader1
@@ -278,7 +278,7 @@ public class TestIndexReaderCloneNorms e
// System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
modifiedNorms.set(i, Float.valueOf(newNorm));
modifiedNorms.set(k, Float.valueOf(origNorm));
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
ir.setNorm(i, "f" + 1, sim.encodeNormValue(newNorm));
ir.setNorm(k, "f" + 1, sim.encodeNormValue(origNorm));
// System.out.println("setNorm i: "+i);
@@ -300,7 +300,7 @@ public class TestIndexReaderCloneNorms e
assertEquals("number of norms mismatches", numDocNorms, b.length);
ArrayList<Float> storedNorms = (i == 1 ? modifiedNorms : norms);
for (int j = 0; j < b.length; j++) {
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
float norm = sim.decodeNormValue(b[j]);
float norm1 = storedNorms.get(j).floatValue();
assertEquals("stored norm value of " + field + " for doc " + j + " is "
@@ -340,7 +340,7 @@ public class TestIndexReaderCloneNorms e
// return unique norm values that are unchanged by encoding/decoding
private float nextNorm(String fname) {
float norm = lastNorm + normDelta;
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
do {
float norm1 = sim.decodeNormValue(
sim.encodeNormValue(norm));
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java Sat Jun 18 13:56:22 2011
@@ -131,7 +131,7 @@ public class TestIndexReaderOnDiskFull e
dir.setMaxSizeInBytes(thisDiskFree);
dir.setRandomIOExceptionRate(rate);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
try {
if (0 == x) {
int docId = 12;
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Sat Jun 18 13:56:22 2011
@@ -606,7 +606,7 @@ public class TestIndexReaderReopen exten
IndexReader reader2 = reader1.reopen();
modifier = IndexReader.open(dir1, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
modifier.setNorm(1, "field1", sim.encodeNormValue(50f));
modifier.setNorm(1, "field2", sim.encodeNormValue(50f));
modifier.close();
@@ -701,7 +701,7 @@ public class TestIndexReaderReopen exten
protected void modifyIndex(int i) throws IOException {
if (i % 3 == 0) {
IndexReader modifier = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
modifier.setNorm(i, "field1", sim.encodeNormValue(50f));
modifier.close();
} else if (i % 3 == 1) {
@@ -982,7 +982,7 @@ public class TestIndexReaderReopen exten
}
case 1: {
IndexReader reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
reader.setNorm(4, "field1", sim.encodeNormValue(123f));
reader.setNorm(44, "field2", sim.encodeNormValue(222f));
reader.setNorm(44, "field4", sim.encodeNormValue(22f));
@@ -1006,7 +1006,7 @@ public class TestIndexReaderReopen exten
}
case 4: {
IndexReader reader = IndexReader.open(dir, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
reader.setNorm(5, "field1", sim.encodeNormValue(123f));
reader.setNorm(55, "field2", sim.encodeNormValue(222f));
reader.close();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Sat Jun 18 13:56:22 2011
@@ -116,8 +116,8 @@ public class TestMaxTermFrequency extend
}
@Override
- public float computeNorm(FieldInvertState state) {
- return (float) state.getMaxTermFrequency();
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue((float) state.getMaxTermFrequency());
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java Sat Jun 18 13:56:22 2011
@@ -46,9 +46,9 @@ public class TestNorms extends LuceneTes
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
// diable length norm
- return state.getBoost();
+ return encodeNormValue(state.getBoost());
}
};
}
@@ -177,7 +177,7 @@ public class TestNorms extends LuceneTes
//System.out.println(" and: for "+k+" from "+newNorm+" to "+origNorm);
modifiedNorms.set(i, Float.valueOf(newNorm));
modifiedNorms.set(k, Float.valueOf(origNorm));
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
ir.setNorm(i, "f"+1, sim.encodeNormValue(newNorm));
ir.setNorm(k, "f"+1, sim.encodeNormValue(origNorm));
}
@@ -192,8 +192,9 @@ public class TestNorms extends LuceneTes
byte b[] = MultiNorms.norms(ir, field);
assertEquals("number of norms mismatches",numDocNorms,b.length);
ArrayList<Float> storedNorms = (i==1 ? modifiedNorms : norms);
+ DefaultSimilarity sim = (DefaultSimilarity) similarityProviderOne.get(field);
for (int j = 0; j < b.length; j++) {
- float norm = similarityProviderOne.get(field).decodeNormValue(b[j]);
+ float norm = sim.decodeNormValue(b[j]);
float norm1 = storedNorms.get(j).floatValue();
assertEquals("stored norm value of "+field+" for doc "+j+" is "+norm+" - a mismatch!", norm, norm1, 0.000001);
}
@@ -229,7 +230,7 @@ public class TestNorms extends LuceneTes
// return unique norm values that are unchanged by encoding/decoding
private float nextNorm(String fname) {
float norm = lastNorm + normDelta;
- Similarity similarity = similarityProviderOne.get(fname);
+ DefaultSimilarity similarity = (DefaultSimilarity) similarityProviderOne.get(fname);
do {
float norm1 = similarity.decodeNormValue(similarity.encodeNormValue(norm));
if (norm1 > lastNorm) {
@@ -259,8 +260,8 @@ public class TestNorms extends LuceneTes
}
@Override
- public float computeNorm(FieldInvertState state) {
- return (float) state.getLength();
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue((float) state.getLength());
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNormsSum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNormsSum.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNormsSum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestNormsSum.java Sat Jun 18 13:56:22 2011
@@ -117,8 +117,8 @@ public class TestNormsSum extends Lucene
}
@Override
- public float computeNorm(FieldInvertState state) {
- return (float) state.getMaxTermFrequency();
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue((float) state.getMaxTermFrequency());
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Sat Jun 18 13:56:22 2011
@@ -40,7 +40,7 @@ public class TestOmitTf extends LuceneTe
public Similarity get(String field) {
return new TFIDFSimilarity() {
- @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java Sat Jun 18 13:56:22 2011
@@ -149,7 +149,7 @@ public class TestParallelReader extends
assertTrue(pr.isCurrent());
IndexReader modifier = IndexReader.open(dir1, false);
- Similarity sim = new DefaultSimilarity();
+ DefaultSimilarity sim = new DefaultSimilarity();
modifier.setNorm(0, "f1", sim.encodeNormValue(100f));
modifier.close();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Sat Jun 18 13:56:22 2011
@@ -262,17 +262,7 @@ final class JustCompileSearch {
}
@Override
- public float decodeNormValue(byte b) {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
- public byte encodeNormValue(float f) {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java Sat Jun 18 13:56:22 2011
@@ -42,9 +42,9 @@ public class MockBM25Similarity extends
* in your sim anyway (sorta dumb to bake into the index)
*/
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTerms = state.getLength() - state.getNumOverlap();
- return numTerms / state.getBoost();
+ return encodeNormValue(numTerms / state.getBoost());
}
/** Cache of decoded bytes. */
@@ -56,12 +56,10 @@ public class MockBM25Similarity extends
}
}
- @Override
public float decodeNormValue(byte b) {
return NORM_TABLE[b & 0xFF];
}
- @Override
public byte encodeNormValue(float f) {
return SmallFloat.floatToByte315(f);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java Sat Jun 18 13:56:22 2011
@@ -57,9 +57,9 @@ public class MockLMSimilarity extends Si
* our decoder precomputes the full formula into the norm table
*/
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTerms = state.getLength() - state.getNumOverlap();
- return numTerms / state.getBoost();
+ return encodeNormValue(numTerms / state.getBoost());
}
/** Cache of decoded bytes. */
@@ -72,12 +72,10 @@ public class MockLMSimilarity extends Si
}
}
- @Override
public float decodeNormValue(byte b) {
return NORM_TABLE[b & 0xFF];
}
- @Override
public byte encodeNormValue(float f) {
return SmallFloat.floatToByte315(f);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Sat Jun 18 13:56:22 2011
@@ -62,9 +62,9 @@ public class TestDisjunctionMaxQuery ext
}
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
// Disable length norm
- return state.getBoost();
+ return encodeNormValue(state.getBoost());
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java Sat Jun 18 13:56:22 2011
@@ -50,34 +50,12 @@ public class TestMatchAllDocsQuery exten
IndexSearcher is = newSearcher(ir);
ScoreDoc[] hits;
- // assert with norms scoring turned off
-
hits = is.search(new MatchAllDocsQuery(), null, 1000).scoreDocs;
assertEquals(3, hits.length);
assertEquals("one", is.doc(hits[0].doc).get("key"));
assertEquals("two", is.doc(hits[1].doc).get("key"));
assertEquals("three four", is.doc(hits[2].doc).get("key"));
- // assert with norms scoring turned on
-
- MatchAllDocsQuery normsQuery = new MatchAllDocsQuery("key");
- hits = is.search(normsQuery, null, 1000).scoreDocs;
- assertEquals(3, hits.length);
-
- assertEquals("three four", is.doc(hits[0].doc).get("key"));
- assertEquals("two", is.doc(hits[1].doc).get("key"));
- assertEquals("one", is.doc(hits[2].doc).get("key"));
-
- // change norm & retest
- is.getIndexReader().setNorm(0, "key", is.getSimilarityProvider().get("key").encodeNormValue(400f));
- normsQuery = new MatchAllDocsQuery("key");
- hits = is.search(normsQuery, null, 1000).scoreDocs;
- assertEquals(3, hits.length);
-
- assertEquals("one", is.doc(hits[0].doc).get("key"));
- assertEquals("three four", is.doc(hits[1].doc).get("key"));
- assertEquals("two", is.doc(hits[2].doc).get("key"));
-
// some artificial queries to trigger the use of skipTo():
BooleanQuery bq = new BooleanQuery();
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSetNorm.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSetNorm.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSetNorm.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSetNorm.java Sat Jun 18 13:56:22 2011
@@ -50,7 +50,7 @@ public class TestSetNorm extends LuceneT
// reset the boost of each instance of this document
IndexReader reader = IndexReader.open(store, false);
- Similarity similarity = new DefaultSimilarity();
+ DefaultSimilarity similarity = new DefaultSimilarity();
reader.setNorm(0, "field", similarity.encodeNormValue(1.0f));
reader.setNorm(1, "field", similarity.encodeNormValue(2.0f));
reader.setNorm(2, "field", similarity.encodeNormValue(4.0f));
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarity.java Sat Jun 18 13:56:22 2011
@@ -43,7 +43,7 @@ public class TestSimilarity extends Luce
public float coord(int overlap, int maxOverlap) { return 1.0f; }
public Similarity get(String field) {
return new DefaultSimilarity() {
- @Override public float computeNorm(FieldInvertState state) { return state.getBoost(); }
+ @Override public byte computeNorm(FieldInvertState state) { return encodeNormValue(state.getBoost()); }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java Sat Jun 18 13:56:22 2011
@@ -107,8 +107,8 @@ public class TestSimilarityProvider exte
private class Sim1 extends TFIDFSimilarity {
@Override
- public float computeNorm(FieldInvertState state) {
- return 1f;
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(1f);
}
@Override
@@ -129,8 +129,8 @@ public class TestSimilarityProvider exte
private class Sim2 extends TFIDFSimilarity {
@Override
- public float computeNorm(FieldInvertState state) {
- return 10f;
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(10f);
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sat Jun 18 13:56:22 2011
@@ -324,8 +324,8 @@ public class TestPayloadNearQuery extend
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost();
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost());
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Sat Jun 18 13:56:22 2011
@@ -315,8 +315,8 @@ public class TestPayloadTermQuery extend
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost();
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost());
}
@Override
Modified: lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/NormValueSource.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/NormValueSource.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/java/org/apache/solr/search/function/NormValueSource.java Sat Jun 18 13:56:22 2011
@@ -20,6 +20,8 @@ package org.apache.solr.search.function;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.TFIDFSimilarity;
+
import java.io.IOException;
import java.util.Map;
@@ -46,7 +48,11 @@ public class NormValueSource extends Val
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- final Similarity similarity = searcher.getSimilarityProvider().get(field);
+ Similarity sim = searcher.getSimilarityProvider().get(field);
+ if (!(sim instanceof TFIDFSimilarity)) {
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
+ }
+ final TFIDFSimilarity similarity = (TFIDFSimilarity) sim;
final byte[] norms = readerContext.reader.norms(field);
if (norms == null) {
return new ConstDoubleDocValues(0.0, this);
Modified: lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1137187&r1=1137186&r2=1137187&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/branches/flexscoring/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java Sat Jun 18 13:56:22 2011
@@ -298,7 +298,7 @@ public class TestFunctionQuery extends S
state.setBoost(1.0f);
state.setLength(4);
assertQ(req("fl","*,score","q", "{!func}norm(a_t)", "fq","id:2"),
- "//float[@name='score']='" + similarity.computeNorm(state) + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
+ "//float[@name='score']='" + similarity.decodeNormValue(similarity.computeNorm(state)) + "'"); // sqrt(4)==2 and is exactly representable when quantized to a byte
// test that ord and rord are working on a global index basis, not just
// at the segment level (since Lucene 2.9 has switched to per-segment searching)