You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/01/08 19:56:32 UTC
svn commit: r1056762 - in /lucene/dev/branches/branch_3x: lucene/
lucene/backwards/src/test/org/apache/lucene/index/
lucene/backwards/src/test/org/apache/lucene/search/
lucene/backwards/src/test/org/apache/lucene/search/payloads/
lucene/contrib/instant...
Author: mikemccand
Date: Sat Jan 8 18:56:31 2011
New Revision: 1056762
URL: http://svn.apache.org/viewvc?rev=1056762&view=rev
Log:
LUCENE-2854: deprecate SimilarityDelegator; force cutover of Similarity.lengthNorm -> Similarity.computeNorm
Removed:
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/search/SolrSimilarity.java
Modified:
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestMultiSearcher.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Sat Jan 8 18:56:31 2011
@@ -81,6 +81,11 @@ Changes in backwards compatibility polic
Searchable are collapsed into IndexSearcher; contrib/remote and
MultiSearcher have been removed. (Mike McCandless)
+* LUCENE-2854: Deprecated SimilarityDelegator and
+ Similarity.lengthNorm; the latter is now final, forcing any custom
+ Similarity impls to cutover to the more general computeNorm (Robert
+ Muir, Mike McCandless)
+
Changes in runtime behavior
* LUCENE-1923: Made IndexReader.toString() produce something
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sat Jan 8 18:56:31 2011
@@ -42,8 +42,8 @@ public class TestIndexReaderCloneNorms e
private class SimilarityOne extends DefaultSimilarity {
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java Sat Jan 8 18:56:31 2011
@@ -30,7 +30,6 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
-import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -42,8 +41,8 @@ public class TestNorms extends LuceneTes
private class SimilarityOne extends DefaultSimilarity {
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java Sat Jan 8 18:56:31 2011
@@ -41,27 +41,6 @@ import org.apache.lucene.search.Explanat
public class TestOmitTf extends LuceneTestCase {
- public static class SimpleSimilarity extends Similarity {
- @Override public float lengthNorm(String field, int numTerms) { return 1.0f; }
- @Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
- @Override public float tf(float freq) { return freq; }
- @Override public float sloppyFreq(int distance) { return 2.0f; }
- @Override public float idf(int docFreq, int numDocs) { return 1.0f; }
- @Override public float coord(int overlap, int maxOverlap) { return 1.0f; }
- @Override public IDFExplanation idfExplain(Collection<Term> terms, Searcher searcher) throws IOException {
- return new IDFExplanation() {
- @Override
- public float getIdf() {
- return 1.0f;
- }
- @Override
- public String explain() {
- return "Inexplicable";
- }
- };
- }
- }
-
// Tests whether the DocumentWriter correctly enable the
// omitTermFreqAndPositions bit in the FieldInfo
public void testOmitTermFreqAndPositions() throws Exception {
@@ -241,150 +220,6 @@ public class TestOmitTf extends LuceneTe
ram.close();
}
- // Test scores with one field with Term Freqs and one without, otherwise with equal content
- public void testBasic() throws Exception {
- Directory dir = new MockRAMDirectory();
- Analyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT);
- IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
- writer.setMergeFactor(2);
- writer.setMaxBufferedDocs(2);
- writer.setSimilarity(new SimpleSimilarity());
-
-
- StringBuilder sb = new StringBuilder(265);
- String term = "term";
- for(int i = 0; i<30; i++){
- Document d = new Document();
- sb.append(term).append(" ");
- String content = sb.toString();
- Field noTf = new Field("noTf", content + (i%2==0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED);
- noTf.setOmitTermFreqAndPositions(true);
- d.add(noTf);
-
- Field tf = new Field("tf", content + (i%2==0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED);
- d.add(tf);
-
- writer.addDocument(d);
- //System.out.println(d);
- }
-
- writer.optimize();
- // flush
- writer.close();
- _TestUtil.checkIndex(dir);
-
- /*
- * Verify the index
- */
- Searcher searcher = new IndexSearcher(dir, true);
- searcher.setSimilarity(new SimpleSimilarity());
-
- Term a = new Term("noTf", term);
- Term b = new Term("tf", term);
- Term c = new Term("noTf", "notf");
- Term d = new Term("tf", "tf");
- TermQuery q1 = new TermQuery(a);
- TermQuery q2 = new TermQuery(b);
- TermQuery q3 = new TermQuery(c);
- TermQuery q4 = new TermQuery(d);
-
-
- searcher.search(q1,
- new CountingHitCollector() {
- private Scorer scorer;
- @Override
- public final void setScorer(Scorer scorer) {
- this.scorer = scorer;
- }
- @Override
- public final void collect(int doc) throws IOException {
- //System.out.println("Q1: Doc=" + doc + " score=" + score);
- float score = scorer.score();
- assertTrue(score==1.0f);
- super.collect(doc);
- }
- });
- //System.out.println(CountingHitCollector.getCount());
-
-
- searcher.search(q2,
- new CountingHitCollector() {
- private Scorer scorer;
- @Override
- public final void setScorer(Scorer scorer) {
- this.scorer = scorer;
- }
- @Override
- public final void collect(int doc) throws IOException {
- //System.out.println("Q2: Doc=" + doc + " score=" + score);
- float score = scorer.score();
- assertTrue(score==1.0f+doc);
- super.collect(doc);
- }
- });
- //System.out.println(CountingHitCollector.getCount());
-
-
-
-
-
- searcher.search(q3,
- new CountingHitCollector() {
- private Scorer scorer;
- @Override
- public final void setScorer(Scorer scorer) {
- this.scorer = scorer;
- }
- @Override
- public final void collect(int doc) throws IOException {
- //System.out.println("Q1: Doc=" + doc + " score=" + score);
- float score = scorer.score();
- assertTrue(score==1.0f);
- assertFalse(doc%2==0);
- super.collect(doc);
- }
- });
- //System.out.println(CountingHitCollector.getCount());
-
-
- searcher.search(q4,
- new CountingHitCollector() {
- private Scorer scorer;
- @Override
- public final void setScorer(Scorer scorer) {
- this.scorer = scorer;
- }
- @Override
- public final void collect(int doc) throws IOException {
- float score = scorer.score();
- //System.out.println("Q1: Doc=" + doc + " score=" + score);
- assertTrue(score==1.0f);
- assertTrue(doc%2==0);
- super.collect(doc);
- }
- });
- //System.out.println(CountingHitCollector.getCount());
-
-
-
- BooleanQuery bq = new BooleanQuery();
- bq.add(q1,Occur.MUST);
- bq.add(q4,Occur.MUST);
-
- searcher.search(bq,
- new CountingHitCollector() {
- @Override
- public final void collect(int doc) throws IOException {
- //System.out.println("BQ: Doc=" + doc + " score=" + score);
- super.collect(doc);
- }
- });
- assertTrue(15 == CountingHitCollector.getCount());
-
- searcher.close();
- dir.close();
- }
-
public static class CountingHitCollector extends Collector {
static int count=0;
static int sum=0;
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Sat Jan 8 18:56:31 2011
@@ -22,6 +22,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
@@ -59,8 +60,8 @@ public class TestDisjunctionMaxQuery ext
else return 0.0f;
}
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1.0f;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
@Override
public float idf(int docFreq, int numDocs) {
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestMultiSearcher.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestMultiSearcher.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestMultiSearcher.java Sat Jan 8 18:56:31 2011
@@ -23,6 +23,8 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SetBasedFieldSelector;
+
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
@@ -372,7 +374,7 @@ public class TestMultiSearcher extends L
@Override
public float coord(int overlap, int maxOverlap) { return 1.0f; }
@Override
- public float lengthNorm(String fieldName, int numTokens) { return 1.0f; }
+ public float computeNorm(String fieldName, FieldInvertState state) { return state.getBoost(); }
@Override
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestTermVectors.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestTermVectors.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestTermVectors.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/TestTermVectors.java Sat Jan 8 18:56:31 2011
@@ -262,7 +262,7 @@ public class TestTermVectors extends Luc
float idf = sim.idf(knownSearcher.docFreq(term), knownSearcher.maxDoc());
//float qNorm = sim.queryNorm()
//This is fine since we don't have stop words
- float lNorm = sim.lengthNorm("field", vector.getTerms().length);
+ //float lNorm = sim.lengthNorm("field", vector.getTerms().length);
//float coord = sim.coord()
//System.out.println("TF: " + tf + " IDF: " + idf + " LenNorm: " + lNorm);
assertTrue(vector != null);
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sat Jan 8 18:56:31 2011
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.Term;
@@ -235,8 +236,8 @@ public class TestPayloadNearQuery extend
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- @Override public float lengthNorm(String fieldName, int numTerms) {
- return 1.0f;
+ @Override public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
@Override public float queryNorm(float sumOfSquaredWeights) {
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Sat Jan 8 18:56:31 2011
@@ -35,6 +35,7 @@ import org.apache.lucene.analysis.LowerC
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.index.Payload;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.RAMDirectory;
@@ -292,8 +293,8 @@ public class TestPayloadTermQuery extend
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Sat Jan 8 18:56:31 2011
@@ -38,6 +38,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
@@ -238,9 +239,10 @@ public class InstantiatedIndexWriter imp
termsInDocument += eFieldTermDocInfoFactoriesByTermText.getValue().size();
if (eFieldTermDocInfoFactoriesByTermText.getKey().indexed && !eFieldTermDocInfoFactoriesByTermText.getKey().omitNorms) {
- float norm = eFieldTermDocInfoFactoriesByTermText.getKey().boost;
- norm *= document.getDocument().getBoost();
- norm *= similarity.lengthNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
+ final FieldInvertState invertState = new FieldInvertState();
+ invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
+ invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
+ final float norm = similarity.computeNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, invertState);
normsByFieldNameAndDocumentNumber.get(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName)[document.getDocumentNumber()] = similarity.encodeNormValue(norm);
} else {
System.currentTimeMillis();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Sat Jan 8 18:56:31 2011
@@ -28,7 +28,7 @@ import org.apache.lucene.util.StringHelp
/**
* Given a directory and a list of fields, updates the fieldNorms in place for every document.
*
- * If Similarity class is specified, uses its lengthNorm method to set norms.
+ * If Similarity class is specified, uses its computeNorm method to set norms.
* If -n command line argument is used, removed field norms, as if
* {@link org.apache.lucene.document.Field.Index}.NO_NORMS was used.
*
@@ -138,12 +138,16 @@ public class FieldNormModifier {
try {
reader = IndexReader.open(dir, false);
+ final FieldInvertState invertState = new FieldInvertState();
+ invertState.setBoost(1.0f);
for (int d = 0; d < termCounts.length; d++) {
if (! reader.isDeleted(d)) {
if (sim == null)
reader.setNorm(d, fieldName, Similarity.encodeNorm(1.0f));
- else
- reader.setNorm(d, fieldName, sim.encodeNormValue(sim.lengthNorm(fieldName, termCounts[d])));
+ else {
+ invertState.setLength(termCounts[d]);
+ reader.setNorm(d, fieldName, sim.encodeNormValue(sim.computeNorm(fieldName, invertState)));
+ }
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Sat Jan 8 18:56:31 2011
@@ -146,7 +146,7 @@ public class SweetSpotSimilarity extends
else
numTokens = state.getLength();
- return state.getBoost() * lengthNorm(fieldName, numTokens);
+ return state.getBoost() * computeLengthNorm(fieldName, numTokens);
}
/**
@@ -167,8 +167,7 @@ public class SweetSpotSimilarity extends
*
* @see #setLengthNormFactors
*/
- @Override
- public float lengthNorm(String fieldName, int numTerms) {
+ public float computeLengthNorm(String fieldName, int numTerms) {
int l = ln_min;
int h = ln_max;
float s = ln_steep;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Sat Jan 8 18:56:31 2011
@@ -43,8 +43,8 @@ public class TestFieldNormModifier exten
/** inverts the normal notion of lengthNorm */
public static Similarity s = new DefaultSimilarity() {
@Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
}
};
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Sat Jan 8 18:56:31 2011
@@ -21,13 +21,14 @@ package org.apache.lucene.misc;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.index.FieldInvertState;
/**
* Test of the SweetSpotSimilarity
*/
public class SweetSpotSimilarityTest extends LuceneTestCase {
- public void testSweetSpotLengthNorm() {
+ public void testSweetSpotComputeNorm() {
SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.setLengthNormFactors(1,1,0.5f);
@@ -37,10 +38,13 @@ public class SweetSpotSimilarityTest ext
// base case, should degrade
-
+ final FieldInvertState invertState = new FieldInvertState();
+ invertState.setBoost(1.0f);
for (int i = 1; i < 1000; i++) {
+ invertState.setLength(i);
assertEquals("base case: i="+i,
- d.lengthNorm("foo",i), s.lengthNorm("foo",i),
+ d.computeNorm("foo", invertState),
+ s.computeNorm("foo", invertState),
0.0f);
}
@@ -49,14 +53,21 @@ public class SweetSpotSimilarityTest ext
ss.setLengthNormFactors(3,10,0.5f);
for (int i = 3; i <=10; i++) {
+ invertState.setLength(i);
assertEquals("3,10: spot i="+i,
- 1.0f, s.lengthNorm("foo",i),
+ 1.0f,
+ s.computeNorm("foo", invertState),
0.0f);
}
for (int i = 10; i < 1000; i++) {
+ invertState.setLength(i-9);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("foo", invertState);
assertEquals("3,10: 10<x : i="+i,
- d.lengthNorm("foo",i-9), s.lengthNorm("foo",i),
+ normD,
+ normS,
0.0f);
}
@@ -68,33 +79,54 @@ public class SweetSpotSimilarityTest ext
for (int i = 3; i <=10; i++) {
+ invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
- 1.0f, s.lengthNorm("foo",i),
+ 1.0f,
+ s.computeNorm("foo", invertState),
0.0f);
}
for (int i = 10; i < 1000; i++) {
+ invertState.setLength(i-9);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("foo", invertState);
assertEquals("f: 3,10: 10<x : i="+i,
- d.lengthNorm("foo",i-9), s.lengthNorm("foo",i),
+ normD,
+ normS,
0.0f);
}
for (int i = 8; i <=13; i++) {
+ invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
- 1.0f, s.lengthNorm("bar",i),
+ 1.0f,
+ s.computeNorm("bar", invertState),
0.0f);
}
for (int i = 6; i <=9; i++) {
+ invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
- 1.0f, s.lengthNorm("yak",i),
+ 1.0f,
+ s.computeNorm("yak", invertState),
0.0f);
}
for (int i = 13; i < 1000; i++) {
+ invertState.setLength(i-12);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("bar", invertState);
assertEquals("f: 8,13: 13<x : i="+i,
- d.lengthNorm("foo",i-12), s.lengthNorm("bar",i),
+ normD,
+ normS,
0.0f);
}
for (int i = 9; i < 1000; i++) {
+ invertState.setLength(i-8);
+ final float normD = d.computeNorm("foo", invertState);
+ invertState.setLength(i);
+ final float normS = s.computeNorm("yak", invertState);
assertEquals("f: 6,9: 9<x : i="+i,
- d.lengthNorm("foo",i-8), s.lengthNorm("yak",i),
+ normD,
+ normS,
0.0f);
}
@@ -105,9 +137,12 @@ public class SweetSpotSimilarityTest ext
ss.setLengthNormFactors("b",5,8,0.1f, false);
for (int i = 9; i < 1000; i++) {
- assertTrue("s: i="+i+" : a="+ss.lengthNorm("a",i)+
- " < b="+ss.lengthNorm("b",i),
- ss.lengthNorm("a",i) < s.lengthNorm("b",i));
+ invertState.setLength(i);
+ final float normSS = ss.computeNorm("a", invertState);
+ final float normS = s.computeNorm("b", invertState);
+ assertTrue("s: i="+i+" : a="+normSS+
+ " < b="+normS,
+ normSS < normS);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Sat Jan 8 18:56:31 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.FieldNormModifier;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
@@ -45,11 +46,11 @@ public class TestLengthNormModifier exte
/** inverts the normal notion of lengthNorm */
public static Similarity s = new DefaultSimilarity() {
- @Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
- }
- };
+ @Override
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ }
+ };
@Override
public void setUp() throws Exception {
@@ -161,11 +162,11 @@ public class TestLengthNormModifier exte
// override the norms to be inverted
Similarity s = new DefaultSimilarity() {
- @Override
- public float lengthNorm(String fieldName, int numTokens) {
- return numTokens;
- }
- };
+ @Override
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ }
+ };
FieldNormModifier fnm = new FieldNormModifier(store, s);
fnm.reSetNorms("field");
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInvertState.java Sat Jan 8 18:56:31 2011
@@ -73,6 +73,10 @@ public final class FieldInvertState {
return length;
}
+ public void setLength(int length) {
+ this.length = length;
+ }
+
/**
* Get the number of terms with <code>positionIncrement == 0</code>.
* @return the numOverlap
@@ -81,6 +85,10 @@ public final class FieldInvertState {
return numOverlap;
}
+ public void setNumOverlap(int numOverlap) {
+ this.numOverlap = numOverlap;
+ }
+
/**
* Get end offset of the last processed term.
* @return the offset
@@ -99,6 +107,10 @@ public final class FieldInvertState {
return boost;
}
+ public void setBoost(float boost) {
+ this.boost = boost;
+ }
+
public AttributeSource getAttributeSource() {
return attributeSource;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Sat Jan 8 18:56:31 2011
@@ -62,10 +62,12 @@ public class BooleanQuery extends Query
}
private ArrayList<BooleanClause> clauses = new ArrayList<BooleanClause>();
- private boolean disableCoord;
+ private final boolean disableCoord;
/** Constructs an empty boolean query. */
- public BooleanQuery() {}
+ public BooleanQuery() {
+ disableCoord = false;
+ }
/** Constructs an empty boolean query.
*
@@ -86,22 +88,6 @@ public class BooleanQuery extends Query
*/
public boolean isCoordDisabled() { return disableCoord; }
- // Implement coord disabling.
- // Inherit javadoc.
- @Override
- public Similarity getSimilarity(Searcher searcher) {
- Similarity result = super.getSimilarity(searcher);
- if (disableCoord) { // disable coord as requested
- result = new SimilarityDelegator(result) {
- @Override
- public float coord(int overlap, int maxOverlap) {
- return 1.0f;
- }
- };
- }
- return result;
- }
-
/**
* Specifies a minimum number of the optional BooleanClauses
* which must be satisfied.
@@ -178,10 +164,12 @@ public class BooleanQuery extends Query
protected Similarity similarity;
protected ArrayList<Weight> weights;
protected int maxCoord; // num optional + num required
+ private final boolean disableCoord;
- public BooleanWeight(Searcher searcher)
+ public BooleanWeight(Searcher searcher, boolean disableCoord)
throws IOException {
this.similarity = getSimilarity(searcher);
+ this.disableCoord = disableCoord;
weights = new ArrayList<Weight>(clauses.size());
for (int i = 0 ; i < clauses.size(); i++) {
BooleanClause c = clauses.get(i);
@@ -284,10 +272,10 @@ public class BooleanQuery extends Query
sumExpl.setMatch(0 < coord ? Boolean.TRUE : Boolean.FALSE);
sumExpl.setValue(sum);
- float coordFactor = similarity.coord(coord, maxCoord);
- if (coordFactor == 1.0f) // coord is no-op
+ final float coordFactor = disableCoord ? 1.0f : similarity.coord(coord, maxCoord);
+ if (coordFactor == 1.0f) {
return sumExpl; // eliminate wrapper
- else {
+ } else {
ComplexExplanation result = new ComplexExplanation(sumExpl.isMatch(),
sum*coordFactor,
"product of:");
@@ -323,7 +311,7 @@ public class BooleanQuery extends Query
// Check if we can return a BooleanScorer
if (!scoreDocsInOrder && topScorer && required.size() == 0 && prohibited.size() < 32) {
- return new BooleanScorer(this, similarity, minNrShouldMatch, optional, prohibited, maxCoord);
+ return new BooleanScorer(this, disableCoord, similarity, minNrShouldMatch, optional, prohibited, maxCoord);
}
if (required.size() == 0 && optional.size() == 0) {
@@ -337,7 +325,7 @@ public class BooleanQuery extends Query
}
// Return a BooleanScorer2
- return new BooleanScorer2(this, similarity, minNrShouldMatch, required, prohibited, optional, maxCoord);
+ return new BooleanScorer2(this, disableCoord, similarity, minNrShouldMatch, required, prohibited, optional, maxCoord);
}
@Override
@@ -363,7 +351,7 @@ public class BooleanQuery extends Query
@Override
public Weight createWeight(Searcher searcher) throws IOException {
- return new BooleanWeight(searcher);
+ return new BooleanWeight(searcher, disableCoord);
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer.java Sat Jan 8 18:56:31 2011
@@ -197,9 +197,9 @@ final class BooleanScorer extends Scorer
private Bucket current;
private int doc = -1;
- BooleanScorer(Weight weight, Similarity similarity, int minNrShouldMatch,
+ BooleanScorer(Weight weight, boolean disableCoord, Similarity similarity, int minNrShouldMatch,
List<Scorer> optionalScorers, List<Scorer> prohibitedScorers, int maxCoord) throws IOException {
- super(similarity, weight);
+ super(null, weight); // Similarity not used
this.minNrShouldMatch = minNrShouldMatch;
if (optionalScorers != null && optionalScorers.size() > 0) {
@@ -222,9 +222,8 @@ final class BooleanScorer extends Scorer
}
coordFactors = new float[optionalScorers.size() + 1];
- Similarity sim = getSimilarity();
for (int i = 0; i < coordFactors.length; i++) {
- coordFactors[i] = sim.coord(i, maxCoord);
+ coordFactors[i] = disableCoord ? 1.0f : similarity.coord(i, maxCoord);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java Sat Jan 8 18:56:31 2011
@@ -42,14 +42,12 @@ class BooleanScorer2 extends Scorer {
int maxCoord = 0; // to be increased for each non prohibited scorer
int nrMatchers; // to be increased by score() of match counting scorers.
- void init() { // use after all scorers have been added.
+ void init(Similarity sim, boolean disableCoord) { // use after all scorers have been added.
coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1];
- Similarity sim = getSimilarity();
for (int i = 0; i < coordFactors.length; i++) {
- coordFactors[i] = sim.coord(i, maxCoord);
+ coordFactors[i] = disableCoord ? 1.0f : sim.coord(i, maxCoord);
}
}
-
}
private final Coordinator coordinator;
@@ -82,9 +80,9 @@ class BooleanScorer2 extends Scorer {
* @param optional
* the list of optional scorers.
*/
- public BooleanScorer2(Weight weight, Similarity similarity, int minNrShouldMatch,
+ public BooleanScorer2(Weight weight, boolean disableCoord, Similarity similarity, int minNrShouldMatch,
List<Scorer> required, List<Scorer> prohibited, List<Scorer> optional, int maxCoord) throws IOException {
- super(similarity, weight);
+ super(null, weight); // Similarity not used
if (minNrShouldMatch < 0) {
throw new IllegalArgumentException("Minimum number of optional scorers should not be negative");
}
@@ -96,8 +94,8 @@ class BooleanScorer2 extends Scorer {
requiredScorers = required;
prohibitedScorers = prohibited;
- coordinator.init();
- countingSumScorer = makeCountingSumScorer();
+ coordinator.init(similarity, disableCoord);
+ countingSumScorer = makeCountingSumScorer(disableCoord, similarity);
}
/** Count a scorer as a single match. */
@@ -109,7 +107,7 @@ class BooleanScorer2 extends Scorer {
private float lastDocScore = Float.NaN;
SingleMatchScorer(Scorer scorer) {
- super(scorer.getSimilarity());
+ super(null); // No similarity used.
this.scorer = scorer;
}
@@ -164,12 +162,12 @@ class BooleanScorer2 extends Scorer {
};
}
- private static final Similarity defaultSimilarity = Similarity.getDefault();
-
- private Scorer countingConjunctionSumScorer(List<Scorer> requiredScorers) throws IOException {
+ private Scorer countingConjunctionSumScorer(boolean disableCoord,
+ Similarity similarity,
+ List<Scorer> requiredScorers) throws IOException {
// each scorer from the list counted as a single matcher
final int requiredNrMatchers = requiredScorers.size();
- return new ConjunctionScorer(defaultSimilarity, requiredScorers) {
+ return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) {
private int lastScoredDoc = -1;
// Save the score of lastScoredDoc, so that we don't compute it more than
// once in score().
@@ -192,8 +190,10 @@ class BooleanScorer2 extends Scorer {
};
}
- private Scorer dualConjunctionSumScorer(Scorer req1, Scorer req2) throws IOException { // non counting.
- return new ConjunctionScorer(defaultSimilarity, new Scorer[]{req1, req2});
+ private Scorer dualConjunctionSumScorer(boolean disableCoord,
+ Similarity similarity,
+ Scorer req1, Scorer req2) throws IOException { // non counting.
+ return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(2, 2), new Scorer[]{req1, req2});
// All scorers match, so defaultSimilarity always has 1 as
// the coordination factor.
// Therefore the sum of the scores of two scorers
@@ -203,13 +203,14 @@ class BooleanScorer2 extends Scorer {
/** Returns the scorer to be used for match counting and score summing.
* Uses requiredScorers, optionalScorers and prohibitedScorers.
*/
- private Scorer makeCountingSumScorer() throws IOException { // each scorer counted as a single matcher
+ private Scorer makeCountingSumScorer(boolean disableCoord,
+ Similarity similarity) throws IOException { // each scorer counted as a single matcher
return (requiredScorers.size() == 0)
- ? makeCountingSumScorerNoReq()
- : makeCountingSumScorerSomeReq();
+ ? makeCountingSumScorerNoReq(disableCoord, similarity)
+ : makeCountingSumScorerSomeReq(disableCoord, similarity);
}
- private Scorer makeCountingSumScorerNoReq() throws IOException { // No required scorers
+ private Scorer makeCountingSumScorerNoReq(boolean disableCoord, Similarity similarity) throws IOException { // No required scorers
// minNrShouldMatch optional scorers are required, but at least 1
int nrOptRequired = (minNrShouldMatch < 1) ? 1 : minNrShouldMatch;
Scorer requiredCountingSumScorer;
@@ -217,24 +218,27 @@ class BooleanScorer2 extends Scorer {
requiredCountingSumScorer = countingDisjunctionSumScorer(optionalScorers, nrOptRequired);
else if (optionalScorers.size() == 1)
requiredCountingSumScorer = new SingleMatchScorer(optionalScorers.get(0));
- else
- requiredCountingSumScorer = countingConjunctionSumScorer(optionalScorers);
+ else {
+ requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, similarity, optionalScorers);
+ }
return addProhibitedScorers(requiredCountingSumScorer);
}
- private Scorer makeCountingSumScorerSomeReq() throws IOException { // At least one required scorer.
+ private Scorer makeCountingSumScorerSomeReq(boolean disableCoord, Similarity similarity) throws IOException { // At least one required scorer.
if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required.
ArrayList<Scorer> allReq = new ArrayList<Scorer>(requiredScorers);
allReq.addAll(optionalScorers);
- return addProhibitedScorers(countingConjunctionSumScorer(allReq));
+ return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, similarity, allReq));
} else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer
Scorer requiredCountingSumScorer =
requiredScorers.size() == 1
? new SingleMatchScorer(requiredScorers.get(0))
- : countingConjunctionSumScorer(requiredScorers);
+ : countingConjunctionSumScorer(disableCoord, similarity, requiredScorers);
if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers
return addProhibitedScorers(
dualConjunctionSumScorer( // non counting
+ disableCoord,
+ similarity,
requiredCountingSumScorer,
countingDisjunctionSumScorer(
optionalScorers,
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java Sat Jan 8 18:56:31 2011
@@ -29,14 +29,14 @@ class ConjunctionScorer extends Scorer {
private final float coord;
private int lastDoc = -1;
- public ConjunctionScorer(Similarity similarity, Collection<Scorer> scorers) throws IOException {
- this(similarity, scorers.toArray(new Scorer[scorers.size()]));
+ public ConjunctionScorer(float coord, Collection<Scorer> scorers) throws IOException {
+ this(coord, scorers.toArray(new Scorer[scorers.size()]));
}
- public ConjunctionScorer(Similarity similarity, Scorer... scorers) throws IOException {
- super(similarity);
+ public ConjunctionScorer(float coord, Scorer... scorers) throws IOException {
+ super(null);
this.scorers = scorers;
- coord = similarity.coord(scorers.length, scorers.length);
+ this.coord = coord;
for (int i = 0; i < scorers.length; i++) {
if (scorers[i].nextDoc() == NO_MORE_DOCS) {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java Sat Jan 8 18:56:31 2011
@@ -37,13 +37,7 @@ public class DefaultSimilarity extends S
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return (state.getBoost() * lengthNorm(field, numTerms));
- }
-
- /** Implemented as <code>1/sqrt(numTerms)</code>. */
- @Override
- public float lengthNorm(String fieldName, int numTerms) {
- return (float)(1.0 / Math.sqrt(numTerms));
+ return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
}
/** Implemented as <code>1/sqrt(sumOfSquaredWeights)</code>. */
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/Similarity.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/Similarity.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/Similarity.java Sat Jan 8 18:56:31 2011
@@ -462,12 +462,14 @@ import java.util.Collection;
* {@link org.apache.lucene.document.Fieldable#setBoost(float) field.setBoost()}
* before adding the field to a document.
* </li>
- * <li>{@link #lengthNorm(String, int) <b>lengthNorm</b>(field)} - computed
+ * <li><b>lengthNorm</b> - computed
* when the document is added to the index in accordance with the number of tokens
* of this field in the document, so that shorter fields contribute more to the score.
* LengthNorm is computed by the Similarity class in effect at indexing.
* </li>
* </ul>
+ * The {@link #computeNorm} method is responsible for
+ * combining all of these factors into a single float.
*
* <p>
* When a document is added to the index, all the above factors are multiplied.
@@ -480,7 +482,7 @@ import java.util.Collection;
* norm(t,d) =
* {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()}
* ·
- * {@link #lengthNorm(String, int) lengthNorm(field)}
+ * lengthNorm
* ·
* </td>
* <td valign="bottom" align="center" rowspan="1">
@@ -591,12 +593,23 @@ public abstract class Similarity impleme
}
/**
- * Compute the normalization value for a field, given the accumulated
+ * Computes the normalization value for a field, given the accumulated
* state of term processing for this field (see {@link FieldInvertState}).
*
* <p>Implementations should calculate a float value based on the field
* state and then return that value.
*
+ * <p>Matches in longer fields are less precise, so implementations of this
+ * method usually return smaller values when <code>state.getLength()</code> is large,
+ * and larger values when <code>state.getLength()</code> is small.
+ *
+ * <p>Note that the return values are computed under
+ * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)}
+ * and then stored using
+ * {@link #encodeNormValue(float)}.
+ * Thus they have limited precision, and documents
+ * must be re-indexed if this method is altered.
+ *
* <p>For backward compatibility this method by default calls
* {@link #lengthNorm(String, int)} passing
* {@link FieldInvertState#getLength()} as the second argument, and
@@ -608,9 +621,7 @@ public abstract class Similarity impleme
* @param state current processing state for this field
* @return the calculated float norm
*/
- public float computeNorm(String field, FieldInvertState state) {
- return (state.getBoost() * lengthNorm(field, state.getLength()));
- }
+ public abstract float computeNorm(String field, FieldInvertState state);
/** Computes the normalization value for a field given the total number of
* terms contained in a field. These values, together with field boosts, are
@@ -634,8 +645,13 @@ public abstract class Similarity impleme
* @return a normalization factor for hits on this field of this document
*
* @see org.apache.lucene.document.Field#setBoost(float)
+ *
+ * @deprecated Please override computeNorm instead
*/
- public abstract float lengthNorm(String fieldName, int numTokens);
+ @Deprecated
+ public final float lengthNorm(String fieldName, int numTokens) {
+ throw new UnsupportedOperationException("please use computeNorm instead");
+ }
/** Computes the normalization value for a query given the sum of the squared
* weights of each of the query terms. This value is multiplied into the
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java Sat Jan 8 18:56:31 2011
@@ -21,7 +21,10 @@ import org.apache.lucene.index.FieldInve
/** Expert: Delegating scoring implementation. Useful in {@link
* Query#getSimilarity(Searcher)} implementations, to override only certain
- * methods of a Searcher's Similarity implementation.. */
+ * methods of a Searcher's Similarity implementation..
+ * @deprecated this class will be removed in 4.0. Please
+ * subclass {@link Similarity} or {@link DefaultSimilarity} instead. */
+@Deprecated
public class SimilarityDelegator extends Similarity {
private Similarity delegee;
@@ -38,12 +41,7 @@ public class SimilarityDelegator extends
public float computeNorm(String fieldName, FieldInvertState state) {
return delegee.computeNorm(fieldName, state);
}
-
- @Override
- public float lengthNorm(String fieldName, int numTerms) {
- return delegee.lengthNorm(fieldName, numTerms);
- }
-
+
@Override
public float queryNorm(float sumOfSquaredWeights) {
return delegee.queryNorm(sumOfSquaredWeights);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Sat Jan 8 18:56:31 2011
@@ -42,8 +42,9 @@ public class TestIndexReaderCloneNorms e
private class SimilarityOne extends DefaultSimilarity {
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ // diable length norm
+ return state.getBoost();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java Sat Jan 8 18:56:31 2011
@@ -41,8 +41,9 @@ public class TestNorms extends LuceneTes
private class SimilarityOne extends DefaultSimilarity {
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ // Disable length norm
+ return state.getBoost();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Sat Jan 8 18:56:31 2011
@@ -41,7 +41,7 @@ import org.apache.lucene.search.Explanat
public class TestOmitTf extends LuceneTestCase {
public static class SimpleSimilarity extends Similarity {
- @Override public float lengthNorm(String field, int numTerms) { return 1.0f; }
+ @Override public float computeNorm(String field, FieldInvertState state) { return state.getBoost(); }
@Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Sat Jan 8 18:56:31 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.PriorityQueue;
@@ -384,7 +385,7 @@ final class JustCompileSearch {
}
@Override
- public float lengthNorm(String fieldName, int numTokens) {
+ public float computeNorm(String fieldName, FieldInvertState state) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java Sat Jan 8 18:56:31 2011
@@ -83,7 +83,7 @@ public class TestBooleanScorer extends L
}
}};
- BooleanScorer bs = new BooleanScorer(null, sim, 1, Arrays.asList(scorers), null, scorers.length);
+ BooleanScorer bs = new BooleanScorer(null, false, sim, 1, Arrays.asList(scorers), null, scorers.length);
assertEquals("should have received 3000", 3000, bs.nextDoc());
assertEquals("should have received NO_MORE_DOCS", DocIdSetIterator.NO_MORE_DOCS, bs.nextDoc());
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Sat Jan 8 18:56:31 2011
@@ -22,6 +22,7 @@ import org.apache.lucene.analysis.Whites
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
@@ -58,8 +59,9 @@ public class TestDisjunctionMaxQuery ext
}
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1.0f;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ // Disable length norm
+ return state.getBoost();
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java Sat Jan 8 18:56:31 2011
@@ -23,6 +23,8 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SetBasedFieldSelector;
+
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -379,7 +381,7 @@ public class TestMultiSearcher extends L
@Override
public float coord(int overlap, int maxOverlap) { return 1.0f; }
@Override
- public float lengthNorm(String fieldName, int numTokens) { return 1.0f; }
+ public float computeNorm(String fieldName, FieldInvertState state) { return state.getBoost(); }
@Override
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSimilarity.java Sat Jan 8 18:56:31 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.util.LuceneTest
import java.io.IOException;
import java.util.Collection;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -38,7 +39,7 @@ import org.apache.lucene.search.Explanat
public class TestSimilarity extends LuceneTestCase {
public static class SimpleSimilarity extends Similarity {
- @Override public float lengthNorm(String field, int numTerms) { return 1.0f; }
+ @Override public float computeNorm(String field, FieldInvertState state) { return state.getBoost(); }
@Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Sat Jan 8 18:56:31 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
@@ -306,8 +307,8 @@ public class TestPayloadNearQuery extend
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
- @Override public float lengthNorm(String fieldName, int numTerms) {
- return 1.0f;
+ @Override public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
@Override public float queryNorm(float sumOfSquaredWeights) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Sat Jan 8 18:56:31 2011
@@ -34,6 +34,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.LowerCaseTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
@@ -298,8 +299,8 @@ public class TestPayloadTermQuery extend
//Make everything else 1 so we see the effect of the payload
//!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@Override
- public float lengthNorm(String fieldName, int numTerms) {
- return 1;
+ public float computeNorm(String fieldName, FieldInvertState state) {
+ return state.getBoost();
}
@Override
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1056762&r1=1056761&r2=1056762&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Sat Jan 8 18:56:31 2011
@@ -418,7 +418,7 @@ public class TestSpans extends LuceneTes
public Similarity getSimilarity(Searcher s) {
return sim;
}
- };
+ };
Scorer spanScorer = snq.weight(searcher).scorer(searcher.getIndexReader(), true, false);