You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/08 07:08:08 UTC
svn commit: r1144158 [1/4] - in /lucene/dev/trunk: lucene/
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/misc/src/java/org/apache/lucene/index/ l...
Author: rmuir
Date: Fri Jul 8 05:08:05 2011
New Revision: 1144158
URL: http://svn.apache.org/viewvc?rev=1144158&view=rev
Log:
LUCENE-2392: decouple vector space scoring from Query/Weight/Scorer
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java (with props)
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/TermContext.java (with props)
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (with props)
Removed:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/MIGRATE.txt
lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/AbstractField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Fieldable.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Explanation.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseScorer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Similarity.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermCollectingRewrite.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Weight.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java
lucene/dev/trunk/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSetNorm.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
lucene/dev/trunk/modules/join/src/java/org/apache/lucene/search/join/BlockJoinQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
lucene/dev/trunk/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
lucene/dev/trunk/solr/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/JoinQParserPlugin.java
lucene/dev/trunk/solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
lucene/dev/trunk/solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Jul 8 05:08:05 2011
@@ -156,6 +156,12 @@ Changes in backwards compatibility polic
the queries module and can be found at o.a.l.queries.function. See MIGRATE.txt
for more information (Chris Male)
+* LUCENE-2392: Decoupled vector space scoring from Query/Weight/Scorer. If you
+ extended Similarity directly before, you should extend TFIDFSimilarity instead.
+ Similarity is now a lower-level API to implement other scoring algorithms.
+ See MIGRATE.txt for more details.
+ (David Nemeskey, Simon Willnauer, Mike Mccandless, Robert Muir)
+
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
Modified: lucene/dev/trunk/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/MIGRATE.txt?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/MIGRATE.txt (original)
+++ lucene/dev/trunk/lucene/MIGRATE.txt Fri Jul 8 05:08:05 2011
@@ -382,3 +382,13 @@ LUCENE-1458, LUCENE-2111: Flexible Index
- o.a.l.search.function.ShortFieldSource -> o.a.l.queries.function.valuesource.ShortFieldSource
- o.a.l.search.function.ValueSource -> o.a.l.queries.function.ValueSource
- o.a.l.search.function.ValueSourceQuery -> o.a.l.queries.function.FunctionQuery
+
+* LUCENE-2392: Enable flexible scoring:
+
+ The existing "Similarity" api is now TFIDFSimilarity, if you were extending
+ Similarity before, you should likely extend this instead.
+
+ Weight.normalize no longer takes a norm value that incorporates the top-level
+ boost from outer queries such as BooleanQuery, instead it takes 2 parameters,
+ the outer boost (topLevelBoost) and the norm. Weight.sumOfSquaredWeights has
+ been renamed to Weight.getValueForNormalization().
Modified: lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (original)
+++ lucene/dev/trunk/lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java Fri Jul 8 05:08:05 2011
@@ -240,8 +240,7 @@ public class InstantiatedIndexWriter imp
final FieldInvertState invertState = new FieldInvertState();
invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost());
invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength);
- final float norm = similarityProvider.get(fieldName).computeNorm(invertState);
- normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).encodeNormValue(norm);
+ normsByFieldNameAndDocumentNumber.get(fieldName)[document.getDocumentNumber()] = similarityProvider.get(fieldName).computeNorm(invertState);
} else {
System.currentTimeMillis();
}
Modified: lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Fri Jul 8 05:08:05 2011
@@ -51,7 +51,6 @@ import org.apache.lucene.index.TermFreqV
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorMapper;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.IndexSearcher;
@@ -1202,19 +1201,18 @@ public class MemoryIndex {
int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
float boost = info != null ? info.getBoost() : 1.0f;
FieldInvertState invertState = new FieldInvertState(0, numTokens, numOverlapTokens, 0, boost);
- float n = fieldSim.computeNorm(invertState);
- byte norm = fieldSim.encodeNormValue(n);
+ byte norm = fieldSim.computeNorm(invertState);
norms = new byte[] {norm};
// cache it for future reuse
cachedNorms = norms;
cachedFieldName = fieldName;
cachedSimilarity = sim;
- if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + n + ":" + norm + ":" + numTokens);
+ if (DEBUG) System.err.println("MemoryIndexReader.norms: " + fieldName + ":" + norm + ":" + numTokens);
}
return norms;
}
-
+
@Override
protected void doSetNorm(int doc, String fieldName, byte value) {
throw new UnsupportedOperationException();
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java Fri Jul 8 05:08:05 2011
@@ -147,7 +147,7 @@ public class FieldNormModifier {
for (int d = 0; d < termCounts.length; d++) {
if (liveDocs == null || liveDocs.get(d)) {
invertState.setLength(termCounts[d]);
- subReader.setNorm(d, field, fieldSim.encodeNormValue(fieldSim.computeNorm(invertState)));
+ subReader.setNorm(d, field, fieldSim.computeNorm(invertState));
}
}
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Fri Jul 8 05:08:05 2011
@@ -106,7 +106,7 @@ public class SweetSpotSimilarity extends
* discountOverlaps is true by default or true for this
* specific field. */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTokens;
if (discountOverlaps)
@@ -114,7 +114,7 @@ public class SweetSpotSimilarity extends
else
numTokens = state.getLength();
- return state.getBoost() * computeLengthNorm(numTokens);
+ return encodeNormValue(state.getBoost() * computeLengthNorm(numTokens));
}
/**
Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Fri Jul 8 05:08:05 2011
@@ -49,8 +49,8 @@ public class TestFieldNormModifier exten
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Fri Jul 8 05:08:05 2011
@@ -21,6 +21,7 @@ package org.apache.lucene.misc;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.index.FieldInvertState;
@@ -58,15 +59,15 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("3,10: spot i="+i,
1.0f,
- s.computeNorm(invertState),
+ ss.decodeNormValue(s.computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = s.computeNorm(invertState);
+ final byte normS = s.computeNorm(invertState);
assertEquals("3,10: 10<x : i="+i,
normD,
normS,
@@ -104,14 +105,14 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 3,10: spot i="+i,
1.0f,
- sp.get("foo").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("foo").computeNorm(invertState)),
0.0f);
}
for (int i = 10; i < 1000; i++) {
invertState.setLength(i-9);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("foo").computeNorm(invertState);
+ final byte normS = sp.get("foo").computeNorm(invertState);
assertEquals("f: 3,10: 10<x : i="+i,
normD,
normS,
@@ -121,21 +122,21 @@ public class SweetSpotSimilarityTest ext
invertState.setLength(i);
assertEquals("f: 8,13: spot i="+i,
1.0f,
- sp.get("bar").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("bar").computeNorm(invertState)),
0.0f);
}
for (int i = 6; i <=9; i++) {
invertState.setLength(i);
assertEquals("f: 6,9: spot i="+i,
1.0f,
- sp.get("yak").computeNorm(invertState),
+ ss.decodeNormValue(sp.get("yak").computeNorm(invertState)),
0.0f);
}
for (int i = 13; i < 1000; i++) {
invertState.setLength(i-12);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("bar").computeNorm(invertState);
+ final byte normS = sp.get("bar").computeNorm(invertState);
assertEquals("f: 8,13: 13<x : i="+i,
normD,
normS,
@@ -143,9 +144,9 @@ public class SweetSpotSimilarityTest ext
}
for (int i = 9; i < 1000; i++) {
invertState.setLength(i-8);
- final float normD = d.computeNorm(invertState);
+ final byte normD = d.computeNorm(invertState);
invertState.setLength(i);
- final float normS = sp.get("yak").computeNorm(invertState);
+ final byte normS = sp.get("yak").computeNorm(invertState);
assertEquals("f: 6,9: 9<x : i="+i,
normD,
normS,
@@ -157,8 +158,8 @@ public class SweetSpotSimilarityTest ext
for (int i = 9; i < 1000; i++) {
invertState.setLength(i);
- final float normSS = sp.get("a").computeNorm(invertState);
- final float normS = sp.get("b").computeNorm(invertState);
+ final byte normSS = sp.get("a").computeNorm(invertState);
+ final byte normS = sp.get("b").computeNorm(invertState);
assertTrue("s: i="+i+" : a="+normSS+
" < b="+normS,
normSS < normS);
@@ -170,8 +171,8 @@ public class SweetSpotSimilarityTest ext
SweetSpotSimilarity ss = new SweetSpotSimilarity();
- Similarity d = new DefaultSimilarity();
- Similarity s = ss;
+ TFIDFSimilarity d = new DefaultSimilarity();
+ TFIDFSimilarity s = ss;
// tf equal
@@ -222,7 +223,7 @@ public class SweetSpotSimilarityTest ext
};
ss.setHyperbolicTfFactors(3.3f, 7.7f, Math.E, 5.0f);
- Similarity s = ss;
+ TFIDFSimilarity s = ss;
for (int i = 1; i <=1000; i++) {
assertTrue("MIN tf: i="+i+" : s="+s.tf(i),
Modified: lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Fri Jul 8 05:08:05 2011
@@ -54,8 +54,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
@@ -175,8 +175,8 @@ public class TestLengthNormModifier exte
public Similarity get(String field) {
return new DefaultSimilarity() {
@Override
- public float computeNorm(FieldInvertState state) {
- return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength());
+ public byte computeNorm(FieldInvertState state) {
+ return encodeNormValue(state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()));
}
};
}
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java Fri Jul 8 05:08:05 2011
@@ -51,7 +51,11 @@ import org.apache.lucene.util.PriorityQu
*/
public class FuzzyLikeThisQuery extends Query
{
- static Similarity sim=new DefaultSimilarity();
+ // TODO: generalize this query (at least it should not reuse this static sim!
+ // a better way might be to convert this into multitermquery rewrite methods.
+ // the rewrite method can 'average' the TermContext's term statistics (docfreq,totalTermFreq)
+ // provided to TermQuery, so that the general idea is agnostic to any scoring system...
+ static TFIDFSimilarity sim=new DefaultSimilarity();
Query rewrittenQuery=null;
ArrayList<FieldVals> fieldVals=new ArrayList<FieldVals>();
Analyzer analyzer;
Modified: lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (original)
+++ lucene/dev/trunk/lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java Fri Jul 8 05:08:05 2011
@@ -44,6 +44,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
@@ -285,7 +286,7 @@ public final class MoreLikeThis {
/**
* For idf() calculations.
*/
- private Similarity similarity;// = new DefaultSimilarity();
+ private TFIDFSimilarity similarity;// = new DefaultSimilarity();
/**
* IndexReader to use
@@ -320,17 +321,17 @@ public final class MoreLikeThis {
this(ir, new DefaultSimilarity());
}
- public MoreLikeThis(IndexReader ir, Similarity sim){
+ public MoreLikeThis(IndexReader ir, TFIDFSimilarity sim){
this.ir = ir;
this.similarity = sim;
}
- public Similarity getSimilarity() {
+ public TFIDFSimilarity getSimilarity() {
return similarity;
}
- public void setSimilarity(Similarity similarity) {
+ public void setSimilarity(TFIDFSimilarity similarity) {
this.similarity = similarity;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/AbstractField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/AbstractField.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/AbstractField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/AbstractField.java Fri Jul 8 05:08:05 2011
@@ -81,13 +81,13 @@ public abstract class AbstractField impl
* default, in the {@link
* org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)} method, the boost value is multiplied
* by the length normalization factor and then
- * rounded by {@link org.apache.lucene.search.Similarity#encodeNormValue(float)} before it is stored in the
+ * rounded by {@link org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
* @see org.apache.lucene.document.Document#setBoost(float)
* @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
+ * @see org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)
*/
public void setBoost(float boost) {
this.boost = boost;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/document/Fieldable.java Fri Jul 8 05:08:05 2011
@@ -48,13 +48,13 @@ public interface Fieldable {
* default, in the {@link
* org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)} method, the boost value is multiplied
* by the length normalization factor
- * and then rounded by {@link org.apache.lucene.search.Similarity#encodeNormValue(float)} before it is stored in the
+ * and then rounded by {@link org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
* @see org.apache.lucene.document.Document#setBoost(float)
* @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.Similarity#encodeNormValue(float)
+ * @see org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)
*/
void setBoost(float boost);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Jul 8 05:08:05 2011
@@ -1025,7 +1025,7 @@ public abstract class IndexReader implem
public abstract byte[] norms(String field) throws IOException;
/** Expert: Resets the normalization factor for the named field of the named
- * document. The norm represents the product of the field's {@link
+ * document. By default, The norm represents the product of the field's {@link
* org.apache.lucene.document.Fieldable#setBoost(float) boost} and its
* length normalization}. Thus, to preserve the length normalization
* values when resetting this, one should base the new value upon the old.
@@ -1034,7 +1034,8 @@ public abstract class IndexReader implem
* this method throws {@link IllegalStateException}.
*
* @see #norms(String)
- * @see Similarity#decodeNormValue(byte)
+ * @see Similarity#computeNorm(FieldInvertState)
+ * @see org.apache.lucene.search.DefaultSimilarity#decodeNormValue(byte)
* @throws StaleReaderException if the index has changed
* since this reader was opened
* @throws CorruptIndexException if the index is corrupt
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java Fri Jul 8 05:08:05 2011
@@ -72,8 +72,7 @@ final class NormsWriterPerField extends
assert norms.length == upto;
norms = ArrayUtil.grow(norms, 1+upto);
}
- final float norm = similarity.computeNorm(fieldState);
- norms[upto] = similarity.encodeNormValue(norm);
+ norms[upto] = similarity.computeNorm(fieldState);
docIDs[upto] = docState.docID;
upto++;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Fri Jul 8 05:08:05 2011
@@ -183,14 +183,11 @@ public class BooleanQuery extends Query
public Query getQuery() { return BooleanQuery.this; }
@Override
- public float getValue() { return getBoost(); }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float sum = 0.0f;
for (int i = 0 ; i < weights.size(); i++) {
// call sumOfSquaredWeights for all clauses in case of side effects
- float s = weights.get(i).sumOfSquaredWeights(); // sum sub weights
+ float s = weights.get(i).getValueForNormalization(); // sum sub weights
if (!clauses.get(i).isProhibited())
// only add to sum for non-prohibited clauses
sum += s;
@@ -206,11 +203,11 @@ public class BooleanQuery extends Query
}
@Override
- public void normalize(float norm) {
- norm *= getBoost(); // incorporate boost
+ public void normalize(float norm, float topLevelBoost) {
+ topLevelBoost *= getBoost(); // incorporate boost
for (Weight w : weights) {
// normalize all clauses, (even if prohibited in case of side affects)
- w.normalize(norm);
+ w.normalize(norm, topLevelBoost);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java Fri Jul 8 05:08:05 2011
@@ -27,7 +27,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
@@ -77,7 +77,7 @@ class ConstantScoreAutoRewrite extends T
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, TermContext states) {
topLevel.add(new TermQuery(term, states), BooleanClause.Occur.SHOULD);
}
@@ -140,9 +140,9 @@ class ConstantScoreAutoRewrite extends T
assert termState != null;
if (pos < 0) {
pos = (-pos)-1;
- array.termState[pos].register(termState, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos].register(termState, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
} else {
- array.termState[pos] = new PerReaderTermState(topReaderContext, termState, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos] = new TermContext(topReaderContext, termState, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
}
return true;
}
@@ -183,9 +183,9 @@ class ConstantScoreAutoRewrite extends T
return true;
}
- /** Special implementation of BytesStartArray that keeps parallel arrays for {@link PerReaderTermState} */
+ /** Special implementation of BytesStartArray that keeps parallel arrays for {@link TermContext} */
static final class TermStateByteStart extends DirectBytesStartArray {
- PerReaderTermState[] termState;
+ TermContext[] termState;
public TermStateByteStart(int initSize) {
super(initSize);
@@ -194,7 +194,7 @@ class ConstantScoreAutoRewrite extends T
@Override
public int[] init() {
final int[] ord = super.init();
- termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ termState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert termState.length >= ord.length;
return ord;
}
@@ -203,7 +203,7 @@ class ConstantScoreAutoRewrite extends T
public int[] grow() {
final int[] ord = super.grow();
if (termState.length < ord.length) {
- PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ TermContext[] tmpTermState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
termState = tmpTermState;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Fri Jul 8 05:08:05 2011
@@ -110,24 +110,19 @@ public class ConstantScoreQuery extends
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
// we calculate sumOfSquaredWeights of the inner weight, but ignore it (just to initialize everything)
- if (innerWeight != null) innerWeight.sumOfSquaredWeights();
+ if (innerWeight != null) innerWeight.getValueForNormalization();
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@Override
- public void normalize(float norm) {
- this.queryNorm = norm;
+ public void normalize(float norm, float topLevelBoost) {
+ this.queryNorm = norm * topLevelBoost;
queryWeight *= this.queryNorm;
// we normalize the inner weight, but ignore it (just to initialize everything)
- if (innerWeight != null) innerWeight.normalize(norm);
+ if (innerWeight != null) innerWeight.normalize(norm, topLevelBoost);
}
@Override
@@ -148,7 +143,7 @@ public class ConstantScoreQuery extends
if (disi == null) {
return null;
}
- return new ConstantScorer(disi, this);
+ return new ConstantScorer(disi, this, queryWeight);
}
@Override
@@ -181,9 +176,9 @@ public class ConstantScoreQuery extends
final DocIdSetIterator docIdSetIterator;
final float theScore;
- public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w) throws IOException {
+ public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) throws IOException {
super(w);
- theScore = w.getValue();
+ this.theScore = theScore;
this.docIdSetIterator = docIdSetIterator;
}
@@ -212,7 +207,7 @@ public class ConstantScoreQuery extends
@Override
public void setScorer(Scorer scorer) throws IOException {
// we must wrap again here, but using the scorer passed in as parameter:
- collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight));
+ collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore));
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java Fri Jul 8 05:08:05 2011
@@ -20,7 +20,7 @@ import org.apache.lucene.index.FieldInve
*/
/** Expert: Default scoring implementation. */
-public class DefaultSimilarity extends Similarity {
+public class DefaultSimilarity extends TFIDFSimilarity {
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
@@ -31,15 +31,15 @@ public class DefaultSimilarity extends S
*
* @lucene.experimental */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
}
-
+
/** Implemented as <code>sqrt(freq)</code>. */
@Override
public float tf(float freq) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Fri Jul 8 05:08:05 2011
@@ -110,16 +110,12 @@ public class DisjunctionMaxQuery extends
@Override
public Query getQuery() { return DisjunctionMaxQuery.this; }
- /** Return our boost */
- @Override
- public float getValue() { return getBoost(); }
-
/** Compute the sub of squared weights of us applied to our subqueries. Used for normalization. */
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float max = 0.0f, sum = 0.0f;
for (Weight currentWeight : weights) {
- float sub = currentWeight.sumOfSquaredWeights();
+ float sub = currentWeight.getValueForNormalization();
sum += sub;
max = Math.max(max, sub);
@@ -130,10 +126,10 @@ public class DisjunctionMaxQuery extends
/** Apply the computed normalization factor to our subqueries */
@Override
- public void normalize(float norm) {
- norm *= getBoost(); // Incorporate our boost
+ public void normalize(float norm, float topLevelBoost) {
+ topLevelBoost *= getBoost(); // Incorporate our boost
for (Weight wt : weights) {
- wt.normalize(norm);
+ wt.normalize(norm, topLevelBoost);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java Fri Jul 8 05:08:05 2011
@@ -23,12 +23,6 @@ import java.util.Arrays;
import org.apache.lucene.index.*;
final class ExactPhraseScorer extends Scorer {
- private final byte[] norms;
- private final float value;
-
- private static final int SCORE_CACHE_SIZE = 32;
- private final float[] scoreCache = new float[SCORE_CACHE_SIZE];
-
private final int endMinus1;
private final static int CHUNK = 4096;
@@ -60,14 +54,12 @@ final class ExactPhraseScorer extends Sc
private int docID = -1;
private int freq;
- private final Similarity similarity;
+ private final Similarity.ExactDocScorer docScorer;
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, byte[] norms) throws IOException {
+ Similarity.ExactDocScorer docScorer) throws IOException {
super(weight);
- this.similarity = similarity;
- this.norms = norms;
- this.value = weight.getValue();
+ this.docScorer = docScorer;
chunkStates = new ChunkState[postings.length];
@@ -88,10 +80,6 @@ final class ExactPhraseScorer extends Sc
return;
}
}
-
- for (int i = 0; i < SCORE_CACHE_SIZE; i++) {
- scoreCache[i] = similarity.tf((float) i) * value;
- }
}
@Override
@@ -206,13 +194,7 @@ final class ExactPhraseScorer extends Sc
@Override
public float score() throws IOException {
- final float raw; // raw score
- if (freq < SCORE_CACHE_SIZE) {
- raw = scoreCache[freq];
- } else {
- raw = similarity.tf((float) freq) * value;
- }
- return norms == null ? raw : raw * similarity.decodeNormValue(norms[docID]); // normalize
+ return docScorer.score(docID, freq);
}
private int phraseFreq() throws IOException {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Explanation.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Explanation.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Explanation.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/Explanation.java Fri Jul 8 05:08:05 2011
@@ -125,25 +125,4 @@ public class Explanation {
return buffer.toString();
}
-
- /**
- * Small Util class used to pass both an idf factor as well as an
- * explanation for that factor.
- *
- * This class will likely be held on a {@link Weight}, so be aware
- * before storing any large or un-serializable fields.
- *
- */
- public static abstract class IDFExplanation {
- /**
- * @return the idf factor
- */
- public abstract float getIdf();
- /**
- * This should be calculated lazily if possible.
- *
- * @return the explanation for the idf factor.
- */
- public abstract String explain();
- }
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/FilteredQuery.java Fri Jul 8 05:08:05 2011
@@ -63,21 +63,15 @@ extends Query {
public Weight createWeight(final IndexSearcher searcher) throws IOException {
final Weight weight = query.createWeight (searcher);
return new Weight() {
- private float value;
-
- // pass these methods through to enclosed query's weight
- @Override
- public float getValue() { return value; }
@Override
- public float sumOfSquaredWeights() throws IOException {
- return weight.sumOfSquaredWeights() * getBoost() * getBoost();
+ public float getValueForNormalization() throws IOException {
+ return weight.getValueForNormalization() * getBoost() * getBoost();
}
@Override
- public void normalize (float v) {
- weight.normalize(v);
- value = weight.getValue() * getBoost();
+ public void normalize (float norm, float topLevelBoost) {
+ weight.normalize(norm, topLevelBoost);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Fri Jul 8 05:08:05 2011
@@ -674,11 +674,11 @@ public class IndexSearcher {
public Weight createNormalizedWeight(Query query) throws IOException {
query = rewrite(query);
Weight weight = query.createWeight(this);
- float sum = weight.sumOfSquaredWeights();
- float norm = getSimilarityProvider().queryNorm(sum);
+ float v = weight.getValueForNormalization();
+ float norm = getSimilarityProvider().queryNorm(v);
if (Float.isInfinite(norm) || Float.isNaN(norm))
norm = 1.0f;
- weight.normalize(norm);
+ weight.normalize(norm, 1.0f);
return weight;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Fri Jul 8 05:08:05 2011
@@ -32,35 +32,17 @@ import java.io.IOException;
*/
public class MatchAllDocsQuery extends Query {
- public MatchAllDocsQuery() {
- this(null);
- }
-
- private final String normsField;
-
- /**
- * @param normsField Field used for normalization factor (document boost). Null if nothing.
- */
- public MatchAllDocsQuery(String normsField) {
- this.normsField = normsField;
- }
-
private class MatchAllScorer extends Scorer {
final float score;
- final byte[] norms;
private int doc = -1;
private final int maxDoc;
private final Bits liveDocs;
- private final Similarity similarity;
-
- MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
- byte[] norms) throws IOException {
+
+ MatchAllScorer(IndexReader reader, Weight w, float score) throws IOException {
super(w);
- this.similarity = similarity;
liveDocs = reader.getLiveDocs();
- score = w.getValue();
+ this.score = score;
maxDoc = reader.maxDoc();
- this.norms = norms;
}
@Override
@@ -82,7 +64,7 @@ public class MatchAllDocsQuery extends Q
@Override
public float score() {
- return norms == null ? score : score * similarity.decodeNormValue(norms[docID()]);
+ return score;
}
@Override
@@ -93,12 +75,10 @@ public class MatchAllDocsQuery extends Q
}
private class MatchAllDocsWeight extends Weight {
- private Similarity similarity;
private float queryWeight;
private float queryNorm;
public MatchAllDocsWeight(IndexSearcher searcher) {
- this.similarity = normsField == null ? null : searcher.getSimilarityProvider().get(normsField);
}
@Override
@@ -112,33 +92,27 @@ public class MatchAllDocsQuery extends Q
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() {
+ public float getValueForNormalization() {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
+ public void normalize(float queryNorm, float topLevelBoost) {
+ this.queryNorm = queryNorm * topLevelBoost;
queryWeight *= this.queryNorm;
}
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new MatchAllScorer(context.reader, similarity, this,
- normsField != null ? context.reader.norms(normsField) : null);
+ return new MatchAllScorer(context.reader, this, queryWeight);
}
@Override
public Explanation explain(AtomicReaderContext context, int doc) {
// explain query weight
Explanation queryExpl = new ComplexExplanation
- (true, getValue(), "MatchAllDocsQuery, product of:");
+ (true, queryWeight, "MatchAllDocsQuery, product of:");
if (getBoost() != 1.0f) {
queryExpl.addDetail(new Explanation(getBoost(),"boost"));
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Fri Jul 8 05:08:05 2011
@@ -22,12 +22,14 @@ import java.util.*;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.Bits;
@@ -129,45 +131,35 @@ public class MultiPhraseQuery extends Qu
private class MultiPhraseWeight extends Weight {
- private Similarity similarity;
- private float value;
- private final IDFExplanation idfExp;
- private float idf;
- private float queryNorm;
- private float queryWeight;
+ private final Similarity similarity;
+ private final Similarity.Stats stats;
public MultiPhraseWeight(IndexSearcher searcher)
throws IOException {
this.similarity = searcher.getSimilarityProvider().get(field);
-
+ final ReaderContext context = searcher.getTopReaderContext();
+
// compute idf
- ArrayList<Term> allTerms = new ArrayList<Term>();
+ ArrayList<TermContext> allTerms = new ArrayList<TermContext>();
for(final Term[] terms: termArrays) {
for (Term term: terms) {
- allTerms.add(term);
+ allTerms.add(TermContext.build(context, term, true));
}
}
- idfExp = similarity.idfExplain(allTerms, searcher);
- idf = idfExp.getIdf();
+ stats = similarity.computeStats(searcher, field, getBoost(), allTerms.toArray(new TermContext[allTerms.size()]));
}
@Override
public Query getQuery() { return MultiPhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
- queryWeight *= queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ public void normalize(float queryNorm, float topLevelBoost) {
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -222,8 +214,7 @@ public class MultiPhraseQuery extends Qu
}
if (slop == 0) {
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity,
- reader.norms(field));
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -231,84 +222,29 @@ public class MultiPhraseQuery extends Qu
}
} else {
return new SloppyPhraseScorer(this, postingsFreqs, similarity,
- slop, reader.norms(field));
+ slop, similarity.sloppyDocScorer(stats, field, context));
}
}
@Override
- public Explanation explain(AtomicReaderContext context, int doc)
- throws IOException {
- ComplexExplanation result = new ComplexExplanation();
- result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
-
- Explanation idfExpl = new Explanation(idf, "idf(" + field + ":" + idfExp.explain() +")");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:");
-
- Explanation boostExpl = new Explanation(getBoost(), "boost");
- if (getBoost() != 1.0f)
- queryExpl.addDetail(boostExpl);
-
- queryExpl.addDetail(idfExpl);
-
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- idfExpl.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- ComplexExplanation fieldExpl = new ComplexExplanation();
- fieldExpl.setDescription("fieldWeight("+getQuery()+" in "+doc+
- "), product of:");
-
+ public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
Scorer scorer = scorer(context, ScorerContext.def());
- if (scorer == null) {
- return new Explanation(0.0f, "no matching docs");
- }
-
- Explanation tfExplanation = new Explanation();
- int d = scorer.advance(doc);
- float phraseFreq;
- if (d == doc) {
- phraseFreq = scorer.freq();
- } else {
- phraseFreq = 0.0f;
+ if (scorer != null) {
+ int newDoc = scorer.advance(doc);
+ if (newDoc == doc) {
+ float freq = scorer.freq();
+ SloppyDocScorer docScorer = similarity.sloppyDocScorer(stats, field, context);
+ ComplexExplanation result = new ComplexExplanation();
+ result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:");
+ Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
+ result.addDetail(scoreExplanation);
+ result.setValue(scoreExplanation.getValue());
+ result.setMatch(true);
+ return result;
+ }
}
-
- tfExplanation.setValue(similarity.tf(phraseFreq));
- tfExplanation.setDescription("tf(phraseFreq=" + phraseFreq + ")");
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(idfExpl);
-
- Explanation fieldNormExpl = new Explanation();
- byte[] fieldNorms = context.reader.norms(field);
- float fieldNorm =
- fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(field="+field+", doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setMatch(Boolean.valueOf(tfExplanation.isMatch()));
- fieldExpl.setValue(tfExplanation.getValue() *
- idfExpl.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
- result.setMatch(fieldExpl.getMatch());
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
-
- if (queryExpl.getValue() == 1.0f)
- return fieldExpl;
-
- return result;
+
+ return new ComplexExplanation(false, 0.0f, "no matching term");
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java Fri Jul 8 05:08:05 2011
@@ -25,7 +25,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
/**
* An abstract {@link Query} that matches documents
@@ -154,7 +154,7 @@ public abstract class MultiTermQuery ext
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, TermContext states) {
final TermQuery tq = new TermQuery(term, states);
tq.setBoost(boost);
topLevel.add(tq, BooleanClause.Occur.SHOULD);
@@ -195,7 +195,7 @@ public abstract class MultiTermQuery ext
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, TermContext states) {
final Query q = new ConstantScoreQuery(new TermQuery(term, states));
q.setBoost(boost);
topLevel.add(q, BooleanClause.Occur.SHOULD);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Jul 8 05:08:05 2011
@@ -22,10 +22,16 @@ import java.util.Set;
import java.util.ArrayList;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -171,18 +177,17 @@ public class PhraseQuery extends Query {
private class PhraseWeight extends Weight {
private final Similarity similarity;
- private float value;
- private float idf;
- private float queryNorm;
- private float queryWeight;
- private IDFExplanation idfExp;
+ private final Similarity.Stats stats;
+ private transient TermContext states[];
public PhraseWeight(IndexSearcher searcher)
throws IOException {
this.similarity = searcher.getSimilarityProvider().get(field);
-
- idfExp = similarity.idfExplain(terms, searcher);
- idf = idfExp.getIdf();
+ final ReaderContext context = searcher.getTopReaderContext();
+ states = new TermContext[terms.size()];
+ for (int i = 0; i < terms.size(); i++)
+ states[i] = TermContext.build(context, terms.get(i), true);
+ stats = similarity.computeStats(searcher, field, getBoost(), states);
}
@Override
@@ -192,19 +197,13 @@ public class PhraseQuery extends Query {
public Query getQuery() { return PhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
- queryWeight *= queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ public void normalize(float queryNorm, float topLevelBoost) {
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -216,21 +215,26 @@ public class PhraseQuery extends Query {
PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];
for (int i = 0; i < terms.size(); i++) {
final Term t = terms.get(i);
+ final TermState state = states[i].get(context.ord);
+ if (state == null) { /* term doesnt exist in this segment */
+ assert termNotInReader(reader, field, t.bytes()) : "no termstate found but term exists in reader";
+ return null;
+ }
DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(liveDocs,
t.field(),
- t.bytes());
+ t.bytes(),
+ state);
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null) {
- if (reader.termDocsEnum(liveDocs, t.field(), t.bytes()) != null) {
- // term does exist, but has no positions
- throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
- } else {
- // term does not exist
- return null;
- }
+ assert (reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader";
+ // term does exist, but has no positions
+ throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
}
- postingsFreqs[i] = new PostingsAndFreq(postingsEnum, reader.docFreq(t.field(), t.bytes()), positions.get(i).intValue(), t);
+ // get the docFreq without seeking
+ TermsEnum te = reader.fields().terms(field).getThreadTermsEnum();
+ te.seekExact(t.bytes(), state);
+ postingsFreqs[i] = new PostingsAndFreq(postingsEnum, te.docFreq(), positions.get(i).intValue(), t);
}
// sort by increasing docFreq order
@@ -239,8 +243,7 @@ public class PhraseQuery extends Query {
}
if (slop == 0) { // optimize exact case
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity,
- reader.norms(field));
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -248,96 +251,35 @@ public class PhraseQuery extends Query {
}
} else {
return
- new SloppyPhraseScorer(this, postingsFreqs, similarity, slop,
- reader.norms(field));
+ new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, similarity.sloppyDocScorer(stats, field, context));
}
}
+
+ private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
+ // only called from assert
+ final Terms terms = reader.terms(field);
+ return terms == null || terms.docFreq(bytes) == 0;
+ }
@Override
- public Explanation explain(AtomicReaderContext context, int doc)
- throws IOException {
-
- ComplexExplanation result = new ComplexExplanation();
- result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
-
- StringBuilder docFreqs = new StringBuilder();
- StringBuilder query = new StringBuilder();
- query.append('\"');
- docFreqs.append(idfExp.explain());
- for (int i = 0; i < terms.size(); i++) {
- if (i != 0) {
- query.append(" ");
- }
-
- Term term = terms.get(i);
-
- query.append(term.text());
- }
- query.append('\"');
-
- Explanation idfExpl =
- new Explanation(idf, "idf(" + field + ":" + docFreqs + ")");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:");
-
- Explanation boostExpl = new Explanation(getBoost(), "boost");
- if (getBoost() != 1.0f)
- queryExpl.addDetail(boostExpl);
- queryExpl.addDetail(idfExpl);
-
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- idfExpl.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- Explanation fieldExpl = new Explanation();
- fieldExpl.setDescription("fieldWeight("+field+":"+query+" in "+doc+
- "), product of:");
-
+ public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
Scorer scorer = scorer(context, ScorerContext.def());
- if (scorer == null) {
- return new Explanation(0.0f, "no matching docs");
- }
- Explanation tfExplanation = new Explanation();
- int d = scorer.advance(doc);
- float phraseFreq;
- if (d == doc) {
- phraseFreq = scorer.freq();
- } else {
- phraseFreq = 0.0f;
+ if (scorer != null) {
+ int newDoc = scorer.advance(doc);
+ if (newDoc == doc) {
+ float freq = scorer.freq();
+ SloppyDocScorer docScorer = similarity.sloppyDocScorer(stats, field, context);
+ ComplexExplanation result = new ComplexExplanation();
+ result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:");
+ Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
+ result.addDetail(scoreExplanation);
+ result.setValue(scoreExplanation.getValue());
+ result.setMatch(true);
+ return result;
+ }
}
-
- tfExplanation.setValue(similarity.tf(phraseFreq));
- tfExplanation.setDescription("tf(phraseFreq=" + phraseFreq + ")");
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(idfExpl);
-
- Explanation fieldNormExpl = new Explanation();
- byte[] fieldNorms = context.reader.norms(field);
- float fieldNorm =
- fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(field="+field+", doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setValue(tfExplanation.getValue() *
- idfExpl.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
- result.setMatch(tfExplanation.isMatch());
- return result;
+ return new ComplexExplanation(false, 0.0f, "no matching term");
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseScorer.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseScorer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/PhraseScorer.java Fri Jul 8 05:08:05 2011
@@ -30,9 +30,6 @@ import java.io.IOException;
* means a match.
*/
abstract class PhraseScorer extends Scorer {
- protected byte[] norms;
- protected float value;
-
private boolean firstTime = true;
private boolean more = true;
protected PhraseQueue pq;
@@ -40,14 +37,12 @@ abstract class PhraseScorer extends Scor
private float freq; //phrase frequency in current doc as computed by phraseFreq().
- protected final Similarity similarity;
+ protected final Similarity.SloppyDocScorer docScorer;
PhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, byte[] norms) {
+ Similarity.SloppyDocScorer docScorer) throws IOException {
super(weight);
- this.similarity = similarity;
- this.norms = norms;
- this.value = weight.getValue();
+ this.docScorer = docScorer;
// convert tps to a list of phrase positions.
// note: phrase-position differs from term-position in that its position
@@ -107,9 +102,7 @@ abstract class PhraseScorer extends Scor
@Override
public float score() throws IOException {
- //System.out.println("scoring " + first.doc);
- float raw = similarity.tf(freq) * value; // raw score
- return norms == null ? raw : raw * similarity.decodeNormValue(norms[first.doc]); // normalize
+ return docScorer.score(first.doc, freq);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1144158&r1=1144157&r2=1144158&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java Fri Jul 8 05:08:05 2011
@@ -28,7 +28,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
@@ -56,7 +56,7 @@ public abstract class ScoringRewrite<Q e
@Override
protected void addClause(BooleanQuery topLevel, Term term, int docCount,
- float boost, PerReaderTermState states) {
+ float boost, TermContext states) {
final TermQuery tq = new TermQuery(term, states);
tq.setBoost(boost);
topLevel.add(tq, BooleanClause.Occur.SHOULD);
@@ -117,7 +117,7 @@ public abstract class ScoringRewrite<Q e
if (size > 0) {
final int sort[] = col.terms.sort(col.termsEnum.getComparator());
final float[] boost = col.array.boost;
- final PerReaderTermState[] termStates = col.array.termState;
+ final TermContext[] termStates = col.array.termState;
for (int i = 0; i < size; i++) {
final int pos = sort[i];
final Term term = new Term(query.getField(), col.terms.get(pos, new BytesRef()));
@@ -150,12 +150,12 @@ public abstract class ScoringRewrite<Q e
if (e < 0 ) {
// duplicate term: update docFreq
final int pos = (-e)-1;
- array.termState[pos].register(state, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos].register(state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums";
} else {
// new entry: we populate the entry initially
array.boost[e] = boostAtt.getBoost();
- array.termState[e] = new PerReaderTermState(topReaderContext, state, readerContext.ord, termsEnum.docFreq());
+ array.termState[e] = new TermContext(topReaderContext, state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
ScoringRewrite.this.checkMaxClauseCount(terms.size());
}
return true;
@@ -165,7 +165,7 @@ public abstract class ScoringRewrite<Q e
/** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */
static final class TermFreqBoostByteStart extends DirectBytesStartArray {
float[] boost;
- PerReaderTermState[] termState;
+ TermContext[] termState;
public TermFreqBoostByteStart(int initSize) {
super(initSize);
@@ -175,7 +175,7 @@ public abstract class ScoringRewrite<Q e
public int[] init() {
final int[] ord = super.init();
boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)];
- termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ termState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert termState.length >= ord.length && boost.length >= ord.length;
return ord;
}
@@ -185,7 +185,7 @@ public abstract class ScoringRewrite<Q e
final int[] ord = super.grow();
boost = ArrayUtil.grow(boost, ord.length);
if (termState.length < ord.length) {
- PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ TermContext[] tmpTermState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
termState = tmpTermState;
}