You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2017/11/01 23:43:53 UTC
lucene-solr:branch_7x: LUCENE-8025: Use totalTermFreq=sumDocFreq when
scoring DOCS_ONLY fields
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 1691a04ec -> 4e1ef13a1
LUCENE-8025: Use totalTermFreq=sumDocFreq when scoring DOCS_ONLY fields
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4e1ef13a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4e1ef13a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4e1ef13a
Branch: refs/heads/branch_7x
Commit: 4e1ef13a1274a3beb17b2696d08318a241e4d86e
Parents: 1691a04
Author: Robert Muir <rm...@apache.org>
Authored: Wed Nov 1 19:31:24 2017 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Wed Nov 1 19:33:31 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 ++++
.../search/similarities/BM25Similarity.java | 20 +++++++++-------
.../search/similarities/SimilarityBase.java | 25 ++++++++++----------
3 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4e1ef13a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9881bd7..dd7df53 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -16,6 +16,11 @@ Bug Fixes
refreshes or commits, and high indexing thread counts, do not
overflow an int (Mykhailo Demianenko via Mike McCandless)
+* LUCENE-8025: Use totalTermFreq=sumDocFreq when scoring DOCS_ONLY fields
+ that omit term frequency information, as it is equivalent in that case.
+ Previously bogus numbers were used, and many similarities would
+ completely degrade. (Robert Muir, Adrien Grand)
+
Optimizations
* LUCENE-8018: Smaller FieldInfos memory footprint by not retaining unnecessary
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4e1ef13a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index e693b2b..b57011d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -83,17 +83,21 @@ public class BM25Similarity extends Similarity {
return 1;
}
- /** The default implementation computes the average as <code>sumTotalTermFreq / docCount</code>,
- * or returns <code>1</code> if the index does not store sumTotalTermFreq:
- * any field that omits frequency information). */
+ /** The default implementation computes the average as <code>sumTotalTermFreq / docCount</code> */
protected float avgFieldLength(CollectionStatistics collectionStats) {
- final long sumTotalTermFreq = collectionStats.sumTotalTermFreq();
- if (sumTotalTermFreq <= 0) {
- return 1f; // field does not exist, or stat is unsupported
+ final long sumTotalTermFreq;
+ if (collectionStats.sumTotalTermFreq() == -1) {
+ // frequencies are omitted (tf=1), its # of postings
+ if (collectionStats.sumDocFreq() == -1) {
+ // theoretical case only: remove!
+ return 1f;
+ }
+ sumTotalTermFreq = collectionStats.sumDocFreq();
} else {
- final long docCount = collectionStats.docCount() == -1 ? collectionStats.maxDoc() : collectionStats.docCount();
- return (float) (sumTotalTermFreq / (double) docCount);
+ sumTotalTermFreq = collectionStats.sumTotalTermFreq();
}
+ final long docCount = collectionStats.docCount() == -1 ? collectionStats.maxDoc() : collectionStats.docCount();
+ return (float) (sumTotalTermFreq / (double) docCount);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4e1ef13a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index fb34f07..72ac89f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -107,7 +107,7 @@ public abstract class SimilarityBase extends Similarity {
long docFreq = termStats.docFreq();
long totalTermFreq = termStats.totalTermFreq();
- // codec does not supply totalTermFreq: substitute docFreq
+ // frequencies are omitted, all postings have tf=1, so totalTermFreq = docFreq
if (totalTermFreq == -1) {
totalTermFreq = docFreq;
}
@@ -115,18 +115,19 @@ public abstract class SimilarityBase extends Similarity {
final long numberOfFieldTokens;
final float avgFieldLength;
- long sumTotalTermFreq = collectionStats.sumTotalTermFreq();
-
- if (sumTotalTermFreq <= 0) {
- // field does not exist;
- // We have to provide something if codec doesnt supply these measures,
- // or if someone omitted frequencies for the field... negative values cause
- // NaN/Inf for some scorers.
- numberOfFieldTokens = docFreq;
- avgFieldLength = 1;
+ if (collectionStats.sumTotalTermFreq() == -1) {
+ // frequencies are omitted, so sumTotalTermFreq = # postings
+ if (collectionStats.sumDocFreq() == -1) {
+ // theoretical case only: remove!
+ numberOfFieldTokens = docFreq;
+ avgFieldLength = 1f;
+ } else {
+ numberOfFieldTokens = collectionStats.sumDocFreq();
+ avgFieldLength = (float) (collectionStats.sumDocFreq() / (double)numberOfDocuments);
+ }
} else {
- numberOfFieldTokens = sumTotalTermFreq;
- avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments;
+ numberOfFieldTokens = collectionStats.sumTotalTermFreq();
+ avgFieldLength = (float) (collectionStats.sumTotalTermFreq() / (double)numberOfDocuments);
}
// TODO: add sumDocFreq for field (numberOfFieldPostings)