You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by zh...@apache.org on 2026/04/17 06:41:05 UTC
(lucene) branch main updated: Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (#15929)
This is an automated email from the ASF dual-hosted git repository.
zhouhui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new fa0e704d2a0 Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (#15929)
fa0e704d2a0 is described below
commit fa0e704d2a0f941f2bbca17184472ae0e2b3743e
Author: Zhou Hui <vs...@163.com>
AuthorDate: Fri Apr 17 14:40:59 2026 +0800
Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (#15929)
---
lucene/CHANGES.txt | 1 +
lucene/MIGRATE.md | 4 ++
.../apache/lucene/search/CombinedFieldQuery.java | 34 +++++++--------
.../{CollectionStatistics.java => FieldStats.java} | 16 +++----
.../org/apache/lucene/search/IndexSearcher.java | 17 ++++----
.../org/apache/lucene/search/MultiPhraseQuery.java | 8 ++--
.../java/org/apache/lucene/search/PhraseQuery.java | 9 ++--
.../org/apache/lucene/search/SynonymQuery.java | 11 +++--
.../java/org/apache/lucene/search/TermQuery.java | 14 +++---
.../java/org/apache/lucene/search/TermScorer.java | 2 +-
.../search/{TermStatistics.java => TermStats.java} | 25 ++++++-----
.../lucene/search/similarities/BM25Similarity.java | 41 +++++++++---------
.../search/similarities/BooleanSimilarity.java | 7 ++-
.../search/similarities/ClassicSimilarity.java | 8 ++--
.../lucene/search/similarities/LMSimilarity.java | 9 ++--
.../search/similarities/MultiSimilarity.java | 9 ++--
.../similarities/PerFieldSimilarityWrapper.java | 9 ++--
.../search/similarities/RawTFSimilarity.java | 7 ++-
.../lucene/search/similarities/Similarity.java | 38 ++++++++--------
.../lucene/search/similarities/SimilarityBase.java | 25 +++++------
.../search/similarities/TFIDFSimilarity.java | 38 ++++++++--------
.../org/apache/lucene/index/TestCustomNorms.java | 7 ++-
.../apache/lucene/index/TestCustomTermFreq.java | 7 ++-
.../apache/lucene/index/TestFieldInvertState.java | 7 ++-
.../org/apache/lucene/index/TestIndexSorting.java | 9 ++--
.../apache/lucene/index/TestMaxTermFrequency.java | 7 ++-
.../test/org/apache/lucene/index/TestNorms.java | 7 ++-
.../test/org/apache/lucene/index/TestOmitTf.java | 7 ++-
.../apache/lucene/index/TestUniqueTermCount.java | 7 ++-
.../apache/lucene/search/JustCompileSearch.java | 3 +-
.../lucene/search/TestCombinedFieldQuery.java | 13 +++---
.../apache/lucene/search/TestMinShouldMatch2.java | 4 +-
.../org/apache/lucene/search/TestSimilarity.java | 2 +-
.../lucene/search/TestSimilarityProvider.java | 6 +--
.../org/apache/lucene/search/TestTermQuery.java | 5 +--
.../search/similarities/TestClassicSimilarity.java | 2 +-
.../search/similarities/TestSimilarityBase.java | 18 ++++----
.../lucene/index/memory/TestMemoryIndex.java | 7 ++-
.../function/valuesource/NormValueSource.java | 8 ++--
.../queries/spans/FieldMaskingSpanQuery.java | 4 +-
.../apache/lucene/queries/spans/SpanWeight.java | 13 +++---
.../queries/payloads/TestPayloadScoreQuery.java | 9 ++--
.../lucene/sandbox/search/CombinedFieldQuery.java | 38 ++++++++--------
.../lucene/sandbox/search/PhraseWildcardQuery.java | 50 ++++++++++++----------
.../lucene/sandbox/search/TermAutomatonQuery.java | 11 +++--
.../sandbox/search/TestCombinedFieldQuery.java | 17 ++++----
.../tests/index/BaseNormsFormatTestCase.java | 7 ++-
.../tests/search/ShardSearchingTestBase.java | 49 ++++++++++-----------
.../search/similarities/AssertingSimilarity.java | 13 +++---
.../similarities/BaseSimilarityTestCase.java | 31 ++++++--------
50 files changed, 322 insertions(+), 378 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 805fd682f5d..94b05c92f88 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -7,6 +7,7 @@ http://s.apache.org/luceneversions
API Changes
---------------------
+* GITHUB#15929: Rename CollectionStatistics to FieldStats and TermStatistics to TermStats. (Zhou Hui)
* GITHUB#15763: Deprecate Operations.complement() method. This operation can be slow and is not
recommended for production use. It will be removed in Lucene 12. (Saurabh Singh)
diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md
index 2a3eee9677d..f1189c6d8d0 100644
--- a/lucene/MIGRATE.md
+++ b/lucene/MIGRATE.md
@@ -174,6 +174,10 @@ Automaton dfa = Operations.determinize(new RegExp(re).toAutomaton(), 10000);
Query query = new AutomatonQuery(new Term("myfield", re), dfa);
```
+### CollectionStatistics and TermStatistics have been renamed to FieldStats and TermStats (GITHUB#15929)
+
+Corresponding methods and parameters have been renamed accordingly.
+
## Migration from Lucene 10.4 to Lucene 10.5
### `[Byte|Float]VectorSimilarityQuery` now performs adaptive HNSW graph traversal
diff --git a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
index 1dcd0565b2f..5adb17d7e61 100644
--- a/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
@@ -52,8 +52,8 @@ import org.apache.lucene.util.SmallFloat;
*
* <ol>
* <li>Given a list of fields and weights, it pretends there is a synthetic combined field where
- * all terms have been indexed. It computes new term and collection statistics for this
- * combined field.
+ * all terms have been indexed. It computes new term and field statistics for this combined
+ * field.
* <li>It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents.
* </ol>
*
@@ -279,41 +279,37 @@ public final class CombinedFieldQuery extends Query implements Accountable {
TermStates ts = TermStates.build(searcher, fieldTerms[i], true);
termStates[i] = ts;
if (ts.docFreq() > 0) {
- TermStatistics termStats =
- searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());
+ TermStats termStats = searcher.termStats(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());
docFreq = Math.max(termStats.docFreq(), docFreq);
totalTermFreq += (double) field.weight * termStats.totalTermFreq();
}
}
if (docFreq > 0) {
- CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher);
- TermStatistics pseudoTermStatistics =
- new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));
- this.simWeight =
- searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics);
+ FieldStats pseudoFieldStats = mergeFieldStats(searcher);
+ TermStats pseudoTermStats =
+ new TermStats(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));
+ this.simWeight = searcher.getSimilarity().scorer(boost, pseudoFieldStats, pseudoTermStats);
} else {
this.simWeight = null;
}
}
- private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher)
- throws IOException {
+ private FieldStats mergeFieldStats(IndexSearcher searcher) throws IOException {
long maxDoc = 0;
long docCount = 0;
long sumTotalTermFreq = 0;
long sumDocFreq = 0;
for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {
- CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field);
- if (collectionStats != null) {
- maxDoc = Math.max(collectionStats.maxDoc(), maxDoc);
- docCount = Math.max(collectionStats.docCount(), docCount);
- sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq);
- sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq();
+ FieldStats fieldStats = searcher.fieldStats(fieldWeight.field);
+ if (fieldStats != null) {
+ maxDoc = Math.max(fieldStats.maxDoc(), maxDoc);
+ docCount = Math.max(fieldStats.docCount(), docCount);
+ sumDocFreq = Math.max(fieldStats.sumDocFreq(), sumDocFreq);
+ sumTotalTermFreq += (double) fieldWeight.weight * fieldStats.sumTotalTermFreq();
}
}
- return new CollectionStatistics(
- "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
+ return new FieldStats("pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java b/lucene/core/src/java/org/apache/lucene/search/FieldStats.java
similarity index 90%
rename from lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java
rename to lucene/core/src/java/org/apache/lucene/search/FieldStats.java
index ba118fa5452..ca63bb75228 100644
--- a/lucene/core/src/java/org/apache/lucene/search/CollectionStatistics.java
+++ b/lucene/core/src/java/org/apache/lucene/search/FieldStats.java
@@ -19,7 +19,7 @@ package org.apache.lucene.search;
import java.util.Objects;
/**
- * Contains statistics for a collection (field).
+ * Contains statistics for a field.
*
* <p>This class holds statistics across all documents for scoring purposes:
*
@@ -54,22 +54,22 @@ import java.util.Objects;
* Terms#getDocCount()
* @param sumTotalTermFreq The total number of tokens for this field , in the range [{@link
* #sumDocFreq()} .. {@link Long#MAX_VALUE}]. This is the "word count" for this field across all
- * documents. It is the sum of {@link TermStatistics#totalTermFreq()} across all terms. It is
- * also the sum of each document's field length across all documents.
+ * documents. It is the sum of {@link TermStats#totalTermFreq()} across all terms. It is also
+ * the sum of each document's field length across all documents.
* <p>This value is always a positive number, and always at least {@link #sumDocFreq()}. @see
* Terms#getSumTotalTermFreq()
* @param sumDocFreq The total number of posting list entries for this field, in the range [{@link
* #docCount()} .. {@link #sumTotalTermFreq()}]. This is the sum of term-document pairs: the sum
- * of {@link TermStatistics#docFreq()} across all terms. It is also the sum of each document's
- * unique term count for this field across all documents.
+ * of {@link TermStats#docFreq()} across all terms. It is also the sum of each document's unique
+ * term count for this field across all documents.
* <p>This value is always a positive number, always at least {@link #docCount()}, and never
* exceeds {@link #sumTotalTermFreq()}. @see Terms#getSumDocFreq()
* @lucene.experimental
*/
-public record CollectionStatistics(
+public record FieldStats(
String field, long maxDoc, long docCount, long sumTotalTermFreq, long sumDocFreq) {
/**
- * Creates statistics instance for a collection (field).
+ * Creates statistics instance for a field.
*
* @throws IllegalArgumentException if {@code maxDoc} is negative or zero.
* @throws IllegalArgumentException if {@code docCount} is negative or zero.
@@ -77,7 +77,7 @@ public record CollectionStatistics(
* @throws IllegalArgumentException if {@code sumDocFreq} is less than {@code docCount}.
* @throws IllegalArgumentException if {@code sumTotalTermFreq} is less than {@code sumDocFreq}.
*/
- public CollectionStatistics {
+ public FieldStats {
Objects.requireNonNull(field);
if (maxDoc <= 0) {
throw new IllegalArgumentException("maxDoc must be positive, maxDoc: " + maxDoc);
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
index b9d3f47cb09..845bae5238e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -1106,32 +1106,31 @@ public class IndexSearcher {
}
/**
- * Returns {@link TermStatistics} for a term.
+ * Returns {@link TermStats} for a term.
*
* <p>This can be overridden for example, to return a term's statistics across a distributed
* collection.
*
* @param docFreq The document frequency of the term. It must be greater or equal to 1.
* @param totalTermFreq The total term frequency.
- * @return A {@link TermStatistics} (never null).
+ * @return A {@link TermStats} (never null).
* @lucene.experimental
*/
- public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)
- throws IOException {
+ public TermStats termStats(Term term, int docFreq, long totalTermFreq) throws IOException {
// This constructor will throw an exception if docFreq <= 0.
- return new TermStatistics(term.bytes(), docFreq, totalTermFreq);
+ return new TermStats(term.bytes(), docFreq, totalTermFreq);
}
/**
- * Returns {@link CollectionStatistics} for a field, or {@code null} if the field does not exist
- * (has no indexed terms)
+ * Returns {@link FieldStats} for a field, or {@code null} if the field does not exist (has no
+ * indexed terms)
*
* <p>This can be overridden for example, to return a field's statistics across a distributed
* collection.
*
* @lucene.experimental
*/
- public CollectionStatistics collectionStatistics(String field) throws IOException {
+ public FieldStats fieldStats(String field) throws IOException {
assert field != null;
long docCount = 0;
long sumTotalTermFreq = 0;
@@ -1145,7 +1144,7 @@ public class IndexSearcher {
if (docCount == 0) {
return null;
}
- return new CollectionStatistics(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);
+ return new FieldStats(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index 00b06901e5e..2812aad6440 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -216,7 +216,7 @@ public class MultiPhraseQuery extends Query {
protected Similarity.SimScorer getStats(IndexSearcher searcher) throws IOException {
// compute idf
- ArrayList<TermStatistics> allTermStats = new ArrayList<>();
+ ArrayList<TermStats> allTermStats = new ArrayList<>();
for (final Term[] terms : termArrays) {
for (Term term : terms) {
TermStates ts = termStates.get(term);
@@ -225,7 +225,7 @@ public class MultiPhraseQuery extends Query {
termStates.put(term, ts);
}
if (scoreMode.needsScores() && ts.docFreq() > 0) {
- allTermStats.add(searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));
+ allTermStats.add(searcher.termStats(term, ts.docFreq(), ts.totalTermFreq()));
}
}
}
@@ -233,9 +233,7 @@ public class MultiPhraseQuery extends Query {
return null; // none of the terms were found, we won't use sim at all
} else {
return similarity.scorer(
- boost,
- searcher.collectionStatistics(field),
- allTermStats.toArray(TermStatistics[]::new));
+ boost, searcher.fieldStats(field), allTermStats.toArray(TermStats[]::new));
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index eea2a47a947..e8c9362629c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -469,7 +469,7 @@ public class PhraseQuery extends Query {
"PhraseWeight requires that the first position is 0, call rewrite first");
}
states = new TermStates[terms.length];
- TermStatistics[] termStats = new TermStatistics[terms.length];
+ TermStats[] termStats = new TermStats[terms.length];
int termUpTo = 0;
for (int i = 0; i < terms.length; i++) {
final Term term = terms[i];
@@ -477,16 +477,13 @@ public class PhraseQuery extends Query {
if (scoreMode.needsScores()) {
TermStates ts = states[i];
if (ts.docFreq() > 0) {
- termStats[termUpTo++] =
- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq());
+ termStats[termUpTo++] = searcher.termStats(term, ts.docFreq(), ts.totalTermFreq());
}
}
}
if (termUpTo > 0) {
return similarity.scorer(
- boost,
- searcher.collectionStatistics(field),
- ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));
+ boost, searcher.fieldStats(field), ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));
} else {
return null; // no terms at all, we won't use similarity
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
index 8529f35f053..3ff634cd0e6 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java
@@ -207,7 +207,7 @@ public final class SynonymQuery extends Query {
super(query);
assert scoreMode.needsScores();
this.scoreMode = scoreMode;
- CollectionStatistics collectionStats = searcher.collectionStatistics(field);
+ FieldStats fieldStats = searcher.fieldStats(field);
long docFreq = 0;
long totalTermFreq = 0;
termStates = new TermStates[terms.length];
@@ -216,17 +216,16 @@ public final class SynonymQuery extends Query {
TermStates ts = TermStates.build(searcher, term, true);
termStates[i] = ts;
if (ts.docFreq() > 0) {
- TermStatistics termStats =
- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq());
+ TermStats termStats = searcher.termStats(term, ts.docFreq(), ts.totalTermFreq());
docFreq = Math.max(termStats.docFreq(), docFreq);
totalTermFreq += termStats.totalTermFreq();
}
}
this.similarity = searcher.getSimilarity();
if (docFreq > 0) {
- TermStatistics pseudoStats =
- new TermStatistics(new BytesRef("synonym pseudo-term"), docFreq, totalTermFreq);
- this.simWeight = similarity.scorer(boost, collectionStats, pseudoStats);
+ TermStats pseudoStats =
+ new TermStats(new BytesRef("synonym pseudo-term"), docFreq, totalTermFreq);
+ this.simWeight = similarity.scorer(boost, fieldStats, pseudoStats);
} else {
this.simWeight = null; // no terms exist at all, we won't use similarity
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 6d5cc2d9094..20b2d85ac16 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -58,18 +58,18 @@ public class TermQuery extends Query {
this.termStates = termStates;
this.similarity = searcher.getSimilarity();
- final CollectionStatistics collectionStats;
- final TermStatistics termStats;
+ final FieldStats fieldStats;
+ final TermStats termStats;
if (scoreMode.needsScores()) {
- collectionStats = searcher.collectionStatistics(term.field());
+ fieldStats = searcher.fieldStats(term.field());
termStats =
termStates.docFreq() > 0
- ? searcher.termStatistics(term, termStates.docFreq(), termStates.totalTermFreq())
+ ? searcher.termStats(term, termStates.docFreq(), termStates.totalTermFreq())
: null;
} else {
// we do not need the actual stats, use fake stats with docFreq=maxDoc=ttf=1
- collectionStats = new CollectionStatistics(term.field(), 1, 1, 1, 1);
- termStats = new TermStatistics(term.bytes(), 1, 1);
+ fieldStats = new FieldStats(term.field(), 1, 1, 1, 1);
+ termStats = new TermStats(term.bytes(), 1, 1);
}
if (termStats == null) {
@@ -79,7 +79,7 @@ public class TermQuery extends Query {
// allocations in case default BM25Scorer is used.
// See: https://github.com/apache/lucene/issues/12297
if (scoreMode.needsScores()) {
- this.simScorer = similarity.scorer(boost, collectionStats, termStats);
+ this.simScorer = similarity.scorer(boost, fieldStats, termStats);
} else {
// Assigning a dummy scorer as this is not expected to be called since scores are not
// needed.
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
index 991b3a83171..4a7169a2e1f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
@@ -83,7 +83,7 @@ public final class TermScorer extends Scorer {
}
/** Returns term frequency in the current document. */
- public final int freq() throws IOException {
+ public int freq() throws IOException {
return postingsEnum.freq();
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java b/lucene/core/src/java/org/apache/lucene/search/TermStats.java
similarity index 83%
rename from lucene/core/src/java/org/apache/lucene/search/TermStatistics.java
rename to lucene/core/src/java/org/apache/lucene/search/TermStats.java
index aeca533e9ba..639f66e0c31 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermStatistics.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermStats.java
@@ -34,8 +34,8 @@ import org.apache.lucene.util.BytesRef;
* <ul>
* <li>All statistics are positive integers: never zero or negative.
* <li>{@code docFreq} <= {@code totalTermFreq}
- * <li>{@code docFreq} <= {@code sumDocFreq} of the collection
- * <li>{@code totalTermFreq} <= {@code sumTotalTermFreq} of the collection
+ * <li>{@code docFreq} <= {@code sumDocFreq} of the field
+ * <li>{@code totalTermFreq} <= {@code sumTotalTermFreq} of the field
* </ul>
*
* <p>Values may include statistics on deleted documents that have not yet been merged away.
@@ -45,24 +45,23 @@ import org.apache.lucene.util.BytesRef;
*
* @param term Term bytes.
* <p>This value is never {@code null}.
- * @param docFreq number of documents containing the term in the collection, in the range [1 ..
- * {@link #totalTermFreq()}].
+ * @param docFreq number of documents containing the term in the field, in the range [1 .. {@link
+ * #totalTermFreq()}].
* <p>This is the document-frequency for the term: the count of documents where the term appears
* at least one time.
* <p>This value is always a positive number, and never exceeds {@link #totalTermFreq}. It also
- * cannot exceed {@link CollectionStatistics#sumDocFreq()}. @see TermsEnum#docFreq()
- * @param totalTermFreq number of occurrences of the term in the collection, in the range [{@link
- * #docFreq()} .. {@link CollectionStatistics#sumTotalTermFreq()}].
+ * cannot exceed {@link FieldStats#sumDocFreq()}. @see TermsEnum#docFreq()
+ * @param totalTermFreq number of occurrences of the term in the field, in the range [{@link
+ * #docFreq()} .. {@link FieldStats#sumTotalTermFreq()}].
* <p>This is the token count for the term: the number of times it appears in the field across
* all documents.
* <p>This value is always a positive number, always at least {@link #docFreq()}, and never
- * exceeds {@link CollectionStatistics#sumTotalTermFreq()}. @see TermsEnum#totalTermFreq()
+ * exceeds {@link FieldStats#sumTotalTermFreq()}. @see TermsEnum#totalTermFreq()
* @lucene.experimental
*/
-// TODO: actually add missing cross-checks to guarantee TermStatistics is in bounds of
-// CollectionStatistics,
-// otherwise many similarity functions will implode.
-public record TermStatistics(BytesRef term, long docFreq, long totalTermFreq) {
+// TODO: actually add missing cross-checks to guarantee TermStats is in bounds of
+// FieldStats, otherwise many similarity functions will implode.
+public record TermStats(BytesRef term, long docFreq, long totalTermFreq) {
/**
* Creates statistics instance for a term.
*
@@ -70,7 +69,7 @@ public record TermStatistics(BytesRef term, long docFreq, long totalTermFreq) {
* @throws IllegalArgumentException if {@code docFreq} is negative or zero.
* @throws IllegalArgumentException if {@code totalTermFreq} is less than {@code docFreq}.
*/
- public TermStatistics {
+ public TermStats {
Objects.requireNonNull(term);
if (docFreq <= 0) {
throw new IllegalArgumentException("docFreq must be positive, docFreq: " + docFreq);
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
index e2a86481aeb..3264b265bdd 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.SmallFloat;
@@ -141,8 +141,8 @@ public class BM25Similarity extends Similarity {
}
/** The default implementation computes the average as <code>sumTotalTermFreq / docCount</code> */
- protected float avgFieldLength(CollectionStatistics collectionStats) {
- return (float) (collectionStats.sumTotalTermFreq() / (double) collectionStats.docCount());
+ protected float avgFieldLength(FieldStats fieldStats) {
+ return (float) (fieldStats.sumTotalTermFreq() / (double) fieldStats.docCount());
}
/** Cache of decoded bytes. */
@@ -163,20 +163,20 @@ public class BM25Similarity extends Similarity {
* idf(docFreq, docCount);
* </code></pre>
*
- * Note that {@link CollectionStatistics#docCount()} is used instead of {@link
+ * Note that {@link FieldStats#docCount()} is used instead of {@link
* org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also {@link
- * TermStatistics#docFreq()} is used, and when the latter is inaccurate, so is {@link
- * CollectionStatistics#docCount()}, and in the same direction. In addition, {@link
- * CollectionStatistics#docCount()} does not skew when fields are sparse.
+ * TermStats#docFreq()} is used, and when the latter is inaccurate, so is {@link
+ * FieldStats#docCount()}, and in the same direction. In addition, {@link FieldStats#docCount()}
+ * does not skew when fields are sparse.
*
- * @param collectionStats collection-level statistics
+ * @param fieldStats field-level statistics
* @param termStats term-level statistics for the term
* @return an Explain object that includes both an idf score factor and an explanation for the
* term.
*/
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {
final long df = termStats.docFreq();
- final long docCount = collectionStats.docCount();
+ final long docCount = fieldStats.docCount();
final float idf = idf(df, docCount);
return Explanation.match(
idf,
@@ -190,16 +190,16 @@ public class BM25Similarity extends Similarity {
*
* <p>The default implementation sums the idf factor for each term in the phrase.
*
- * @param collectionStats collection-level statistics
+ * @param fieldStats field-level statistics
* @param termStats term-level statistics for the terms in the phrase
* @return an Explain object that includes both an idf score factor for the phrase and an
* explanation for each term.
*/
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {
double idf = 0d; // sum into a double before casting into a float
List<Explanation> details = new ArrayList<>();
- for (final TermStatistics stat : termStats) {
- Explanation idfExplain = idfExplain(collectionStats, stat);
+ for (final TermStats stat : termStats) {
+ Explanation idfExplain = idfExplain(fieldStats, stat);
details.add(idfExplain);
idf += idfExplain.getValue().floatValue();
}
@@ -207,13 +207,12 @@ public class BM25Similarity extends Similarity {
}
@Override
- public final SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
Explanation idf =
termStats.length == 1
- ? idfExplain(collectionStats, termStats[0])
- : idfExplain(collectionStats, termStats);
- float avgdl = avgFieldLength(collectionStats);
+ ? idfExplain(fieldStats, termStats[0])
+ : idfExplain(fieldStats, termStats);
+ float avgdl = avgFieldLength(fieldStats);
float[] cache = new float[256];
for (int i = 0; i < cache.length; i++) {
@@ -222,7 +221,7 @@ public class BM25Similarity extends Similarity {
return new BM25Scorer(boost, k1, b, idf, avgdl, cache);
}
- /** Collection statistics for the BM25 model. */
+ /** Field statistics for the BM25 model. */
private static class BM25Scorer extends SimScorer {
/** query boost */
private final float boost;
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
index db28e845448..3f3bf798b11 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BooleanSimilarity.java
@@ -16,9 +16,9 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/**
* Simple similarity that gives terms a score that is equal to their query boost. This similarity is
@@ -34,8 +34,7 @@ public class BooleanSimilarity extends Similarity {
public BooleanSimilarity() {}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
return new BooleanWeight(boost);
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
index 5fad406c585..bcb93d7f07f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/ClassicSimilarity.java
@@ -16,9 +16,9 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/**
* Expert: Historical scoring implementation. You might want to consider using {@link
@@ -53,9 +53,9 @@ public class ClassicSimilarity extends TFIDFSimilarity {
}
@Override
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {
final long df = termStats.docFreq();
- final long docCount = collectionStats.docCount();
+ final long docCount = fieldStats.docCount();
final float idf = idf(df, docCount);
return Explanation.match(
idf,
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
index 5bd48f37a34..b7a3f395dc7 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;
import java.util.List;
import java.util.Locale;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/**
* Abstract superclass for language modeling Similarities. The following inner types are introduced:
@@ -66,9 +66,8 @@ public abstract class LMSimilarity extends SimilarityBase {
* Computes the collection probability of the current term in addition to the usual statistics.
*/
@Override
- protected void fillBasicStats(
- BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
- super.fillBasicStats(stats, collectionStats, termStats);
+ protected void fillBasicStats(BasicStats stats, FieldStats fieldStats, TermStats termStats) {
+ super.fillBasicStats(stats, fieldStats, termStats);
LMStats lmStats = (LMStats) stats;
lmStats.setCollectionProbability(collectionModel.computeProbability(stats));
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
index 22dea01d6f6..7de0e018028 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
@@ -19,9 +19,9 @@ package org.apache.lucene.search.similarities;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/**
* Implements the CombSUM method for combining evidence from multiple similarity values described
@@ -44,11 +44,10 @@ public class MultiSimilarity extends Similarity {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
SimScorer[] subScorers = new SimScorer[sims.length];
for (int i = 0; i < subScorers.length; i++) {
- subScorers[i] = sims[i].scorer(boost, collectionStats, termStats);
+ subScorers[i] = sims[i].scorer(boost, fieldStats, termStats);
}
return new MultiSimScorer(subScorers);
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
index 2107b151031..3689787e114 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/PerFieldSimilarityWrapper.java
@@ -17,8 +17,8 @@
package org.apache.lucene.search.similarities;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/**
* Provides the ability to use a different {@link Similarity} for different fields.
@@ -39,9 +39,8 @@ public abstract class PerFieldSimilarityWrapper extends Similarity {
}
@Override
- public final SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return get(collectionStats.field()).scorer(boost, collectionStats, termStats);
+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
+ return get(fieldStats.field()).scorer(boost, fieldStats, termStats);
}
/** Returns a {@link Similarity} for scoring a field. */
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java
index 043d831dbe1..508bd54f621 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/RawTFSimilarity.java
@@ -16,8 +16,8 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
/** Similarity that returns the raw TF as score. */
public class RawTFSimilarity extends Similarity {
@@ -33,8 +33,7 @@ public class RawTFSimilarity extends Similarity {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
index dc39ac3f321..dad5da0ce7d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java
@@ -22,10 +22,10 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.util.SmallFloat;
/**
@@ -52,8 +52,7 @@ import org.apache.lucene.util.SmallFloat;
* single byte, this might not be suitable for all purposes.
*
* <p>Many formulas require the use of average document length, which can be computed via a
- * combination of {@link CollectionStatistics#sumTotalTermFreq()} and {@link
- * CollectionStatistics#docCount()}.
+ * combination of {@link FieldStats#sumTotalTermFreq()} and {@link FieldStats#docCount()}.
*
* <p>Additional scoring factors can be stored in named {@link NumericDocValuesField}s and accessed
* at query-time with {@link org.apache.lucene.index.LeafReader#getNumericDocValues(String)}.
@@ -70,13 +69,12 @@ import org.apache.lucene.util.SmallFloat;
* steps:
*
* <ol>
- * <li>The {@link #scorer(float, CollectionStatistics, TermStatistics...)} method is called a
- * single time, allowing the implementation to compute any statistics (such as IDF, average
- * document length, etc) across <i>the entire collection</i>. The {@link TermStatistics} and
- * {@link CollectionStatistics} passed in already contain all of the raw statistics involved,
- * so a Similarity can freely use any combination of statistics without causing any additional
- * I/O. Lucene makes no assumption about what is stored in the returned {@link
- * Similarity.SimScorer} object.
+ * <li>The {@link #scorer(float, FieldStats, TermStats...)} method is called a single time,
+ * allowing the implementation to compute any statistics (such as IDF, average document
+ * length, etc) across <i>the entire field</i>. The {@link TermStats} and {@link FieldStats}
+ * passed in already contain all of the raw statistics involved, so a Similarity can freely
+ * use any combination of statistics without causing any additional I/O. Lucene makes no
+ * assumption about what is stored in the returned {@link Similarity.SimScorer} object.
* <li>Then {@link SimScorer#score(float, long)} is called for every matching document to compute
* its score.
* </ol>
@@ -179,23 +177,21 @@ public abstract class Similarity {
}
/**
- * Compute any collection-level weight (e.g. IDF, average document length, etc) needed for scoring
- * a query.
+ * Compute any field-level weight (e.g. IDF, average document length, etc) needed for scoring a
+ * query.
*
* @param boost a multiplicative factor to apply to the produces scores
- * @param collectionStats collection-level statistics, such as the number of tokens in the
- * collection.
+ * @param fieldStats field-level statistics, such as the number of tokens in the field.
* @param termStats term-level statistics, such as the document frequency of a term across the
- * collection.
+ * field.
* @return SimWeight object with the information this Similarity needs to score a query.
*/
- public abstract SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats);
+ public abstract SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats);
/**
- * Stores the weight for a query across the indexed collection. This abstract implementation is
- * empty; descendants of {@code Similarity} should subclass {@code SimWeight} and define the
- * statistics they require in the subclass. Examples include idf, average field length, etc.
+ * Stores the weight for a query across the indexed field. This abstract implementation is empty;
+ * descendants of {@code Similarity} should subclass {@code SimWeight} and define the statistics
+ * they require in the subclass. Examples include idf, average field length, etc.
*/
public abstract static class SimScorer {
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
index af63b7bebeb..7ec2f50ff95 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
@@ -18,9 +18,9 @@ package org.apache.lucene.search.similarities;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.util.SmallFloat;
/**
@@ -52,12 +52,11 @@ public abstract class SimilarityBase extends Similarity {
}
@Override
- public final SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
SimScorer[] weights = new SimScorer[termStats.length];
for (int i = 0; i < termStats.length; i++) {
- BasicStats stats = newStats(collectionStats.field(), boost);
- fillBasicStats(stats, collectionStats, termStats[i]);
+ BasicStats stats = newStats(fieldStats.field(), boost);
+ fillBasicStats(stats, fieldStats, termStats[i]);
weights[i] = new BasicSimScorer(stats);
}
if (weights.length == 1) {
@@ -76,17 +75,15 @@ public abstract class SimilarityBase extends Similarity {
* Fills all member fields defined in {@code BasicStats} in {@code stats}. Subclasses can override
* this method to fill additional stats.
*/
- protected void fillBasicStats(
- BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
+ protected void fillBasicStats(BasicStats stats, FieldStats fieldStats, TermStats termStats) {
// TODO: validate this for real, somewhere else
- assert termStats.totalTermFreq() <= collectionStats.sumTotalTermFreq();
- assert termStats.docFreq() <= collectionStats.sumDocFreq();
+ assert termStats.totalTermFreq() <= fieldStats.sumTotalTermFreq();
+ assert termStats.docFreq() <= fieldStats.sumDocFreq();
// TODO: add sumDocFreq for field (numberOfFieldPostings)
- stats.setNumberOfDocuments(collectionStats.docCount());
- stats.setNumberOfFieldTokens(collectionStats.sumTotalTermFreq());
- stats.setAvgFieldLength(
- collectionStats.sumTotalTermFreq() / (double) collectionStats.docCount());
+ stats.setNumberOfDocuments(fieldStats.docCount());
+ stats.setNumberOfFieldTokens(fieldStats.sumTotalTermFreq());
+ stats.setAvgFieldLength(fieldStats.sumTotalTermFreq() / (double) fieldStats.docCount());
stats.setDocFreq(termStats.docFreq());
stats.setTotalTermFreq(termStats.totalTermFreq());
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
index 47385de6583..04b18870e1f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
@@ -18,10 +18,10 @@ package org.apache.lucene.search.similarities;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.util.SmallFloat;
/**
@@ -357,20 +357,20 @@ public abstract class TFIDFSimilarity extends Similarity {
* idf(docFreq, docCount);
* </code></pre>
*
- * Note that {@link CollectionStatistics#docCount()} is used instead of {@link
+ * Note that {@link FieldStats#docCount()} is used instead of {@link
* org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also {@link
- * TermStatistics#docFreq()} is used, and when the latter is inaccurate, so is {@link
- * CollectionStatistics#docCount()}, and in the same direction. In addition, {@link
- * CollectionStatistics#docCount()} does not skew when fields are sparse.
+ * TermStats#docFreq()} is used, and when the latter is inaccurate, so is {@link
+ * FieldStats#docCount()}, and in the same direction. In addition, {@link FieldStats#docCount()}
+ * does not skew when fields are sparse.
*
- * @param collectionStats collection-level statistics
+ * @param fieldStats collection-level statistics
* @param termStats term-level statistics for the term
* @return an Explain object that includes both an idf score factor and an explanation for the
* term.
*/
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {
final long df = termStats.docFreq();
- final long docCount = collectionStats.docCount();
+ final long docCount = fieldStats.docCount();
final float idf = idf(df, docCount);
return Explanation.match(
idf,
@@ -384,16 +384,16 @@ public abstract class TFIDFSimilarity extends Similarity {
*
* <p>The default implementation sums the idf factor for each term in the phrase.
*
- * @param collectionStats collection-level statistics
+ * @param fieldStats collection-level statistics
* @param termStats term-level statistics for the terms in the phrase
* @return an Explain object that includes both an idf score factor for the phrase and an
* explanation for each term.
*/
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {
double idf = 0d; // sum into a double before casting into a float
List<Explanation> subs = new ArrayList<>();
- for (final TermStatistics stat : termStats) {
- Explanation idfExplain = idfExplain(collectionStats, stat);
+ for (final TermStats stat : termStats) {
+ Explanation idfExplain = idfExplain(fieldStats, stat);
subs.add(idfExplain);
idf += idfExplain.getValue().floatValue();
}
@@ -433,12 +433,11 @@ public abstract class TFIDFSimilarity extends Similarity {
}
@Override
- public final SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public final SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
final Explanation idf =
termStats.length == 1
- ? idfExplain(collectionStats, termStats[0])
- : idfExplain(collectionStats, termStats);
+ ? idfExplain(fieldStats, termStats[0])
+ : idfExplain(fieldStats, termStats);
float[] normTable = new float[256];
for (int i = 1; i < 256; ++i) {
float norm = lengthNorm(LENGTH_TABLE[i]);
@@ -448,10 +447,7 @@ public abstract class TFIDFSimilarity extends Similarity {
return new TFIDFScorer(boost, idf, normTable);
}
- /**
- * Collection statistics for the TF-IDF model. The only statistic of interest to this model is
- * idf.
- */
+ /** Field statistics for the TF-IDF model. The only statistic of interest to this model is idf. */
class TFIDFScorer extends SimScorer {
/** The idf and its explanation */
private final Explanation idf;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
index 705c905012b..9a40aab030b 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomNorms.java
@@ -22,8 +22,8 @@ import java.util.stream.IntStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
@@ -105,8 +105,7 @@ public class TestCustomNorms extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
index e42f3387630..fe76b2b3af0 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCustomTermFreq.java
@@ -26,8 +26,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -493,8 +493,7 @@ public class TestCustomTermFreq extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
index 378388d86cb..eda73d9ae82 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldInvertState.java
@@ -22,8 +22,8 @@ import java.util.Map;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.CannedTokenStream;
@@ -50,8 +50,7 @@ public class TestFieldInvertState extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 09d097a56c2..123d211f6db 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -63,9 +63,9 @@ import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
@@ -77,7 +77,7 @@ import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.search.SortedSetSelector;
import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldCollectorManager;
import org.apache.lucene.search.similarities.Similarity;
@@ -2295,9 +2295,8 @@ public class TestIndexSorting extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return in.scorer(boost, collectionStats, termStats);
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
+ return in.scorer(boost, fieldStats, termStats);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
index 7715045549c..55d92cf0963 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
@@ -22,8 +22,8 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -102,8 +102,7 @@ public class TestMaxTermFrequency extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
return new SimScorer() {
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
index e5a887a8821..5a25b18f592 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNorms.java
@@ -24,8 +24,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
@@ -108,8 +108,7 @@ public class TestNorms extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
index 2b59333c5ee..d236f091b9f 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
@@ -25,16 +25,16 @@ import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.CollectorManager;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -61,8 +61,7 @@ public class TestOmitTf extends LuceneTestCase {
}
@Override
- public Explanation idfExplain(
- CollectionStatistics collectionStats, TermStatistics[] termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {
return Explanation.match(1.0f, "Inexplicable");
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
index 234ea194779..2d3b619a4a5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestUniqueTermCount.java
@@ -20,8 +20,8 @@ import java.util.ArrayList;
import java.util.HashSet;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -100,8 +100,7 @@ public class TestUniqueTermCount extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
index df30c5ee005..3b68c094a99 100644
--- a/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java
@@ -188,8 +188,7 @@ final class JustCompileSearch {
static final class JustCompileSimilarity extends Similarity {
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java
index 325d1344d62..f32627af784 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java
@@ -480,13 +480,12 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new BM25Similarity().scorer(boost, collectionStats, termStats);
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
+ return new BM25Similarity().scorer(boost, fieldStats, termStats);
}
}
- public void testOverrideCollectionStatistics() throws IOException {
+ public void testOverrideFieldStats() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig();
Similarity similarity = randomCompatibleSimilarity();
@@ -533,8 +532,8 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
IndexSearcher searcher =
new IndexSearcher(reader) {
@Override
- public CollectionStatistics collectionStatistics(String field) throws IOException {
- CollectionStatistics shardStatistics = super.collectionStatistics(field);
+ public FieldStats fieldStats(String field) throws IOException {
+ FieldStats shardStatistics = super.fieldStats(field);
int extraSumTotalTermFreq;
if (field.equals("a")) {
extraSumTotalTermFreq = extraSumTotalTermFreqA;
@@ -545,7 +544,7 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
} else {
throw new AssertionError("should never be called");
}
- return new CollectionStatistics(
+ return new FieldStats(
field,
shardStatistics.maxDoc() + extraMaxDoc,
shardStatistics.docCount() + extraDocCount,
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
index e06951df164..0a1269dd8c7 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
@@ -371,8 +371,8 @@ public class TestMinShouldMatch2 extends LuceneTestCase {
sims[(int) ord] =
weight.similarity.scorer(
1f,
- searcher.collectionStatistics("field"),
- searcher.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));
+ searcher.fieldStats("field"),
+ searcher.termStats(term, ts.docFreq(), ts.totalTermFreq()));
}
}
norms = reader.getNormValues("field");
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
index 1916efeee0f..c178e65c78d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
@@ -49,7 +49,7 @@ public class TestSimilarity extends LuceneTestCase {
}
@Override
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] stats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] stats) {
return Explanation.match(1.0f, "Inexplicable");
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
index afdbef97b49..30580f18c78 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestSimilarityProvider.java
@@ -109,8 +109,7 @@ public class TestSimilarityProvider extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
return new SimScorer() {
@Override
@@ -129,8 +128,7 @@ public class TestSimilarityProvider extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
return new SimScorer() {
@Override
public float score(float freq, long norm) {
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
index f87acb83bdf..33888b3a902 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTermQuery.java
@@ -194,10 +194,9 @@ public class TestTermQuery extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
scorerCalled.set(true);
- return existingSimilarity.scorer(boost, collectionStats, termStats);
+ return existingSimilarity.scorer(boost, fieldStats, termStats);
}
});
TermQuery termQuery =
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
index 3ec08d75f85..be7c18e3919 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestClassicSimilarity.java
@@ -157,7 +157,7 @@ public class TestClassicSimilarity extends BaseSimilarityTestCase {
public void testSaneNormValues() throws IOException {
ClassicSimilarity sim = new ClassicSimilarity();
TFIDFSimilarity.TFIDFScorer stats =
- (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.collectionStatistics("test"));
+ (TFIDFSimilarity.TFIDFScorer) sim.scorer(1f, indexSearcher.fieldStats("test"));
for (int i = 0; i < 256; i++) {
float boost = stats.normTable[i];
assertFalse("negative boost: " + boost + ", byte=" + i, boost < 0.0f);
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
index af2cf4c8268..0083069001f 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
@@ -27,12 +27,12 @@ import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.SimilarityBase.BasicSimScorer;
import org.apache.lucene.store.Directory;
@@ -187,7 +187,7 @@ public class TestSimilarityBase extends LuceneTestCase {
return stats;
}
- private CollectionStatistics toCollectionStats(BasicStats stats) {
+ private FieldStats toFieldStats(BasicStats stats) {
long sumTtf = stats.getNumberOfFieldTokens();
long sumDf;
if (sumTtf == -1) {
@@ -200,12 +200,11 @@ public class TestSimilarityBase extends LuceneTestCase {
int docCount = Math.toIntExact(Math.min(sumDf, stats.getNumberOfDocuments()));
int maxDoc = TestUtil.nextInt(random(), docCount, docCount + 10);
- return new CollectionStatistics(stats.field, maxDoc, docCount, sumTtf, sumDf);
+ return new FieldStats(stats.field, maxDoc, docCount, sumTtf, sumDf);
}
- private TermStatistics toTermStats(BasicStats stats) {
- return new TermStatistics(
- new BytesRef("spoofyText"), stats.getDocFreq(), stats.getTotalTermFreq());
+ private TermStats toTermStats(BasicStats stats) {
+ return new TermStats(new BytesRef("spoofyText"), stats.getDocFreq(), stats.getTotalTermFreq());
}
/**
@@ -217,8 +216,7 @@ public class TestSimilarityBase extends LuceneTestCase {
for (SimilarityBase sim : sims) {
BasicStats realStats =
((BasicSimScorer)
- sim.scorer(
- (float) stats.getBoost(), toCollectionStats(stats), toTermStats(stats)))
+ sim.scorer((float) stats.getBoost(), toFieldStats(stats), toTermStats(stats)))
.stats;
float score = (float) sim.score(realStats, freq, docLen);
float explScore =
@@ -471,7 +469,7 @@ public class TestSimilarityBase extends LuceneTestCase {
BasicStats stats = createStats();
BasicStats realStats =
((BasicSimScorer)
- sim.scorer((float) stats.getBoost(), toCollectionStats(stats), toTermStats(stats)))
+ sim.scorer((float) stats.getBoost(), toFieldStats(stats), toTermStats(stats)))
.stats;
float score = (float) sim.score(realStats, FREQ, DOC_LEN);
assertEquals(sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON);
diff --git a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
index 1a98699bb7b..0c6782d671f 100644
--- a/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
+++ b/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
@@ -74,8 +74,8 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.KnnByteVectorQuery;
import org.apache.lucene.search.KnnFloatVectorQuery;
@@ -83,7 +83,7 @@ import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.search.similarities.BM25Similarity;
@@ -217,8 +217,7 @@ public class TestMemoryIndex extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
});
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
index 4373d77d381..cf180d19c4a 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
@@ -23,9 +23,9 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
-import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.util.BytesRef;
@@ -73,9 +73,7 @@ public class NormValueSource extends ValueSource {
// is 1 when docCount == docFreq == 1
final SimScorer simScorer =
similarity.scorer(
- 1f,
- new CollectionStatistics(field, 1, 1, 1, 1),
- new TermStatistics(new BytesRef("bogus"), 1, 1));
+ 1f, new FieldStats(field, 1, 1, 1, 1), new TermStats(new BytesRef("bogus"), 1, 1));
final NumericDocValues norms = readerContext.reader().getNormValues(field);
return new FloatDocValues(this) {
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java
index 554a3669fc1..9916c4e80ef 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/spans/FieldMaskingSpanQuery.java
@@ -61,8 +61,8 @@ import org.apache.lucene.search.ScoreMode;
* teacherid 2 (which has a 'james' in position 0 and 'jones' in position 1).
*
* <p>Note: as {@link #getField()} returns the masked field, scoring will be done using the
- * Similarity and collection statistics of the field name supplied, but with the term statistics of
- * the real field. This may lead to exceptions, poor performance, and unexpected scoring behaviour.
+ * Similarity and field statistics of the field name supplied, but with the term statistics of the
+ * real field. This may lead to exceptions, poor performance, and unexpected scoring behavior.
*/
public final class FieldMaskingSpanQuery extends SpanQuery {
private final SpanQuery maskedQuery;
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java b/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java
index 8c7d8dbdb3a..4d130dca258 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/spans/SpanWeight.java
@@ -26,8 +26,8 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermStates;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Matches;
import org.apache.lucene.search.MatchesIterator;
@@ -35,7 +35,7 @@ import org.apache.lucene.search.MatchesUtils;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
@@ -102,19 +102,18 @@ public abstract class SpanWeight extends Weight {
SpanQuery query, IndexSearcher searcher, Map<Term, TermStates> termStates, float boost)
throws IOException {
if (termStates == null || termStates.size() == 0 || query.getField() == null) return null;
- TermStatistics[] termStats = new TermStatistics[termStates.size()];
+ TermStats[] termStats = new TermStats[termStates.size()];
int termUpTo = 0;
for (Map.Entry<Term, TermStates> entry : termStates.entrySet()) {
TermStates ts = entry.getValue();
if (ts.docFreq() > 0) {
termStats[termUpTo++] =
- searcher.termStatistics(entry.getKey(), ts.docFreq(), ts.totalTermFreq());
+ searcher.termStats(entry.getKey(), ts.docFreq(), ts.totalTermFreq());
}
}
- CollectionStatistics collectionStats = searcher.collectionStatistics(query.getField());
+ FieldStats fieldStats = searcher.fieldStats(query.getField());
if (termUpTo > 0) {
- return similarity.scorer(
- boost, collectionStats, ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));
+ return similarity.scorer(boost, fieldStats, ArrayUtil.copyOfSubArray(termStats, 0, termUpTo));
} else {
return null; // no terms at all exist, we won't use similarity
}
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
index cc5f16bfe9e..76f120bed67 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/payloads/TestPayloadScoreQuery.java
@@ -33,10 +33,10 @@ import org.apache.lucene.queries.spans.SpanNearQuery;
import org.apache.lucene.queries.spans.SpanOrQuery;
import org.apache.lucene.queries.spans.SpanQuery;
import org.apache.lucene.queries.spans.SpanTermQuery;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.similarities.ClassicSimilarity;
@@ -367,13 +367,12 @@ public class TestPayloadScoreQuery extends LuceneTestCase {
// idf used for phrase queries
@Override
- public Explanation idfExplain(
- CollectionStatistics collectionStats, TermStatistics[] termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats[] termStats) {
return Explanation.match(1.0f, "Inexplicable");
}
@Override
- public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ public Explanation idfExplain(FieldStats fieldStats, TermStats termStats) {
return Explanation.match(1.0f, "Inexplicable");
}
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
index debd10ea83d..b595790a827 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
@@ -38,11 +38,11 @@ import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.DisiWrapper;
import org.apache.lucene.search.DisjunctionDISIApproximation;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Matches;
import org.apache.lucene.search.Query;
@@ -52,7 +52,7 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermScorer;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.DFRSimilarity;
@@ -72,8 +72,8 @@ import org.apache.lucene.util.SmallFloat;
*
* <ol>
* <li>Given a list of fields and weights, it pretends there is a synthetic combined field where
- * all terms have been indexed. It computes new term and collection statistics for this
- * combined field.
+ * all terms have been indexed. It computes new term and field statistics for this combined
+ * field.
* <li>It uses a disjunction iterator and {@link IndexSearcher#getSimilarity} to score documents.
* </ol>
*
@@ -314,41 +314,37 @@ public final class CombinedFieldQuery extends Query implements Accountable {
TermStates ts = TermStates.build(searcher, fieldTerms[i], true);
termStates[i] = ts;
if (ts.docFreq() > 0) {
- TermStatistics termStats =
- searcher.termStatistics(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());
+ TermStats termStats = searcher.termStats(fieldTerms[i], ts.docFreq(), ts.totalTermFreq());
docFreq = Math.max(termStats.docFreq(), docFreq);
totalTermFreq += (double) field.weight * termStats.totalTermFreq();
}
}
if (docFreq > 0) {
- CollectionStatistics pseudoCollectionStats = mergeCollectionStatistics(searcher);
- TermStatistics pseudoTermStatistics =
- new TermStatistics(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));
- this.simWeight =
- searcher.getSimilarity().scorer(boost, pseudoCollectionStats, pseudoTermStatistics);
+ FieldStats pseudoFieldStats = mergeFieldStats(searcher);
+ TermStats pseudoTermStats =
+ new TermStats(new BytesRef("pseudo_term"), docFreq, Math.max(1, totalTermFreq));
+ this.simWeight = searcher.getSimilarity().scorer(boost, pseudoFieldStats, pseudoTermStats);
} else {
this.simWeight = null;
}
}
- private CollectionStatistics mergeCollectionStatistics(IndexSearcher searcher)
- throws IOException {
+ private FieldStats mergeFieldStats(IndexSearcher searcher) throws IOException {
long maxDoc = 0;
long docCount = 0;
long sumTotalTermFreq = 0;
long sumDocFreq = 0;
for (FieldAndWeight fieldWeight : fieldAndWeights.values()) {
- CollectionStatistics collectionStats = searcher.collectionStatistics(fieldWeight.field);
- if (collectionStats != null) {
- maxDoc = Math.max(collectionStats.maxDoc(), maxDoc);
- docCount = Math.max(collectionStats.docCount(), docCount);
- sumDocFreq = Math.max(collectionStats.sumDocFreq(), sumDocFreq);
- sumTotalTermFreq += (double) fieldWeight.weight * collectionStats.sumTotalTermFreq();
+ FieldStats fieldStats = searcher.fieldStats(fieldWeight.field);
+ if (fieldStats != null) {
+ maxDoc = Math.max(fieldStats.maxDoc(), maxDoc);
+ docCount = Math.max(fieldStats.docCount(), docCount);
+ sumDocFreq = Math.max(fieldStats.sumDocFreq(), sumDocFreq);
+ sumTotalTermFreq += (double) fieldWeight.weight * fieldStats.sumTotalTermFreq();
}
}
- return new CollectionStatistics(
- "pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
+ return new FieldStats("pseudo_field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java
index 218b2048f0f..9b86fba1125 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/PhraseWildcardQuery.java
@@ -51,7 +51,6 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.SloppyPhraseMatcher;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.ArrayUtil;
@@ -153,7 +152,7 @@ public class PhraseWildcardQuery extends Query {
List<LeafReaderContext> sizeSortedSegments =
new SegmentTermsSizeComparator().createTermsSizeSortedCopyOf(reader.leaves());
- // TermsData will contain the collected TermState and TermStatistics for all the terms
+ // TermsData will contain the collected TermState and TermStats for all the terms
// of the phrase. It is filled during PhraseTerm.collectTermData() calls below.
TermsData termsData = createTermsData(sizeSortedSegments.size());
@@ -255,8 +254,8 @@ public class PhraseWildcardQuery extends Query {
.getSimilarity()
.scorer(
boost,
- searcher.collectionStatistics(field),
- termsData.termStatsList.toArray(new TermStatistics[0]));
+ searcher.fieldStats(field),
+ termsData.termStatsList.toArray(new org.apache.lucene.search.TermStats[0]));
}
@Override
@@ -362,7 +361,8 @@ public class PhraseWildcardQuery extends Query {
}
/**
- * Collects the {@link TermState} and {@link TermStatistics} for a single-term without expansion.
+ * Collects the {@link TermState} and {@link org.apache.lucene.search.TermStats} for a single-term
+ * without expansion.
*
* @param termsData receives the collected data.
*/
@@ -406,13 +406,14 @@ public class PhraseWildcardQuery extends Query {
// Collect the term stats across all segments.
if (termStates.docFreq() > 0) {
termsData.termStatsList.add(
- searcher.termStatistics(term, termStates.docFreq(), termStates.totalTermFreq()));
+ searcher.termStats(term, termStates.docFreq(), termStates.totalTermFreq()));
}
return numMatches;
}
/**
- * Collects the {@link TermState} and {@link TermStatistics} for a multi-term with expansion.
+ * Collects the {@link TermState} and {@link org.apache.lucene.search.TermStats} for a multi-term
+ * with expansion.
*
* @param remainingMultiTerms the number of remaining multi-terms to process, including the
* current one, excluding the multi-terms already processed.
@@ -473,8 +474,8 @@ public class PhraseWildcardQuery extends Query {
}
/**
- * Collects the {@link TermState} list and {@link TermStatistics} for a multi-term on a specific
- * index segment.
+ * Collects the {@link TermState} list and {@link org.apache.lucene.search.TermStats} for a
+ * multi-term on a specific index segment.
*
* @param remainingExpansions the number of remaining expansions allowed for the segment.
* @param shouldStopSegmentIteration to be set to true to stop the segment iteration calling this
@@ -537,7 +538,8 @@ public class PhraseWildcardQuery extends Query {
* Collect the term stats across all segments.
*
* @param termStatsMap input map of already collected {@link TermStats}.
- * @param termsData receives the {@link TermStatistics} computed for all {@link TermStats}.
+ * @param termsData receives the {@link org.apache.lucene.search.TermStats} computed for all
+ * {@link TermStats}.
* @param termData receives all the collected {@link Term}.
*/
protected void collectMultiTermStats(
@@ -555,7 +557,7 @@ public class PhraseWildcardQuery extends Query {
TermStats termStats = termStatsEntry.getValue();
if (termStats.docFreq > 0) {
termsData.termStatsList.add(
- searcher.termStatistics(term, termStats.docFreq, termStats.totalTermFreq));
+ searcher.termStats(term, termStats.docFreq, termStats.totalTermFreq));
}
}
}
@@ -689,10 +691,11 @@ public class PhraseWildcardQuery extends Query {
protected abstract Query getQuery();
/**
- * Collects {@link TermState} and {@link TermStatistics} for the term without expansion. It must
- * be called only if {@link #hasExpansions()} returns false. Simplified version of {@code
- * #collectTermData(PhraseWildcardQuery, IndexSearcher, List, int, int, TermsData)} with less
- * arguments. This method throws {@link UnsupportedOperationException} if not overridden.
+ * Collects {@link TermState} and {@link org.apache.lucene.search.TermStats} for the term
+ * without expansion. It must be called only if {@link #hasExpansions()} returns false.
+ * Simplified version of {@code #collectTermData(PhraseWildcardQuery, IndexSearcher, List, int,
+ * int, TermsData)} with less arguments. This method throws {@link
+ * UnsupportedOperationException} if not overridden.
*/
protected int collectTermData(
PhraseWildcardQuery query,
@@ -704,7 +707,8 @@ public class PhraseWildcardQuery extends Query {
}
/**
- * Collects {@link TermState} and {@link TermStatistics} for the term (potentially expanded).
+ * Collects {@link TermState} and {@link org.apache.lucene.search.TermStats} for the term
+ * (potentially expanded).
*
* @param termsData {@link TermsData} to update with the collected terms and stats.
* @return The number of expansions or matches in all segments; or 0 if this term does not match
@@ -842,14 +846,14 @@ public class PhraseWildcardQuery extends Query {
}
/**
- * Holds the {@link TermState} and {@link TermStatistics} for all the matched and collected {@link
- * Term}, for all phrase terms, for all segments.
+ * Holds the {@link TermState} and {@link org.apache.lucene.search.TermStats} for all the matched
+ * and collected {@link Term}, for all phrase terms, for all segments.
*/
protected static class TermsData {
protected final int numTerms;
protected final int numSegments;
- protected final List<TermStatistics> termStatsList;
+ protected final List<org.apache.lucene.search.TermStats> termStatsList;
protected final TermData[] termDataPerPosition;
protected int numTermsMatching;
@@ -885,14 +889,14 @@ public class PhraseWildcardQuery extends Query {
builder.append("numSegments=").append(numSegments);
builder.append(", termDataPerPosition=").append(Arrays.asList(termDataPerPosition));
builder.append(", termsStatsList=[");
- for (TermStatistics termStatistics : termStatsList) {
+ for (org.apache.lucene.search.TermStats termStats : termStatsList) {
builder
.append("{")
- .append(termStatistics.term().utf8ToString())
+ .append(termStats.term().utf8ToString())
.append(", ")
- .append(termStatistics.docFreq())
+ .append(termStats.docFreq())
.append(", ")
- .append(termStatistics.totalTermFreq())
+ .append(termStats.totalTermFreq())
.append("}");
}
builder.append("]");
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java
index cbde5bf0ba1..f77478bed93 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/TermAutomatonQuery.java
@@ -45,7 +45,7 @@ import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.ScorerSupplier;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Accountable;
@@ -376,14 +376,13 @@ public class TermAutomatonQuery extends Query implements Accountable {
this.automaton = automaton;
this.termStates = termStates;
this.similarity = searcher.getSimilarity();
- List<TermStatistics> allTermStats = new ArrayList<>();
+ List<TermStats> allTermStats = new ArrayList<>();
for (IntObjectHashMap.IntObjectCursor<BytesRef> ent : idToTerm) {
if (ent.value != null) {
TermStates ts = termStates.get(ent.key);
if (ts.docFreq() > 0) {
allTermStats.add(
- searcher.termStatistics(
- new Term(field, ent.value), ts.docFreq(), ts.totalTermFreq()));
+ searcher.termStats(new Term(field, ent.value), ts.docFreq(), ts.totalTermFreq()));
}
}
}
@@ -394,8 +393,8 @@ public class TermAutomatonQuery extends Query implements Accountable {
stats =
similarity.scorer(
boost,
- searcher.collectionStatistics(field),
- allTermStats.toArray(new TermStatistics[allTermStats.size()]));
+ searcher.fieldStats(field),
+ allTermStats.toArray(new TermStats[allTermStats.size()]));
}
}
diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java
index 7d8f9a5bdec..aff1df264d7 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/search/TestCombinedFieldQuery.java
@@ -31,13 +31,13 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.CollectorManager;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.TopScoreDocCollectorManager;
@@ -586,13 +586,12 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
- return new BM25Similarity().scorer(boost, collectionStats, termStats);
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
+ return new BM25Similarity().scorer(boost, fieldStats, termStats);
}
}
- public void testOverrideCollectionStatistics() throws IOException {
+ public void testOverrideFieldStats() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig();
Similarity similarity = randomCompatibleSimilarity();
@@ -639,8 +638,8 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
IndexSearcher searcher =
new IndexSearcher(reader) {
@Override
- public CollectionStatistics collectionStatistics(String field) throws IOException {
- CollectionStatistics shardStatistics = super.collectionStatistics(field);
+ public FieldStats fieldStats(String field) throws IOException {
+ FieldStats shardStatistics = super.fieldStats(field);
int extraSumTotalTermFreq;
if (field.equals("a")) {
extraSumTotalTermFreq = extraSumTotalTermFreqA;
@@ -651,7 +650,7 @@ public class TestCombinedFieldQuery extends LuceneTestCase {
} else {
throw new AssertionError("should never be called");
}
- return new CollectionStatistics(
+ return new FieldStats(
field,
shardStatistics.maxDoc() + extraMaxDoc,
shardStatistics.docCount() + extraDocCount,
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java
index 5c3f97db33c..a0d6c624ef4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BaseNormsFormatTestCase.java
@@ -40,9 +40,9 @@ import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
@@ -634,8 +634,7 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
throw new UnsupportedOperationException();
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java
index 7a3785d2b59..585cb715664 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/ShardSearchingTestBase.java
@@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermStates;
-import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.FieldStats;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
@@ -37,7 +37,7 @@ import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherLifetimeManager;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
@@ -101,7 +101,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
}
}
- // We share collection stats for these fields on each node
+ // We share field stats for these fields on each node
// reopen:
private final String[] fieldsToShare = new String[] {"body", "title"};
@@ -109,7 +109,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
// other nodes. This is just a mock (since it goes and
// directly updates all other nodes, in RAM)... in a real
// env this would hit the wire, sending version &
- // collection stats to all other nodes:
+ // field stats to all other nodes:
void broadcastNodeReopen(int nodeID, long version, IndexSearcher newSearcher) throws IOException {
if (VERBOSE) {
@@ -122,16 +122,16 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
+ newSearcher.getIndexReader().maxDoc());
}
- // Broadcast new collection stats for this node to all
+ // Broadcast new field stats for this node to all
// other nodes:
for (String field : fieldsToShare) {
- final CollectionStatistics stats = newSearcher.collectionStatistics(field);
+ final FieldStats stats = newSearcher.fieldStats(field);
if (stats != null) {
for (NodeState node : nodes) {
- // Don't put my own collection stats into the cache;
+ // Don't put my own field stats into the cache;
// we pull locally:
if (node.myNodeID != nodeID) {
- node.collectionStatsCache.put(new FieldAndShardVersion(nodeID, version, field), stats);
+ node.fieldStatsCache.put(new FieldAndShardVersion(nodeID, version, field), stats);
}
}
}
@@ -169,10 +169,10 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
// Mock: in a real env, this would hit the wire and get
// term stats from remote node
- Map<Term, TermStatistics> getNodeTermStats(Set<Term> terms, int nodeID, long version)
+ Map<Term, TermStats> getNodeTermStats(Set<Term> terms, int nodeID, long version)
throws IOException {
final NodeState node = nodes[nodeID];
- final Map<Term, TermStatistics> stats = new HashMap<>();
+ final Map<Term, TermStats> stats = new HashMap<>();
final IndexSearcher s = node.searchers.acquire(version);
if (s == null) {
throw new SearcherExpiredException("node=" + nodeID + " version=" + version);
@@ -181,7 +181,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
for (Term term : terms) {
final TermStates ts = TermStates.build(s, term, true);
if (ts.docFreq() > 0) {
- stats.put(term, s.termStatistics(term, ts.docFreq(), ts.totalTermFreq()));
+ stats.put(term, s.termStats(term, ts.docFreq(), ts.totalTermFreq()));
}
}
} finally {
@@ -204,10 +204,8 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
// local cache...? And still LRU otherwise (for the
// still-live searchers).
- private final Map<FieldAndShardVersion, CollectionStatistics> collectionStatsCache =
- new ConcurrentHashMap<>();
- private final Map<TermAndShardVersion, TermStatistics> termStatsCache =
- new ConcurrentHashMap<>();
+ private final Map<FieldAndShardVersion, FieldStats> fieldStatsCache = new ConcurrentHashMap<>();
+ private final Map<TermAndShardVersion, TermStats> termStatsCache = new ConcurrentHashMap<>();
/**
* Matches docs in the local shard but scores based on aggregated stats ("mock distributed
@@ -249,7 +247,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
}
}
if (missing.size() != 0) {
- for (Map.Entry<Term, TermStatistics> ent :
+ for (Map.Entry<Term, TermStats> ent :
getNodeTermStats(missing, nodeID, nodeVersions[nodeID]).entrySet()) {
if (ent.getValue() != null) {
final TermAndShardVersion key =
@@ -264,16 +262,15 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
}
@Override
- public TermStatistics termStatistics(Term term, int docFreq, long totalTermFreq)
- throws IOException {
+ public TermStats termStats(Term term, int docFreq, long totalTermFreq) throws IOException {
assert term != null;
long distributedDocFreq = 0;
long distributedTotalTermFreq = 0;
for (int nodeID = 0; nodeID < nodeVersions.length; nodeID++) {
- final TermStatistics subStats;
+ final TermStats subStats;
if (nodeID == myNodeID) {
- subStats = super.termStatistics(term, docFreq, totalTermFreq);
+ subStats = super.termStats(term, docFreq, totalTermFreq);
} else {
final TermAndShardVersion key =
new TermAndShardVersion(nodeID, nodeVersions[nodeID], term);
@@ -290,11 +287,11 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
distributedTotalTermFreq += nodeTotalTermFreq;
}
assert distributedDocFreq > 0;
- return new TermStatistics(term.bytes(), distributedDocFreq, distributedTotalTermFreq);
+ return new TermStats(term.bytes(), distributedDocFreq, distributedTotalTermFreq);
}
@Override
- public CollectionStatistics collectionStatistics(String field) throws IOException {
+ public FieldStats fieldStats(String field) throws IOException {
// TODO: we could compute this on init and cache,
// since we are re-inited whenever any nodes have a
// new reader
@@ -306,11 +303,11 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
for (int nodeID = 0; nodeID < nodeVersions.length; nodeID++) {
final FieldAndShardVersion key =
new FieldAndShardVersion(nodeID, nodeVersions[nodeID], field);
- final CollectionStatistics nodeStats;
+ final FieldStats nodeStats;
if (nodeID == myNodeID) {
- nodeStats = super.collectionStatistics(field);
+ nodeStats = super.fieldStats(field);
} else {
- nodeStats = collectionStatsCache.get(key);
+ nodeStats = fieldStatsCache.get(key);
}
if (nodeStats == null) {
continue; // field not in sub at all
@@ -332,7 +329,7 @@ public abstract class ShardSearchingTestBase extends LuceneTestCase {
if (maxDoc == 0) {
return null; // field not found across any node whatsoever
} else {
- return new CollectionStatistics(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
+ return new FieldStats(field, maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java
index 8943c8f7a89..fe08c39f8c4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/AssertingSimilarity.java
@@ -17,9 +17,9 @@
package org.apache.lucene.tests.search.similarities;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.Similarity;
/** wraps a similarity with checks for testing */
@@ -48,16 +48,15 @@ public class AssertingSimilarity extends Similarity {
}
@Override
- public SimScorer scorer(
- float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ public SimScorer scorer(float boost, FieldStats fieldStats, TermStats... termStats) {
assert boost >= 0;
- assert collectionStats != null;
+ assert fieldStats != null;
assert termStats.length > 0;
- for (TermStatistics term : termStats) {
+ for (TermStats term : termStats) {
assert term != null;
}
// TODO: check that TermStats is in bounds with respect to collection? e.g. docFreq <= maxDoc
- SimScorer scorer = delegate.scorer(boost, collectionStats, termStats);
+ SimScorer scorer = delegate.scorer(boost, fieldStats, termStats);
assert scorer != null;
return new AssertingSimScorer(scorer, boost);
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java
index c56a6417384..4dbfa9f7910 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/search/similarities/BaseSimilarityTestCase.java
@@ -22,9 +22,9 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.FieldStats;
+import org.apache.lucene.search.TermStats;
import org.apache.lucene.search.similarities.IndriDirichletSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.BulkSimScorer;
@@ -83,7 +83,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
/**
* returns a random corpus that is at least possible given the norm value for a single document.
*/
- static CollectionStatistics newCorpus(Random random, int norm) {
+ static FieldStats newCorpus(Random random, int norm) {
// lower bound of tokens in the collection (you produced this norm somehow)
final int lowerBound;
if (norm == 0) {
@@ -177,13 +177,13 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
sumTotalTermFreq = TestUtil.nextLong(random, sumDocFreq - 1 + lowerBound, upperBound);
break;
}
- return new CollectionStatistics("field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
+ return new FieldStats("field", maxDoc, docCount, sumTotalTermFreq, sumDocFreq);
}
private static final BytesRef TERM = new BytesRef("term");
/** returns new random term, that fits within the bounds of the corpus */
- static TermStatistics newTerm(Random random, CollectionStatistics corpus) {
+ static TermStats newTerm(Random random, FieldStats corpus) {
final long docFreq;
switch (random.nextInt(3)) {
case 0:
@@ -227,7 +227,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
break;
}
}
- return new TermStatistics(TERM, docFreq, totalTermFreq);
+ return new TermStats(TERM, docFreq, totalTermFreq);
}
/**
@@ -262,9 +262,9 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
for (int j = 0; j < 3; j++) {
// for each norm value...
for (int k = 1; k < 256; k++) {
- CollectionStatistics corpus = newCorpus(random, k);
+ FieldStats corpus = newCorpus(random, k);
for (int l = 0; l < 10; l++) {
- TermStatistics term = newTerm(random, corpus);
+ TermStats term = newTerm(random, corpus);
final float freq;
if (term.totalTermFreq() == term.docFreq()) {
// omit TF
@@ -356,12 +356,7 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
* for that scenario
*/
private static void doTestScoring(
- Similarity similarity,
- CollectionStatistics corpus,
- TermStatistics term,
- float boost,
- float freq,
- int norm)
+ Similarity similarity, FieldStats corpus, TermStats term, float boost, float freq, int norm)
throws IOException {
boolean success = false;
SimScorer scorer = similarity.scorer(boost, corpus, term);
@@ -463,8 +458,8 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
// check score(term-1), given the same freq/norm it should be >= score(term) [scores
// non-decreasing as terms get rarer]
if (term.docFreq() > 1 && freq < term.totalTermFreq()) {
- TermStatistics prevTerm =
- new TermStatistics(term.term(), term.docFreq() - 1, term.totalTermFreq() - 1);
+ TermStats prevTerm =
+ new TermStats(term.term(), term.docFreq() - 1, term.totalTermFreq() - 1);
SimScorer prevTermScorer = similarity.scorer(boost, corpus, term);
float prevTermScore = prevTermScorer.score(freq, norm);
// check that score isn't infinite or negative
@@ -523,8 +518,8 @@ public abstract class BaseSimilarityTestCase extends LuceneTestCase {
public void testBulkScore() throws IOException {
Random random = random();
Similarity similarity = getSimilarity(random);
- CollectionStatistics corpus = newCorpus(random, 1);
- TermStatistics term = newTerm(random, corpus);
+ FieldStats corpus = newCorpus(random, 1);
+ TermStats term = newTerm(random, corpus);
SimScorer scorer = similarity.scorer(random().nextFloat(5f), corpus, term);
BulkSimScorer bulkScorer = scorer.asBulkSimScorer();
int freqUpperBound =