You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2025/02/19 12:32:39 UTC
(lucene) branch main updated: Move `CombinedFieldQuery` to the core module. (#14236)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 239059e69cf Move `CombinedFieldQuery` to the core module. (#14236)
239059e69cf is described below
commit 239059e69cf498b6f230968959075fb8d3ebd49e
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Wed Feb 19 13:32:34 2025 +0100
Move `CombinedFieldQuery` to the core module. (#14236)
`CombinedFieldQuery` is Lucene's most robust way of scoring across
multiple fields, let's move it to core and recommend using it to query
multiple fields.
---
lucene/CHANGES.txt | 3 +
.../apache/lucene}/search/CombinedFieldQuery.java | 104 ++--
.../lucene/search/MultiNormsLeafSimScorer.java | 177 +++++++
.../org/apache/lucene/search/package-info.java | 24 +
.../lucene/search/TestCombinedFieldQuery.java | 563 +++++++++++++++++++++
.../lucene/sandbox/search/CombinedFieldQuery.java | 4 +-
6 files changed, 805 insertions(+), 70 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index cd39edc3227..761fafeba09 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -52,6 +52,9 @@ API Changes
* GITHUB#14209: Deprecate Operations.union(Automaton,Automaton) and
concatenate(Automaton,Automaton) in favor of the methods taking List. (Robert Muir)
+* GITHUB#14236: CombinedFieldQuery moved from lucene-sandbox to lucene-core.
+ (Adrien Grand)
+
New Features
---------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
similarity index 85%
copy from lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
copy to lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
index 1ca3f790f43..42659e3b37e 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java
@@ -14,18 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.sandbox.search;
+package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -36,28 +33,9 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermStates;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.CollectionStatistics;
-import org.apache.lucene.search.DisiWrapper;
-import org.apache.lucene.search.DisjunctionDISIApproximation;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Matches;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.QueryVisitor;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.ScorerSupplier;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TermScorer;
-import org.apache.lucene.search.TermStatistics;
-import org.apache.lucene.search.Weight;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.DFRSimilarity;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.search.similarities.SimilarityBase;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOSupplier;
@@ -65,8 +43,9 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.SmallFloat;
/**
- * A {@link Query} that treats multiple fields as a single stream and scores terms as if you had
- * indexed them as a single term in a single field.
+ * A {@link Query} that treats multiple fields as a single stream and scores terms as if they had
+ * been indexed in a single field whose values would be the union of the values of the provided
+ * fields.
*
* <p>The query works as follows:
*
@@ -80,12 +59,15 @@ import org.apache.lucene.util.SmallFloat;
* <p>In order for a similarity to be compatible, {@link Similarity#computeNorm} must be additive:
* the norm of the combined field is the sum of norms for each individual field. The norms must also
* be encoded using {@link SmallFloat#intToByte4}. These requirements hold for all similarities that
- * compute norms the same way as {@link SimilarityBase#computeNorm}, which includes {@link
- * BM25Similarity} and {@link DFRSimilarity}. Per-field similarities are not supported.
+ * don't customize {@link Similarity#computeNorm}, which includes {@link BM25Similarity} and {@link
+ * DFRSimilarity}. Per-field similarities are not supported.
*
* <p>The query also requires that either all fields or no fields have norms enabled. Having only
* some fields with norms enabled can result in errors.
*
+ * <p>This query assumes that all fields share the same analyzer. Scores may not make much sense if
+ * all fields don't have the same analyzer.
+ *
* <p>The scoring is based on BM25F's simple formula described in:
* http://www.staff.city.ac.uk/~sb317/papers/foundations_bm25_review.pdf. This query implements the
* same approach but allows other similarities besides {@link
@@ -100,7 +82,17 @@ public final class CombinedFieldQuery extends Query implements Accountable {
/** A builder for {@link CombinedFieldQuery}. */
public static class Builder {
private final Map<String, FieldAndWeight> fieldAndWeights = new HashMap<>();
- private final Set<BytesRef> termsSet = new HashSet<>();
+ private final BytesRef term;
+
+ /** Create a builder for the given term {@link String}. */
+ public Builder(String term) {
+ this.term = new BytesRef(term);
+ }
+
+ /** Create a builder for the given term bytes. */
+ public Builder(BytesRef term) {
+ this.term = BytesRef.deepCopyOf(term);
+ }
/**
* Adds a field to this builder.
@@ -125,23 +117,12 @@ public final class CombinedFieldQuery extends Query implements Accountable {
return this;
}
- /** Adds a term to this builder. */
- public Builder addTerm(BytesRef term) {
- if (termsSet.size() >= IndexSearcher.getMaxClauseCount()) {
- throw new IndexSearcher.TooManyClauses();
- }
- termsSet.add(term);
- return this;
- }
-
/** Builds the {@link CombinedFieldQuery}. */
public CombinedFieldQuery build() {
- int size = fieldAndWeights.size() * termsSet.size();
- if (size > IndexSearcher.getMaxClauseCount()) {
+ if (fieldAndWeights.size() > IndexSearcher.getMaxClauseCount()) {
throw new IndexSearcher.TooManyClauses();
}
- BytesRef[] terms = termsSet.toArray(new BytesRef[0]);
- return new CombinedFieldQuery(new TreeMap<>(fieldAndWeights), terms);
+ return new CombinedFieldQuery(new TreeMap<>(fieldAndWeights), term);
}
}
@@ -149,38 +130,30 @@ public final class CombinedFieldQuery extends Query implements Accountable {
// sorted map for fields.
private final TreeMap<String, FieldAndWeight> fieldAndWeights;
- // array of terms, sorted.
- private final BytesRef[] terms;
- // array of terms per field, sorted
+ // term bytes
+ private final BytesRef term;
+ // array of terms per field, sorted by field
private final Term[] fieldTerms;
private final long ramBytesUsed;
- private CombinedFieldQuery(TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef[] terms) {
+ private CombinedFieldQuery(TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef term) {
this.fieldAndWeights = fieldAndWeights;
- this.terms = terms;
- int numFieldTerms = fieldAndWeights.size() * terms.length;
- if (numFieldTerms > IndexSearcher.getMaxClauseCount()) {
+ this.term = Objects.requireNonNull(term);
+ if (fieldAndWeights.size() > IndexSearcher.getMaxClauseCount()) {
throw new IndexSearcher.TooManyClauses();
}
- this.fieldTerms = new Term[numFieldTerms];
- Arrays.sort(terms);
+ this.fieldTerms = new Term[fieldAndWeights.size()];
int pos = 0;
for (String field : fieldAndWeights.keySet()) {
- for (BytesRef term : terms) {
- fieldTerms[pos++] = new Term(field, term);
- }
+ fieldTerms[pos++] = new Term(field, term);
}
this.ramBytesUsed =
BASE_RAM_BYTES
+ RamUsageEstimator.sizeOfObject(fieldAndWeights)
+ RamUsageEstimator.sizeOfObject(fieldTerms)
- + RamUsageEstimator.sizeOfObject(terms);
- }
-
- public List<Term> getTerms() {
- return Collections.unmodifiableList(Arrays.asList(fieldTerms));
+ + RamUsageEstimator.sizeOfObject(term);
}
@Override
@@ -198,13 +171,7 @@ public final class CombinedFieldQuery extends Query implements Accountable {
}
}
builder.append(")(");
- pos = 0;
- for (BytesRef term : terms) {
- if (pos++ != 0) {
- builder.append(" ");
- }
- builder.append(term.utf8ToString());
- }
+ builder.append(Term.toString(term));
builder.append("))");
return builder.toString();
}
@@ -214,15 +181,14 @@ public final class CombinedFieldQuery extends Query implements Accountable {
if (this == o) return true;
if (sameClassAs(o) == false) return false;
CombinedFieldQuery that = (CombinedFieldQuery) o;
- return Objects.equals(fieldAndWeights, that.fieldAndWeights)
- && Arrays.equals(terms, that.terms);
+ return Objects.equals(fieldAndWeights, that.fieldAndWeights) && term.equals(that.term);
}
@Override
public int hashCode() {
int result = classHash();
result = 31 * result + Objects.hash(fieldAndWeights);
- result = 31 * result + Arrays.hashCode(terms);
+ result = 31 * result + term.hashCode();
return result;
}
@@ -233,7 +199,7 @@ public final class CombinedFieldQuery extends Query implements Accountable {
@Override
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
- if (terms.length == 0 || fieldAndWeights.isEmpty()) {
+ if (fieldAndWeights.isEmpty()) {
return new BooleanQuery.Builder().build();
}
return this;
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.java b/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.java
new file mode 100644
index 00000000000..c6720b362d9
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiNormsLeafSimScorer.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.search.CombinedFieldQuery.FieldAndWeight;
+import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.util.SmallFloat;
+
+/**
+ * Scorer that sums document's norms from multiple fields.
+ *
+ * <p>For all fields, norms must be encoded using {@link SmallFloat#intToByte4}. This scorer also
+ * requires that either all fields or no fields have norms enabled. Having only some fields with
+ * norms enabled can result in errors or undefined behavior.
+ */
+final class MultiNormsLeafSimScorer {
+ /** Cache of decoded norms. */
+ private static final float[] LENGTH_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ LENGTH_TABLE[i] = SmallFloat.byte4ToInt((byte) i);
+ }
+ }
+
+ private final SimScorer scorer;
+ private final NumericDocValues norms;
+
+ /** Sole constructor: Score documents of {@code reader} with {@code scorer}. */
+ MultiNormsLeafSimScorer(
+ SimScorer scorer,
+ LeafReader reader,
+ Collection<FieldAndWeight> normFields,
+ boolean needsScores)
+ throws IOException {
+ this.scorer = Objects.requireNonNull(scorer);
+ if (needsScores) {
+ final List<NumericDocValues> normsList = new ArrayList<>();
+ final List<Float> weightList = new ArrayList<>();
+ final Set<String> duplicateCheckingSet = new HashSet<>();
+ for (FieldAndWeight field : normFields) {
+ assert duplicateCheckingSet.add(field.field())
+ : "There is a duplicated field ["
+ + field.field()
+ + "] used to construct MultiNormsLeafSimScorer";
+
+ NumericDocValues norms = reader.getNormValues(field.field());
+ if (norms != null) {
+ normsList.add(norms);
+ weightList.add(field.weight());
+ }
+ }
+
+ if (normsList.isEmpty()) {
+ norms = null;
+ } else {
+ final NumericDocValues[] normsArr = normsList.toArray(new NumericDocValues[0]);
+ final float[] weightArr = new float[normsList.size()];
+ for (int i = 0; i < weightList.size(); i++) {
+ weightArr[i] = weightList.get(i);
+ }
+ norms = new MultiFieldNormValues(normsArr, weightArr);
+ }
+ } else {
+ norms = null;
+ }
+ }
+
+ SimScorer getSimScorer() {
+ return scorer;
+ }
+
+ private long getNormValue(int doc) throws IOException {
+ if (norms != null) {
+ boolean found = norms.advanceExact(doc);
+ assert found;
+ return norms.longValue();
+ } else {
+ return 1L; // default norm
+ }
+ }
+
+ /**
+ * Score the provided document assuming the given term document frequency. This method must be
+ * called on non-decreasing sequences of doc ids.
+ *
+ * @see SimScorer#score(float, long)
+ */
+ public float score(int doc, float freq) throws IOException {
+ return scorer.score(freq, getNormValue(doc));
+ }
+
+ /**
+ * Explain the score for the provided document assuming the given term document frequency. This
+ * method must be called on non-decreasing sequences of doc ids.
+ *
+ * @see SimScorer#explain(Explanation, long)
+ */
+ public Explanation explain(int doc, Explanation freqExpl) throws IOException {
+ return scorer.explain(freqExpl, getNormValue(doc));
+ }
+
+ private static class MultiFieldNormValues extends NumericDocValues {
+ private final NumericDocValues[] normsArr;
+ private final float[] weightArr;
+ private long current;
+ private int docID = -1;
+
+ MultiFieldNormValues(NumericDocValues[] normsArr, float[] weightArr) {
+ this.normsArr = normsArr;
+ this.weightArr = weightArr;
+ }
+
+ @Override
+ public long longValue() {
+ return current;
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ float normValue = 0;
+ boolean found = false;
+ for (int i = 0; i < normsArr.length; i++) {
+ if (normsArr[i].advanceExact(target)) {
+ normValue +=
+ weightArr[i] * LENGTH_TABLE[Byte.toUnsignedInt((byte) normsArr[i].longValue())];
+ found = true;
+ }
+ }
+ current = SmallFloat.intToByte4(Math.round(normValue));
+ return found;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int nextDoc() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/package-info.java b/lucene/core/src/java/org/apache/lucene/search/package-info.java
index 98b60d94222..664e8e61693 100644
--- a/lucene/core/src/java/org/apache/lucene/search/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/search/package-info.java
@@ -278,6 +278,30 @@
* <p>See the {@link org.apache.lucene.search.similarities} package documentation for information on
* the built-in available scoring models and extending or changing Similarity.
*
+ * <h3>Scoring multiple fields</h3>
+ *
+ * <p>In the real world, documents often have multiple fields with different degrees of relevance. A
+ * robust way of scoring across multiple fields is called BM25F, which is implemented via {@link
+ * org.apache.lucene.search.CombinedFieldQuery}. It scores documents with multiple fields as if
+ * their content had been indexed in a single combined field. It supports configuring per-field
+ * boosts where the value of the boost is interpreted as the number of times that the content of the
+ * field exists in the virtual combined field.
+ *
+ * <p>Here is an example that constructs a query on "apache OR lucene" on fields "title" with a
+ * boost of 10, and "body" with a boost of 1:
+ *
+ * <pre class="prettyprint">
+ * BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ * for (String term : new String[] { "apache", "lucene" }) {
+ * Query query = new CombinedFieldQuery(term)
+ * .addField("title", 10f)
+ * .addField("body", 1f)
+ * .build();
+ * builder.add(query, Occur.SHOULD);
+ * }
+ * Query query = builder.build();
+ * </pre>
+ *
* <h3>Integrating field values into the score</h3>
*
* <p>While similarities help score a document relatively to a query, it is also common for
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java
new file mode 100644
index 00000000000..ed0c051d0ba
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestCombinedFieldQuery.java
@@ -0,0 +1,563 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomBoolean;
+import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween;
+
+import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.BooleanSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
+import org.apache.lucene.search.similarities.LMDirichletSimilarity;
+import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.search.CheckHits;
+import org.apache.lucene.tests.util.LuceneTestCase;
+
+public class TestCombinedFieldQuery extends LuceneTestCase {
+ public void testInvalid() {
+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("foo");
+ IllegalArgumentException exc =
+ expectThrows(IllegalArgumentException.class, () -> builder.addField("foo", 0.5f));
+ assertEquals(exc.getMessage(), "weight must be greater or equal to 1");
+ }
+
+ public void testRewrite() throws IOException {
+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("foo");
+ IndexReader reader = new MultiReader();
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Query actual = searcher.rewrite(builder.build());
+ assertEquals(new MatchNoDocsQuery(), actual);
+ builder.addField("field", 1f);
+ Query query = builder.build();
+ actual = searcher.rewrite(builder.build());
+ assertEquals(query, actual);
+ }
+
+ public void testEqualsAndHashCode() {
+ CombinedFieldQuery query1 =
+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2").build();
+
+ CombinedFieldQuery query2 =
+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2", 1.3f).build();
+ assertNotEquals(query1, query2);
+ assertNotEquals(query1.hashCode(), query2.hashCode());
+
+ CombinedFieldQuery query3 =
+ new CombinedFieldQuery.Builder("value").addField("field3").addField("field4").build();
+ assertNotEquals(query1, query3);
+ assertNotEquals(query1.hashCode(), query2.hashCode());
+
+ CombinedFieldQuery duplicateQuery1 =
+ new CombinedFieldQuery.Builder("value").addField("field1").addField("field2").build();
+ assertEquals(query1, duplicateQuery1);
+ assertEquals(query1.hashCode(), duplicateQuery1.hashCode());
+ }
+
+ public void testToString() {
+ CombinedFieldQuery.Builder builder = new CombinedFieldQuery.Builder("bar");
+ assertEquals("CombinedFieldQuery(()(bar))", builder.build().toString());
+ builder.addField("foo", 1f);
+ assertEquals("CombinedFieldQuery((foo)(bar))", builder.build().toString());
+ builder.addField("title", 3f);
+ assertEquals("CombinedFieldQuery((foo title^3.0)(bar))", builder.build().toString());
+ }
+
+ public void testSameScore() throws IOException {
+ Directory dir = newDirectory();
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ Document doc = new Document();
+ doc.add(new StringField("f", "a", Store.NO));
+ w.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new StringField("g", "a", Store.NO));
+ for (int i = 0; i < 10; ++i) {
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("a").addField("f", 1f).addField("g", 1f).build();
+ TopScoreDocCollectorManager collectorManager =
+ new TopScoreDocCollectorManager(
+ Math.min(reader.numDocs(), Integer.MAX_VALUE), Integer.MAX_VALUE);
+ TopDocs topDocs = searcher.search(query, collectorManager);
+ assertEquals(new TotalHits(11, TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
+ // All docs must have the same score
+ for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
+ assertEquals(topDocs.scoreDocs[0].score, topDocs.scoreDocs[i].score, 0.0f);
+ }
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testScoringWithMultipleFieldTermsMatch() throws IOException {
+ int numMatchDoc = randomIntBetween(100, 500);
+ int numHits = randomIntBetween(1, 100);
+ int boost1 = Math.max(1, random().nextInt(5));
+ int boost2 = Math.max(1, random().nextInt(5));
+
+ Directory dir = newDirectory();
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ // adding potentially matching doc
+ for (int i = 0; i < numMatchDoc; i++) {
+ Document doc = new Document();
+
+ int freqA = random().nextInt(20) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo", Store.NO));
+ }
+
+ freqA = random().nextInt(20) + 1;
+ if (randomBoolean()) {
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo" + j, Store.NO));
+ }
+ }
+
+ freqA = random().nextInt(20) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "zoo", Store.NO));
+ }
+
+ int freqB = random().nextInt(20) + 1;
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "zoo", Store.NO));
+ }
+
+ freqB = random().nextInt(20) + 1;
+ if (randomBoolean()) {
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "zoo" + j, Store.NO));
+ }
+ }
+
+ int freqC = random().nextInt(20) + 1;
+ for (int j = 0; j < freqC; j++) {
+ doc.add(new TextField("c", "bla" + j, Store.NO));
+ }
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("foo")
+ .addField("a", (float) boost1)
+ .addField("b", (float) boost2)
+ .build();
+
+ CollectorManager<TopScoreDocCollector, TopDocs> completeManager =
+ new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);
+
+ searcher.search(query, completeManager);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testNormsDisabled() throws IOException {
+ Directory dir = newDirectory();
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ Document doc = new Document();
+ doc.add(new StringField("a", "value", Store.NO));
+ doc.add(new StringField("b", "value", Store.NO));
+ doc.add(new TextField("c", "value", Store.NO));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new StringField("a", "value", Store.NO));
+ doc.add(new TextField("c", "value", Store.NO));
+ w.addDocument(doc);
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Similarity searchSimilarity = randomCompatibleSimilarity();
+ searcher.setSimilarity(searchSimilarity);
+ TopScoreDocCollectorManager collectorManager = new TopScoreDocCollectorManager(10, 10);
+
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("value").addField("a", 1.0f).addField("b", 1.0f).build();
+ TopDocs topDocs = searcher.search(query, collectorManager);
+ assertEquals(new TotalHits(2, TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
+
+ CombinedFieldQuery invalidQuery =
+ new CombinedFieldQuery.Builder("value").addField("b", 1.0f).addField("c", 1.0f).build();
+ IllegalArgumentException e =
+ expectThrows(
+ IllegalArgumentException.class, () -> searcher.search(invalidQuery, collectorManager));
+ assertTrue(e.getMessage().contains("requires norms to be consistent across fields"));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCopyField() throws IOException {
+ Directory dir = newDirectory();
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ int numMatch = atLeast(10);
+ int boost1 = Math.max(1, random().nextInt(5));
+ int boost2 = Math.max(1, random().nextInt(5));
+ for (int i = 0; i < numMatch; i++) {
+ Document doc = new Document();
+ if (random().nextBoolean()) {
+ doc.add(new TextField("a", "baz", Store.NO));
+ doc.add(new TextField("b", "baz", Store.NO));
+ for (int k = 0; k < boost1 + boost2; k++) {
+ doc.add(new TextField("ab", "baz", Store.NO));
+ }
+ w.addDocument(doc);
+ doc.clear();
+ }
+ int freqA = random().nextInt(5) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo", Store.NO));
+ }
+ int freqB = random().nextInt(5) + 1;
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "foo", Store.NO));
+ }
+ int freqAB = freqA * boost1 + freqB * boost2;
+ for (int j = 0; j < freqAB; j++) {
+ doc.add(new TextField("ab", "foo", Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ searcher.setSimilarity(similarity);
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("foo")
+ .addField("a", (float) boost1)
+ .addField("b", (float) boost2)
+ .build();
+
+ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCopyFieldWithSingleField() throws IOException {
+ Directory dir = new MMapDirectory(createTempDir());
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ int boost = Math.max(1, random().nextInt(5));
+ int numMatch = atLeast(10);
+ for (int i = 0; i < numMatch; i++) {
+ Document doc = new Document();
+ int freqA = random().nextInt(5) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo", Store.NO));
+ }
+
+ int freqB = freqA * boost;
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "foo", Store.NO));
+ }
+
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("foo").addField("a", (float) boost).build();
+
+ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("b", "foo")));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCopyFieldWithMissingFields() throws IOException {
+ Directory dir = new MMapDirectory(createTempDir());
+ Similarity similarity = randomCompatibleSimilarity();
+
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ int boost1 = Math.max(1, random().nextInt(5));
+ int boost2 = Math.max(1, random().nextInt(5));
+ int numMatch = atLeast(10);
+ for (int i = 0; i < numMatch; i++) {
+ Document doc = new Document();
+ int freqA = random().nextInt(5) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo", Store.NO));
+ }
+
+ // Choose frequencies such that sometimes we don't add field B
+ int freqB = random().nextInt(3);
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "foo", Store.NO));
+ }
+
+ int freqAB = freqA * boost1 + freqB * boost2;
+ for (int j = 0; j < freqAB; j++) {
+ doc.add(new TextField("ab", "foo", Store.NO));
+ }
+
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(similarity);
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("foo")
+ .addField("a", (float) boost1)
+ .addField("b", (float) boost2)
+ .build();
+
+ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ private static Similarity randomCompatibleSimilarity() {
+ return RandomPicks.randomFrom(
+ random(),
+ Arrays.asList(
+ new BM25Similarity(),
+ new BooleanSimilarity(),
+ new ClassicSimilarity(),
+ new LMDirichletSimilarity(),
+ new LMJelinekMercerSimilarity(0.1f)));
+ }
+
+ private void checkExpectedHits(
+ IndexSearcher searcher, int numHits, Query firstQuery, Query secondQuery) throws IOException {
+ TopScoreDocCollectorManager collectorManager =
+ new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);
+
+ TopDocs firstTopDocs = searcher.search(firstQuery, collectorManager);
+ assertEquals(numHits, firstTopDocs.totalHits.value());
+
+ collectorManager = new TopScoreDocCollectorManager(numHits, Integer.MAX_VALUE);
+ TopDocs secondTopDocs = searcher.search(secondQuery, collectorManager);
+ CheckHits.checkEqual(firstQuery, secondTopDocs.scoreDocs, firstTopDocs.scoreDocs);
+ }
+
+ public void testDocWithNegativeNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(new NegativeNormSimilarity());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ String queryString = "foo";
+
+ Document doc = new Document();
+ // both fields must contain tokens that match the query string "foo"
+ doc.add(new TextField("f", "foo", Store.NO));
+ doc.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc);
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder(queryString).addField("f").addField("g").build();
+ TopDocs topDocs = searcher.search(query, 10);
+ CheckHits.checkDocIds("queried docs do not match", new int[] {0}, topDocs.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMultipleDocsNegativeNorms() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ iwc.setSimilarity(new NegativeNormSimilarity());
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ String queryString = "foo";
+
+ Document doc0 = new Document();
+ doc0.add(new TextField("f", "foo", Store.NO));
+ doc0.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc0);
+
+ Document doc1 = new Document();
+ // add another match on the query string to the second doc
+ doc1.add(new TextField("f", "foo is foo", Store.NO));
+ doc1.add(new TextField("g", "foo baz", Store.NO));
+ w.addDocument(doc1);
+
+ IndexReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder(queryString).addField("f").addField("g").build();
+ TopDocs topDocs = searcher.search(query, 10);
+ // Return doc1 ahead of doc0 since its tf is higher
+ CheckHits.checkDocIds("queried docs do not match", new int[] {1, 0}, topDocs.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ private static final class NegativeNormSimilarity extends Similarity {
+ @Override
+ public long computeNorm(FieldInvertState state) {
+ return -128;
+ }
+
+ @Override
+ public SimScorer scorer(
+ float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return new BM25Similarity().scorer(boost, collectionStats, termStats);
+ }
+ }
+
+ public void testOverrideCollectionStatistics() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig();
+ Similarity similarity = randomCompatibleSimilarity();
+ iwc.setSimilarity(similarity);
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);
+
+ int numMatch = atLeast(10);
+ for (int i = 0; i < numMatch; i++) {
+ Document doc = new Document();
+ if (random().nextBoolean()) {
+ doc.add(new TextField("a", "baz", Store.NO));
+ doc.add(new TextField("b", "baz", Store.NO));
+ for (int k = 0; k < 2; k++) {
+ doc.add(new TextField("ab", "baz", Store.NO));
+ }
+ w.addDocument(doc);
+ doc.clear();
+ }
+ int freqA = random().nextInt(5) + 1;
+ for (int j = 0; j < freqA; j++) {
+ doc.add(new TextField("a", "foo", Store.NO));
+ }
+ int freqB = random().nextInt(5) + 1;
+ for (int j = 0; j < freqB; j++) {
+ doc.add(new TextField("b", "foo", Store.NO));
+ }
+ int freqAB = freqA + freqB;
+ for (int j = 0; j < freqAB; j++) {
+ doc.add(new TextField("ab", "foo", Store.NO));
+ }
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = w.getReader();
+
+ int extraMaxDoc = randomIntBetween(0, 10);
+ int extraDocCount = randomIntBetween(0, extraMaxDoc);
+ int extraSumDocFreq = extraDocCount + randomIntBetween(0, 10);
+
+ int extraSumTotalTermFreqA = extraSumDocFreq + randomIntBetween(0, 10);
+ int extraSumTotalTermFreqB = extraSumDocFreq + randomIntBetween(0, 10);
+ int extraSumTotalTermFreqAB = extraSumTotalTermFreqA + extraSumTotalTermFreqB;
+
+ IndexSearcher searcher =
+ new IndexSearcher(reader) {
+ @Override
+ public CollectionStatistics collectionStatistics(String field) throws IOException {
+ CollectionStatistics shardStatistics = super.collectionStatistics(field);
+ int extraSumTotalTermFreq;
+ if (field.equals("a")) {
+ extraSumTotalTermFreq = extraSumTotalTermFreqA;
+ } else if (field.equals("b")) {
+ extraSumTotalTermFreq = extraSumTotalTermFreqB;
+ } else if (field.equals("ab")) {
+ extraSumTotalTermFreq = extraSumTotalTermFreqAB;
+ } else {
+ throw new AssertionError("should never be called");
+ }
+ return new CollectionStatistics(
+ field,
+ shardStatistics.maxDoc() + extraMaxDoc,
+ shardStatistics.docCount() + extraDocCount,
+ shardStatistics.sumTotalTermFreq() + extraSumTotalTermFreq,
+ shardStatistics.sumDocFreq() + extraSumDocFreq);
+ }
+ };
+ searcher.setSimilarity(similarity);
+ CombinedFieldQuery query =
+ new CombinedFieldQuery.Builder("foo").addField("a").addField("b").build();
+
+ checkExpectedHits(searcher, numMatch, query, new TermQuery(new Term("ab", "foo")));
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
index 1ca3f790f43..debd10ea83d 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/search/CombinedFieldQuery.java
@@ -92,7 +92,9 @@ import org.apache.lucene.util.SmallFloat;
* org.apache.lucene.search.similarities.BM25Similarity}.
*
* @lucene.experimental
+ * @deprecated Use {@link org.apache.lucene.search.CombinedFieldQuery} instead.
*/
+@Deprecated
public final class CombinedFieldQuery extends Query implements Accountable {
private static final long BASE_RAM_BYTES =
RamUsageEstimator.shallowSizeOfInstance(CombinedFieldQuery.class);
@@ -203,7 +205,7 @@ public final class CombinedFieldQuery extends Query implements Accountable {
if (pos++ != 0) {
builder.append(" ");
}
- builder.append(term.utf8ToString());
+ builder.append(Term.toString(term));
}
builder.append("))");
return builder.toString();