You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by at...@apache.org on 2025/03/26 18:14:51 UTC
(lucene) 01/05: LUCENE-XXXX: Add AnytimeRankingSearcher for SLA-constrained ranking with heuristic estimation and optional range boosting
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_querytime_processing
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 7d2c7829c87bac46af08a7041c7b022cf400972f
Author: Atri Sharma <at...@Atris-MacBook-Pro.local>
AuthorDate: Wed Mar 26 23:02:27 2025 +0530
LUCENE-XXXX: Add AnytimeRankingSearcher for SLA-constrained ranking with heuristic estimation and optional range boosting
This patch introduces AnytimeRankingSearcher, a new search utility that enforces SLA-based early termination using a custom TopDocsCollector.
Key features:
- SLA estimation via HeuristicSLAEstimator, using a weighted combination of:
- Term count
- Query structural complexity
- Sampled average document frequency
- Early termination using nanosecond tracking and periodic cutoff checks.
- Optional range-based boosting via BoostQuery-wrapped IntPoint ranges, sorted by descending boost score.
- Supports a broad range of Query types (TermQuery, BooleanQuery, DisjunctionMaxQuery, PhraseQuery, etc.) including nested structures.
- CollectorManager implements fallback merging logic to handle shard-level early termination.
Includes test coverage for heuristic estimation, range prioritization, SLA enforcement, and concurrency.
---
.../lucene/search/AnytimeRankingCollector.java | 94 +++++++++
.../search/AnytimeRankingCollectorManager.java | 65 ++++++
.../lucene/search/AnytimeRankingSearcher.java | 93 ++++++++
.../apache/lucene/util/HeuristicSLAEstimator.java | 212 +++++++++++++++++++
.../java/org/apache/lucene/util/SLAEstimator.java | 27 +++
.../search/TestAnytimeRankingRelevanceSearch.java | 137 ++++++++++++
.../lucene/search/TestAnytimeRankingSearch.java | 234 +++++++++++++++++++++
.../lucene/util/TestHeuristicSLAEstimator.java | 123 +++++++++++
8 files changed, 985 insertions(+)
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
new file mode 100644
index 00000000000..ea67590a0b4
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+import java.io.IOException;
+
+public class AnytimeRankingCollector extends TopDocsCollector<ScoreDoc> {
+ private static final int MIN_SLA_CHECK_INTERVAL = 32;
+ private static final int MAX_SLA_CHECK_INTERVAL = 512;
+ private static final int DOCS_PER_SLA_CHECK = 100;
+
+ private final long slaThresholdNanos;
+ private final long startTime;
+ private int docCounter = 0;
+
+ protected AnytimeRankingCollector(int topK, long slaThresholdMs) {
+ super(new HitQueue(topK, false));
+ this.slaThresholdNanos = slaThresholdMs * 1_000_000L;
+ this.startTime = System.nanoTime();
+ }
+
+ @Override
+ protected int topDocsSize() {
+ return pq.size();
+ }
+
+ @Override
+ protected TopDocs newTopDocs(ScoreDoc[] results, int start) {
+ return results == null
+ ? new TopDocs(new TotalHits(totalHits, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), new ScoreDoc[0])
+ : new TopDocs(new TotalHits(totalHits, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), results);
+ }
+
+ @Override
+ public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
+ final int docBase = context.docBase;
+ int estimatedInterval = context.reader().numDocs() / DOCS_PER_SLA_CHECK;
+ int slaCheckInterval = Math.max(MIN_SLA_CHECK_INTERVAL,
+ Math.min(MAX_SLA_CHECK_INTERVAL, estimatedInterval));
+
+ return new LeafCollector() {
+ private Scorable scorer;
+
+ @Override
+ public void setScorer(Scorable scorer) {
+ this.scorer = scorer;
+ }
+
+ @Override
+ public void collect(int doc) throws IOException {
+ float score = scorer.score();
+
+ if (Float.isNaN(score) || score <= 0) {
+ return;
+ }
+
+ int globalDoc = doc + docBase;
+ ScoreDoc scoreDoc = new ScoreDoc(globalDoc, score);
+
+ if (pq.insertWithOverflow(scoreDoc) != scoreDoc) {
+ totalHits++;
+ }
+
+ if ((docCounter++ % slaCheckInterval) == 0) {
+ long elapsed = System.nanoTime() - startTime;
+ if (elapsed >= slaThresholdNanos) {
+ throw new CollectionTerminatedException();
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.TOP_SCORES;
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java
new file mode 100644
index 00000000000..9a273c6236c
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+
+public class AnytimeRankingCollectorManager implements CollectorManager<AnytimeRankingCollector, TopDocs> {
+ private final int topK;
+ private final long slaThresholdMs;
+
+ public AnytimeRankingCollectorManager(int topK, long slaThresholdMs) {
+ this.topK = topK;
+ this.slaThresholdMs = slaThresholdMs;
+ }
+
+ @Override
+ public AnytimeRankingCollector newCollector() {
+ return new AnytimeRankingCollector(topK, slaThresholdMs);
+ }
+
+ @Override
+ public TopDocs reduce(Collection<AnytimeRankingCollector> collectors) throws IOException {
+ PriorityQueue<ScoreDoc> sortedDocs = new PriorityQueue<>(topK, Comparator.comparingDouble(sd -> sd.score));
+ long totalHits = 0;
+
+ for (AnytimeRankingCollector collector : collectors) {
+ TopDocs topDocs = collector.topDocs();
+ totalHits += topDocs.totalHits.value();
+
+ for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+ if (sortedDocs.size() < topK) {
+ sortedDocs.offer(scoreDoc);
+ } else if (scoreDoc.score > sortedDocs.peek().score) {
+ sortedDocs.poll();
+ sortedDocs.offer(scoreDoc);
+ }
+ }
+ }
+
+ // Cannot use TopDocs.merge() since it assumes each shard has returned atleast K results
+ // which is not necessarily true for early termination
+ ScoreDoc[] results = sortedDocs.toArray(new ScoreDoc[0]);
+ Arrays.sort(results, (a, b) -> Float.compare(b.score, a.score)); // Descending order
+
+ return new TopDocs(new TotalHits(totalHits, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO), results);
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
new file mode 100644
index 00000000000..87dd7858542
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.util.HeuristicSLAEstimator;
+import org.apache.lucene.util.SLAEstimator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * AnytimeRankingSearcher runs SLA-aware queries with adaptive scoring and soft range boosting.
+ */
+public class AnytimeRankingSearcher {
+ private final IndexSearcher searcher;
+ private final int topK;
+ private final long slaThresholdMs;
+ private final SLAEstimator slaEstimator;
+ private final String rangeField;
+
+ public AnytimeRankingSearcher(IndexSearcher searcher, int topK, long slaThresholdMs, String contentField) {
+ this(searcher, topK, slaThresholdMs, contentField, "docID");
+ }
+
+ public AnytimeRankingSearcher(IndexSearcher searcher, int topK, long slaThresholdMs,
+ String contentField, String rangeField) {
+ this.searcher = searcher;
+ this.topK = topK;
+ this.slaThresholdMs = slaThresholdMs;
+ this.slaEstimator = new HeuristicSLAEstimator(contentField);
+ this.rangeField = rangeField;
+ }
+
+ /**
+ * Add range boosts
+ */
+ private Query maybeWrapWithRanges(Query baseQuery, Map<Integer, Double> rangeScores, int maxDocID) {
+ if (rangeScores == null || rangeScores.isEmpty()) {
+ return baseQuery;
+ }
+
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ builder.add(baseQuery, BooleanClause.Occur.MUST);
+
+ List<Integer> ranges = new ArrayList<>(rangeScores.keySet());
+ ranges.sort(Comparator.comparingDouble(rangeScores::get).reversed());
+
+ // Split docID space evenly into range bins
+ int binSize = Math.max(1, (int) Math.ceil((double) maxDocID / (ranges.size() + 1)));
+
+ for (int range : ranges) {
+ int lower = range * binSize;
+ int upper = (range + 1) * binSize - 1;
+ Query rangeQuery = IntPoint.newRangeQuery(rangeField, lower, upper);
+
+ double boost = rangeScores.getOrDefault(range, 1.0);
+ if (boost > 0.0) {
+ rangeQuery = new BoostQuery(rangeQuery, (float) boost);
+ }
+
+ builder.add(rangeQuery, BooleanClause.Occur.SHOULD);
+ }
+
+ return builder.build();
+ }
+
+ public TopDocs search(Query query, Map<Integer, Double> rangeScores, int maxDocID) throws IOException {
+ double estimatedSLA = slaEstimator.estimate(query, searcher.getIndexReader(), slaThresholdMs);
+ AnytimeRankingCollectorManager collectorManager =
+ new AnytimeRankingCollectorManager(topK, (long) estimatedSLA);
+
+ Query finalQuery = maybeWrapWithRanges(query, rangeScores, maxDocID);
+ return searcher.search(finalQuery, collectorManager);
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/util/HeuristicSLAEstimator.java b/lucene/core/src/java/org/apache/lucene/util/HeuristicSLAEstimator.java
new file mode 100644
index 00000000000..656294634dd
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/HeuristicSLAEstimator.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
+import org.apache.lucene.search.TermQuery;
+
+import java.io.IOException;
+
+/**
+ * Estimate SLA based the number of query terms, query complexity,
+ * and average document frequency
+ */
+public class HeuristicSLAEstimator implements SLAEstimator {
+
+ private static final int MAX_SAMPLE_TERMS = 64;
+ private static final double MAX_MULT = 3.0;
+ private static final double TERM_COST = 0.2;
+ private static final double FREQ_COST = 0.0015;
+ private static final double QUERY_COMPLEXITY = 0.15;
+
+ private final String field;
+
+ public HeuristicSLAEstimator(String field) {
+ this.field = field;
+ }
+
+ @Override
+ public double estimate(Query query, IndexReader reader, long baseSlaMs) throws IOException {
+ final int termCount = estimateTermCount(query);
+ final double avgDocFreq = estimateAvgDocFreq(reader);
+ final int queryCost = estimateQueryCost(query);
+
+ double m = 1.0;
+ m += TERM_COST * termCount;
+ m += FREQ_COST * avgDocFreq;
+ m += QUERY_COMPLEXITY * queryCost;
+
+ return baseSlaMs * Math.min(MAX_MULT, m);
+ }
+
+ /**
+ * Estimate of number of terms that will participate in the scoring
+ */
+ private int estimateTermCount(Query query) {
+ if (query instanceof MatchAllDocsQuery) {
+ return 0;
+ }
+
+ if (query instanceof TermQuery) {
+ return 1;
+ }
+
+ if (query instanceof PhraseQuery pq) {
+ return pq.getTerms().length;
+ }
+
+ if (query instanceof BooleanQuery bq) {
+ int total = 0;
+ for (BooleanClause clause : bq) {
+ BooleanClause.Occur occur = clause.occur();
+ if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.SHOULD) {
+ total += estimateTermCount(clause.query());
+ }
+ }
+ return total > 0 ? total : 1;
+ }
+
+ if (query instanceof DisjunctionMaxQuery dmq) {
+ // Take the cost of max cost branch
+ int max = 0;
+ for (Query sub : dmq.getDisjuncts()) {
+ max = Math.max(max, estimateTermCount(sub));
+ }
+ return max > 0 ? max : 1;
+ }
+
+ if (query instanceof SynonymQuery sq) {
+ return sq.getTerms().size();
+ }
+
+ if (query instanceof MultiTermQuery) {
+ // Conservative measure - no way to actually estimate
+ return 2;
+ }
+
+ if (query instanceof BoostQuery bq) {
+ return estimateTermCount(bq.getQuery());
+ }
+
+ if (query instanceof ConstantScoreQuery csq) {
+ return estimateTermCount(csq.getQuery());
+ }
+
+ // Default baseline cost - fallback if unknown query type
+ // Returning 0 will reduce the SLA to impractical values
+
+ return 1;
+ }
+
+ /**
+ * Coarse estimate of query execution cost.
+ */
+ private int estimateQueryCost(Query query) {
+ if (query instanceof MatchAllDocsQuery) {
+ return 1;
+ }
+
+ if (query instanceof TermQuery) {
+ return 2;
+ }
+
+ if (query instanceof PhraseQuery pq) {
+ return 3 + pq.getTerms().length;
+ }
+
+ if (query instanceof BooleanQuery bq) {
+ int total = 0;
+ for (BooleanClause clause : bq) {
+ BooleanClause.Occur occur = clause.occur();
+ if (occur == BooleanClause.Occur.MUST || occur == BooleanClause.Occur.SHOULD) {
+ total += estimateQueryCost(clause.query());
+ }
+ }
+ return Math.max(2, total);
+ }
+
+ if (query instanceof DisjunctionMaxQuery dmq) {
+ // Max cost branch
+ int max = 0;
+ for (Query sub : dmq.getDisjuncts()) {
+ max = Math.max(max, estimateQueryCost(sub));
+ }
+ return 2 + max;
+ }
+
+ if (query instanceof SynonymQuery sq) {
+ return 2 + sq.getTerms().size();
+ }
+
+ if (query instanceof MultiTermQuery) {
+ // Conservative estimate - no way to actually measure
+ return 6;
+ }
+
+ if (query instanceof BoostQuery bq) {
+ return estimateQueryCost(bq.getQuery());
+ }
+
+ if (query instanceof ConstantScoreQuery csq) {
+ return estimateQueryCost(csq.getQuery());
+ }
+
+ // Default baseline cost - fallback if unknown query type
+ // Returning 0 will reduce the SLA to impractical values
+ return 4;
+ }
+
+ /**
+ * Computes average document frequency across a sample of terms in the field.
+ */
+ private double estimateAvgDocFreq(IndexReader reader) throws IOException {
+ long docFreqSum = 0;
+ int sampled = 0;
+
+ for (LeafReaderContext leaf : reader.leaves()) {
+ Terms terms = leaf.reader().terms(field);
+ if (terms == null) {
+ continue;
+ }
+
+ TermsEnum te = terms.iterator();
+ while (te.next() != null && sampled < MAX_SAMPLE_TERMS) {
+ docFreqSum += te.docFreq();
+ sampled++;
+ }
+
+ if (sampled >= MAX_SAMPLE_TERMS) {
+ break;
+ }
+ }
+
+ return sampled > 0 ? (double) docFreqSum / sampled : 0.0;
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/util/SLAEstimator.java b/lucene/core/src/java/org/apache/lucene/util/SLAEstimator.java
new file mode 100644
index 00000000000..bb15cf52ef5
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/util/SLAEstimator.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+
+import java.io.IOException;
+
+public interface SLAEstimator {
+ double estimate(Query query, IndexReader reader, long baseSlaMs) throws IOException;
+}
+
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java
new file mode 100644
index 00000000000..0e46299aed5
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashMap;
+
+public class TestAnytimeRankingRelevanceSearch extends LuceneTestCase {
+
+ private String indexPath;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ indexPath = Files.createTempDirectory("testindex_relevance").toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+
+ for (int i = 1; i <= 50; i++) {
+ addDocument(writer, "Lucene test document " + i + " containing ranking relevance efficiency performance scoring", i);
+ if (i % 5 == 0) {
+ addDocument(writer, "Lucene boosted search efficiency query expansion", i + 50);
+ }
+ if (i % 3 == 0) {
+ addDocument(writer, "Lucene document test ranking optimization BM25 tuning", i + 100);
+ }
+ if (i % 7 == 0) {
+ addDocument(writer, "Lucene scalable indexing techniques for optimal retrieval", i + 150);
+ }
+ }
+
+ writer.commit();
+ }
+ }
+
+ @Test
+ public void testRelevanceAcrossDiverseDocuments() throws Exception {
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ Query query = new TermQuery(new Term("content", "lucene"));
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 20, "content");
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+
+ assertNotNull("Results should not be null", results);
+ assertTrue("Results should be greater than zero", results.scoreDocs.length > 0);
+ }
+ }
+
+ @Test
+ public void testPerformanceQueries() throws Exception {
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 15, 30, "content");
+ Query query = new TermQuery(new Term("content", "performance"));
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+
+ assertNotNull("Results should not be null", results);
+ assertTrue("Results should be greater than zero", results.scoreDocs.length > 0);
+ }
+ }
+
+ @Test
+ public void testBM25ScoringDiversity() throws Exception {
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 50, "content");
+ Query query = new TermQuery(new Term("content", "scoring"));
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+
+ assertNotNull("Results should not be null", results);
+ assertTrue("Results should be greater than zero", results.scoreDocs.length > 0);
+ }
+ }
+
+ @Test
+ public void testEmptyQuery() throws Exception {
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 5, 20, "content");
+ Query query = new TermQuery(new Term("content", " "));
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+
+ assertNotNull("Empty queries should return zero results", results);
+ assertEquals("Results should be zero for an empty query", 0, results.scoreDocs.length);
+ }
+ }
+
+ private void addDocument(IndexWriter writer, String content, int docID) throws IOException {
+ Document doc = new Document();
+ doc.add(new IntPoint("docID", docID));
+ doc.add(new StoredField("docID", docID));
+ doc.add(new TextField("content", content, Field.Store.NO));
+ writer.addDocument(doc);
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
new file mode 100644
index 00000000000..dc630a69c3b
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.IntPoint;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+public class TestAnytimeRankingSearch extends LuceneTestCase {
+ public void testAnytimeRanking() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex1")
+ .toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+
+ for (int i = 1; i <= 10000; i++) {
+ String content = "Lucene document " + i + " ranking relevance retrieval performance precision recall diversity feedback tuning".split(" ")[i % 10];
+ addDocument(writer, content, i);
+ }
+
+ writer.commit();
+ }
+
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+ Query query = new TermQuery(new Term("content", "lucene"));
+
+ Map<Integer, Double> rangeScores = new HashMap<>();
+ for (int i = 1; i <= 10; i++) {
+ rangeScores.put(i, Math.random());
+ }
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 50, "content");
+ ExecutorService executor = Executors.newCachedThreadPool();
+ List<Future<TopDocs>> futures = new ArrayList<>();
+
+ searcher.setSimilarity(new BM25Similarity(5.0f, 0.2f)); // Stronger term frequency weighting
+
+ long startTime = System.nanoTime();
+ for (int i = 0; i < 10; i++) {
+ futures.add(executor.submit(() -> anytimeSearcher.search(query, rangeScores, reader.maxDoc())));
+ }
+
+ for (Future<TopDocs> future : futures) {
+ TopDocs results = future.get();
+ assertNotNull("Results should not be null", results);
+ assertTrue("At least one result should be retrieved", results.scoreDocs.length > 0);
+ }
+ executor.shutdown();
+ long duration = (System.nanoTime() - startTime) / 1_000_000;
+ System.out.println("Total query execution time: " + duration + " ms");
+ }
+ }
+
+ public void testDifferentQueries() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex2")
+ .toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+
+ for (int i = 1; i <= 10000; i++) {
+ String content = "Lucene document " + i + " " + " ranking relevance " + TestUtil.randomSimpleString(random());
+ addDocument(writer, content, i);
+ }
+
+ writer.commit();
+ }
+
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ Query[] queries = {
+ new TermQuery(new Term("content", "lucene")),
+ new TermQuery(new Term("content", "document")),
+ new TermQuery(new Term("content", "relevance"))
+ };
+
+ for (Query query : queries) {
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 100, "content");
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+ assertNotNull("Results should not be null", results);
+ assertTrue("At least one result should be retrieved", results.scoreDocs.length > 0);
+ }
+ }
+ }
+
+ public void testEmptyIndex() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex3")
+ .toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+ writer.commit(); // Commit empty index
+ }
+
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+ Query query = new TermQuery(new Term("content", "lucene"));
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 50, "content");
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+ assertNotNull("Results should not be null", results);
+ assertEquals("No results should be found in an empty index", 0, results.totalHits.value());
+ }
+ }
+
+ public void testConcurrentQueries() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex4")
+ .toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+
+ for (int i = 1; i <= 10000; i++) {
+ String content = "Lucene document " + i + " "
+ + " ranking relevance " + " " + TestUtil.randomSimpleString(random());
+ addDocument(writer, content, i);
+ }
+
+ writer.commit();
+ }
+
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ Query[] queries = {
+ new TermQuery(new Term("content", "lucene")),
+ new TermQuery(new Term("content", "document")),
+ new TermQuery(new Term("content", "ranking"))
+ };
+
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 50, "content");
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ List<Future<TopDocs>> futures = new ArrayList<>();
+
+ for (Query query : queries) {
+ futures.add(executor.submit(() -> anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc())));
+ }
+
+ for (Future<TopDocs> future : futures) {
+ TopDocs results = future.get();
+ assertNotNull("Results should not be null", results);
+ System.out.println("Concurrent query executed with " + results.totalHits.value() + " results.");
+ }
+ executor.shutdown();
+ }
+ }
+
+ @Test
+ public void testSlaCutoffTriggersEarlyTermination() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex5")
+ .toAbsolutePath().toString();
+
+ try (Directory directory = FSDirectory.open(Paths.get(indexPath));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new StandardAnalyzer()))) {
+
+ for (int i = 1; i <= 10000; i++) {
+ String content = "Lucene document " + i + " " + " ranking relevance "
+ + " " + TestUtil.randomSimpleString(random());
+ addDocument(writer, content, i);
+ }
+
+ writer.commit();
+ }
+
+ try (IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(indexPath)))) {
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ Query query = new TermQuery(new Term("content", "lucene"));
+ AnytimeRankingSearcher anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 1, "content"); // Intentionally tight SLA
+ TopDocs results = anytimeSearcher.search(query, new HashMap<>(), reader.maxDoc());
+
+ assertNotNull(results);
+ assertTrue("Expect partial results due to SLA cutoff", results.scoreDocs.length > 0);
+ }
+ }
+
+
+ private void addDocument(IndexWriter writer, String content, int docID) throws IOException {
+ Document doc = new Document();
+ doc.add(new IntPoint("docID", docID)); // Enables range queries
+ doc.add(new StoredField("docID", docID)); // Allows retrieval
+ doc.add(new TextField("content", content, Field.Store.YES));
+ writer.addDocument(doc);
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/util/TestHeuristicSLAEstimator.java b/lucene/core/src/test/org/apache/lucene/util/TestHeuristicSLAEstimator.java
new file mode 100644
index 00000000000..fbd9a0454f3
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/TestHeuristicSLAEstimator.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.util;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+public class TestHeuristicSLAEstimator extends LuceneTestCase {
+
+ private static final String FIELD = "content";
+ private IndexReader reader;
+ private HeuristicSLAEstimator estimator;
+
+ @Before
+ public void setup() throws Exception {
+ String indexPath = Files.createTempDirectory("testindex1")
+ .toAbsolutePath().toString();
+ Directory dir = FSDirectory.open(Paths.get(indexPath));
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+ IndexWriter writer = new IndexWriter(dir, config);
+
+ for (int i = 0; i < 100; i++) {
+ Document doc = new Document();
+ doc.add(new TextField(FIELD,
+ "lucene search relevance ranking performance document", Field.Store.NO));
+ writer.addDocument(doc);
+ }
+
+ writer.commit();
+ writer.close();
+
+ reader = DirectoryReader.open(dir);
+ estimator = new HeuristicSLAEstimator(FIELD);
+ }
+
+ @Test
+ public void testTermQueryEstimate() throws IOException {
+ Query query = new TermQuery(new Term(FIELD, "lucene"));
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("Estimate should be >= base SLA", estimate >= 100);
+ }
+
+ @Test
+ public void testPhraseQueryEstimate() throws IOException {
+ PhraseQuery query = new PhraseQuery(FIELD, "lucene", "search", "ranking");
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("Phrase query should increase SLA", estimate > 100);
+ }
+
+ @Test
+ public void testBooleanQueryEstimate() throws IOException {
+ BooleanQuery.Builder builder = new BooleanQuery.Builder();
+ builder.add(new TermQuery(new Term(FIELD, "lucene")), BooleanClause.Occur.MUST);
+ builder.add(new TermQuery(new Term(FIELD, "search")), BooleanClause.Occur.SHOULD);
+ Query query = builder.build();
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("Boolean query should increase SLA", estimate > 100);
+ }
+
+ @Test
+ public void testDisjunctionMaxQueryEstimate() throws IOException {
+ DisjunctionMaxQuery query = new DisjunctionMaxQuery(
+ java.util.Arrays.asList(
+ new TermQuery(new Term(FIELD, "lucene")),
+ new TermQuery(new Term(FIELD, "ranking"))
+ ), 0.1f);
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("DisjunctionMaxQuery should increase SLA", estimate > 100);
+ }
+
+ @Test
+ public void testNestedBooleanQueryEstimate() throws IOException {
+ BooleanQuery.Builder inner = new BooleanQuery.Builder();
+ inner.add(new TermQuery(new Term(FIELD, "lucene")), BooleanClause.Occur.MUST);
+ inner.add(new TermQuery(new Term(FIELD, "search")), BooleanClause.Occur.MUST);
+
+ BooleanQuery.Builder outer = new BooleanQuery.Builder();
+ outer.add(inner.build(), BooleanClause.Occur.MUST);
+ outer.add(new TermQuery(new Term(FIELD, "ranking")), BooleanClause.Occur.SHOULD);
+
+ Query query = outer.build();
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("Nested Boolean query should increase SLA", estimate > 100);
+ }
+
+ @Test
+ public void testMatchAllDocsQueryEstimate() throws IOException {
+ Query query = new MatchAllDocsQuery();
+ double estimate = estimator.estimate(query, reader, 100);
+ assertTrue("MatchAllDocsQuery should be close to base SLA", estimate >= 100);
+ }
+}