You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by at...@apache.org on 2025/04/17 18:28:21 UTC
(lucene) 02/18: Add more benchmarks and relevance tests
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit b56a562c87187739f637e753ec77b9a86bb12ac1
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Wed Apr 16 21:21:08 2025 +0530
Add more benchmarks and relevance tests
---
.../jmh/AnytimeMultiSegmentBenchmark.java | 161 +++++++++++++++
.../jmh/AnytimeQueryLatencyVarianceBenchmark.java | 164 ++++++++++++++++
.../jmh/AnytimeRankingRecallStressBenchmark.java | 217 +++++++++++++++++++++
.../lucene/codecs/ApproximateDocGraphBuilder.java | 1 +
.../search/TestAnytimeRankingRelevanceMatch.java | 155 +++++++++++++++
.../lucene/search/TestArcaneRelevanceBoost.java | 130 ++++++++++++
6 files changed, 828 insertions(+)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
new file mode 100644
index 00000000000..09c3884b23c
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmark for multi-segment bin-aware search with AnytimeRanking. */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeMultiSegmentBenchmark {
+
+ private Directory dir;
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "5000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binning;
+
+ @Setup
+ public void setup() throws Exception {
+ Path path = Files.createTempDirectory("benchmark-segments");
+ dir = new MMapDirectory(path);
+ IndexWriterConfig iwc = new IndexWriterConfig(new FixedTokenAnalyzer());
+ iwc.setCodec(new Lucene101Codec());
+ iwc.setUseCompoundFile(false);
+ iwc.setMaxBufferedDocs(50); // force multiple flushes
+
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setTokenized(true);
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binning) {
+ type.putAttribute("postingsFormat", "Lucene101");
+ type.putAttribute("doBinning", "true");
+ type.putAttribute("bin.count", "4");
+ }
+ type.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene fast ranking test" : "filler junk";
+ doc.add(new Field("field", content, type));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineSearch() throws IOException {
+ return searcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeSearch() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws Exception {
+ reader.close();
+ dir.close();
+ }
+
+ /** Simple analyzer that emits a single token. */
+ private static final class FixedTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
new file mode 100644
index 00000000000..0e70b3f0f8f
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeQueryLatencyVarianceBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Setup
+ public void setup() throws Exception {
+ Path tempDir = Files.createTempDirectory("query-latency-variance");
+ directory = new MMapDirectory(tempDir);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(64);
+
+ FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binningEnabled) {
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
+ }
+ fieldType.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene latency benchmark" : "random filler content";
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineQueryLatency() throws IOException {
+ return baselineSearcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeQueryLatency() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
new file mode 100644
index 00000000000..d51a6d3582f
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 1)
+@Measurement(iterations = 3)
+@Fork(1)
+public class AnytimeRankingRecallStressBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"20000"})
+ private int docCount;
+
+ @Param({"1", "3"})
+ private int sla;
+
+ @Setup
+ public void setup() throws Exception {
+ Path path = Files.createTempDirectory("recall-stress");
+ directory = new MMapDirectory(path);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(100);
+
+ FieldType ft = new FieldType();
+ ft.setTokenized(true);
+ ft.setStored(false);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 200 == 0) ? "lucene precision recall document" : "random junk text";
+ doc.add(new Field("field", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, sla, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public float baselineRecall() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return countRelevant(topDocs) / 10.0f;
+ }
+
+ @Benchmark
+ public float anytimeRecall() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return countRelevant(topDocs) / 10.0f;
+ }
+
+ @Benchmark
+ public float baselineMRR() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return computeMRR(topDocs);
+ }
+
+ @Benchmark
+ public float anytimeMRR() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return computeMRR(topDocs);
+ }
+
+ @Benchmark
+ public float baselineNDCG() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return computeNDCG(topDocs);
+ }
+
+ @Benchmark
+ public float anytimeNDCG() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return computeNDCG(topDocs);
+ }
+
+ private int countRelevant(TopDocs topDocs) {
+ int relevant = 0;
+ for (ScoreDoc doc : topDocs.scoreDocs) {
+ if (doc.doc % 200 == 0) {
+ relevant++;
+ }
+ }
+ return relevant;
+ }
+
+ private float computeMRR(TopDocs topDocs) {
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ if (topDocs.scoreDocs[i].doc % 200 == 0) {
+ return 1.0f / (i + 1);
+ }
+ }
+ return 0f;
+ }
+
+ private float computeNDCG(TopDocs topDocs) {
+ float dcg = 0f;
+ float idcg = 0f;
+ int count = 0;
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ int rank = i + 1;
+ if (topDocs.scoreDocs[i].doc % 200 == 0) {
+ dcg += 1.0 / (Math.log(rank + 1) / Math.log(2));
+ count++;
+ }
+ }
+ for (int j = 1; j <= count; j++) {
+ idcg += 1.0 / (Math.log(j + 1) / Math.log(2));
+ }
+ return idcg > 0 ? dcg / idcg : 0f;
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
index 9a6cb45de47..e60b08fc062 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
@@ -29,6 +29,7 @@ import org.apache.lucene.util.BytesRef;
*/
public final class ApproximateDocGraphBuilder {
+ /* Default max number of edges per doc */
public static final int DEFAULT_MAX_EDGES = 10;
private final String field;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
new file mode 100644
index 00000000000..f2be02f635d
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.Test;
+
+public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
+
+ private static final int DOC_COUNT = 20000;
+ private static final int RELEVANT_FREQ = 200;
+
+ @Test
+ public void testRelevanceMetricsComparison() throws Exception {
+ Directory dir = new MMapDirectory(createTempDir("relevance-test"));
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(100);
+
+ FieldType ft = new FieldType();
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.setTokenized(true);
+ ft.setStored(false);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int i = 0; i < DOC_COUNT; i++) {
+ Document doc = new Document();
+ String content = (i % RELEVANT_FREQ == 0) ? "lucene precision recall document" : "random filler content";
+ doc.add(new Field("field", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher anytime = new AnytimeRankingSearcher(baselineSearcher, 10, 3, "field");
+
+ TermQuery query = new TermQuery(new Term("field", "lucene"));
+ TopDocs baseline = baselineSearcher.search(query, 10);
+ TopDocs anytimeResults = anytime.search(query);
+
+ float baselineRecall = countRelevant(baseline) / 10.0f;
+ float anytimeRecall = countRelevant(anytimeResults) / 10.0f;
+
+ float baselineMRR = computeMRR(baseline);
+ float anytimeMRR = computeMRR(anytimeResults);
+
+ float baselineNDCG = computeNDCG(baseline);
+ float anytimeNDCG = computeNDCG(anytimeResults);
+
+ System.out.printf(Locale.ROOT,
+ "Recall: baseline=%.2f anytime=%.2f | MRR: baseline=%.2f anytime=%.2f | NDCG: baseline=%.2f anytime=%.2f%n",
+ baselineRecall, anytimeRecall, baselineMRR, anytimeMRR, baselineNDCG, anytimeNDCG);
+
+ assertTrue("Anytime recall should be >= baseline", anytimeRecall >= baselineRecall);
+ assertTrue("Anytime MRR should be >= baseline", anytimeMRR >= baselineMRR);
+ assertTrue("Anytime NDCG should be >= baseline", anytimeNDCG >= baselineNDCG);
+
+ reader.close();
+ dir.close();
+ }
+
+ private int countRelevant(TopDocs topDocs) {
+ int relevant = 0;
+ for (ScoreDoc doc : topDocs.scoreDocs) {
+ if (doc.doc % RELEVANT_FREQ == 0) {
+ relevant++;
+ }
+ }
+ return relevant;
+ }
+
+ private float computeMRR(TopDocs topDocs) {
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ if (topDocs.scoreDocs[i].doc % RELEVANT_FREQ == 0) {
+ return 1.0f / (i + 1);
+ }
+ }
+ return 0f;
+ }
+
+ private float computeNDCG(TopDocs topDocs) {
+ float dcg = 0f;
+ float idcg = 0f;
+ int count = 0;
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ int rank = i + 1;
+ if (topDocs.scoreDocs[i].doc % RELEVANT_FREQ == 0) {
+ dcg += (float) (1.0 / (Math.log(rank + 1) / Math.log(2)));
+ count++;
+ }
+ }
+ for (int j = 1; j <= count; j++) {
+ idcg += (float) (1.0 / (Math.log(j + 1) / Math.log(2)));
+ }
+ return idcg > 0 ? dcg / idcg : 0f;
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
new file mode 100644
index 00000000000..5fe2bf188c3
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+public class TestArcaneRelevanceBoost extends LuceneTestCase {
+
+ public void testArcaneImprovesRecallUnderTruncation() throws Exception {
+ final int totalDocs = 5000;
+ final int relevantEvery = 200;
+ final int truncationLimit = 20;
+
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ iwc.setCodec(TestUtil.getDefaultCodec());
+ iwc.setMaxBufferedDocs(100);
+ iwc.setUseCompoundFile(false);
+
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.setTokenized(true);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc)) {
+ for (int i = 0; i < totalDocs; i++) {
+ Document doc = new Document();
+ String content = (i % relevantEvery == 0) ? "lucene highly relevant text" : "noise filler irrelevant";
+ doc.add(new StoredField("docID", i));
+ doc.add(new Field("content", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher baseline = newSearcher(reader);
+ baseline.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher arcane =
+ new AnytimeRankingSearcher(baseline, 10, truncationLimit, "content");
+
+ TermQuery query = new TermQuery(new Term("content", "lucene"));
+
+ AtomicInteger baselineHits = new AtomicInteger();
+ baseline.search(query, new CollectorManager<SimpleCollector, Void>() {
+ @Override
+ public SimpleCollector newCollector() {
+ return new SimpleCollector() {
+ private int seen = 0;
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (doc % relevantEvery == 0) {
+ baselineHits.incrementAndGet();
+ }
+ if (++seen >= truncationLimit) {
+ throw new CollectionTerminatedException();
+ }
+ }
+
+ @Override
+ public void doSetNextReader(LeafReaderContext context) {}
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.COMPLETE_NO_SCORES;
+ }
+ };
+ }
+
+ @Override
+ public Void reduce(Collection<SimpleCollector> collectors) {
+ return null;
+ }
+ });
+
+ TopDocs arcaneResults = arcane.search(query);
+ int arcaneHits = 0;
+ for (ScoreDoc sd : arcaneResults.scoreDocs) {
+ if (sd.doc % relevantEvery == 0) {
+ arcaneHits++;
+ }
+ }
+
+ assertTrue(
+ "ARCANE should retrieve more relevant results under truncation. "
+ + "Baseline hits=" + baselineHits.get() + ", ARCANE hits=" + arcaneHits,
+ arcaneHits > baselineHits.get());
+
+ reader.close();
+ dir.close();
+ }
+}
\ No newline at end of file