You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by at...@apache.org on 2025/04/18 10:10:30 UTC
(lucene) branch index_binning_changes updated: Add JMH benchmarks and tidy output
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch index_binning_changes
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/index_binning_changes by this push:
new cb0b2c38649 Add JMH benchmarks and tidy output
cb0b2c38649 is described below
commit cb0b2c3864941c731636ab4071c790efd709b5e6
Author: Atri Sharma <at...@Atris-MacBook-Pro-3.local>
AuthorDate: Fri Apr 18 15:40:15 2025 +0530
Add JMH benchmarks and tidy output
---
.../jmh/AnytimeMultiSegmentBenchmark.java | 160 ++++++++++++++++++++
.../jmh/AnytimeQueryLatencyVarianceBenchmark.java | 163 ++++++++++++++++++++
.../benchmark/jmh/AnytimeRankingBenchmark.java | 164 +++++++++++++++++++++
.../jmh/BinningIndexingLatencyBenchmark.java | 152 +++++++++++++++++++
.../apache/lucene/index/BinScoreLeafReader.java | 6 +-
.../java/org/apache/lucene/index/BinScoreUtil.java | 46 +++---
.../lucene/search/AnytimeRankingCollector.java | 13 +-
.../search/TestAnytimeRankingRelevanceSearch.java | 14 +-
.../lucene/search/TestAnytimeRankingSearch.java | 3 -
.../lucene/search/TestBinningRelevanceBoost.java | 5 +-
10 files changed, 681 insertions(+), 45 deletions(-)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
new file mode 100644
index 00000000000..d79683f4c14
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmark for multi-segment bin-aware search with AnytimeRanking. */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeMultiSegmentBenchmark {
+
+ private Directory dir;
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "5000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binning;
+
+ @Setup
+ public void setup() throws Exception {
+ Path path = Files.createTempDirectory("benchmark-segments");
+ dir = new MMapDirectory(path);
+ IndexWriterConfig iwc = new IndexWriterConfig(new FixedTokenAnalyzer());
+ iwc.setCodec(new Lucene101Codec());
+ iwc.setUseCompoundFile(false);
+ iwc.setMaxBufferedDocs(50); // force multiple flushes
+
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setTokenized(true);
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binning) {
+ type.putAttribute("postingsFormat", "Lucene101");
+ type.putAttribute("doBinning", "true");
+ type.putAttribute("bin.count", "4");
+ }
+ type.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene fast ranking test" : "filler junk";
+ doc.add(new Field("field", content, type));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineSearch() throws IOException {
+ return searcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeSearch() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws Exception {
+ reader.close();
+ dir.close();
+ }
+
+ /** Simple analyzer that emits a single token. */
+ private static final class FixedTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
new file mode 100644
index 00000000000..1a90fe85952
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeQueryLatencyVarianceBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Setup
+ public void setup() throws Exception {
+ Path tempDir = Files.createTempDirectory("query-latency-variance");
+ directory = new MMapDirectory(tempDir);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(64);
+
+ FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binningEnabled) {
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
+ }
+ fieldType.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene latency benchmark" : "random filler content";
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineQueryLatency() throws IOException {
+ return baselineSearcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeQueryLatency() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+ }
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
new file mode 100644
index 00000000000..bf764226c5b
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeRankingBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Setup
+ public void setup() throws Exception {
+ Path tempDir = Files.createTempDirectory("anytime-benchmark");
+ directory = new MMapDirectory(tempDir);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+
+ FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binningEnabled) {
+ fieldType.putAttribute("postingsFormat", "Lucene101");
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
+ }
+ fieldType.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 5 == 0) ? "lucene search fast scoring" : "noise filler";
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ writer.forceMerge(1);
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineLuceneSearch() throws Exception {
+ return baselineSearcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeRankingSearch() throws Exception {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws Exception {
+ reader.close();
+ directory.close();
+ }
+
+ /**
+ * Minimal analyzer that emits a single token — avoids dependency on any Lucene analyzer modules.
+ */
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean done = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (done) return false;
+ clearAttributes();
+ termAttr.append("lucene"); // emit single fixed token
+ done = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ done = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+ }
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
new file mode 100644
index 00000000000..852d3ecc4eb
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.concurrent.TimeUnit;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Thread)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class BinningIndexingLatencyBenchmark {
+
+ @Param({"10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Param({"exact", "approx", "auto"})
+ private String graphBuilder;
+
+ private Path tempDir;
+ private FieldType fieldType;
+
+ @Setup(Level.Trial)
+ public void setup() {
+ fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+
+ if (binningEnabled) {
+ fieldType.putAttribute("postingsFormat", "Lucene101");
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
+ fieldType.putAttribute("graph.builder", graphBuilder);
+ }
+
+ fieldType.freeze();
+ }
+
+ @Benchmark
+ public void indexBatch() throws IOException {
+ tempDir = Files.createTempDirectory("lucene-binning-benchmark");
+ try (Directory dir = new MMapDirectory(tempDir)) {
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 200 == 0) ? "lucene relevant content" : "noise filler text " + i;
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+ } finally {
+ Files.walk(tempDir)
+ .sorted(Comparator.reverseOrder())
+ .forEach(
+ path -> {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException _) {
+ // best-effort cleanup
+ }
+ });
+ }
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ attr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
index 83a76948c35..e9456d15ea5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import org.apache.lucene.util.IOUtils;
/**
- * Reader wrapper that exposes bin-aware scoring.
- * Supports resource tracking for clean closure of auxiliary components.
+ * Reader wrapper that exposes bin-aware scoring. Supports resource tracking for clean closure of
+ * auxiliary components.
*/
public class BinScoreLeafReader extends FilterLeafReader {
@@ -61,4 +61,4 @@ public class BinScoreLeafReader extends FilterLeafReader {
public CacheHelper getCoreCacheHelper() {
return in.getCoreCacheHelper();
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
index dedaa75d65f..7161ea94d43 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
@@ -6,17 +6,16 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.WeakHashMap;
import java.util.Random;
-
+import java.util.WeakHashMap;
import org.apache.lucene.codecs.BinMapReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
/**
- * Utility methods for wrapping readers with bin-aware scoring support.
- * Enables loading bin map metadata and injecting scoring logic at search time.
+ * Utility methods for wrapping readers with bin-aware scoring support. Enables loading bin map
+ * metadata and injecting scoring logic at search time.
*/
public final class BinScoreUtil {
@@ -29,8 +28,8 @@ public final class BinScoreUtil {
private BinScoreUtil() {}
/**
- * Wraps an IndexReader with bin-aware scoring if binmap metadata is present.
- * Supports both LeafReader and CompositeReader types.
+ * Wraps an IndexReader with bin-aware scoring if binmap metadata is present. Supports both
+ * LeafReader and CompositeReader types.
*
* @param reader the input IndexReader
* @return wrapped reader if binmap is found; otherwise original reader
@@ -47,10 +46,9 @@ public final class BinScoreUtil {
}
/**
- * Wraps a CompositeReader (e.g., DirectoryReader) with bin-aware logic.
- * Leaf readers are shuffled to mitigate early termination bias, and each is
- * wrapped individually. All associated compound readers are tracked and must
- * be closed via {@link #closeResources}.
+ * Wraps a CompositeReader (e.g., DirectoryReader) with bin-aware logic. Leaf readers are shuffled
+ * to mitigate early termination bias, and each is wrapped individually. All associated compound
+ * readers are tracked and must be closed via {@link #closeResources}.
*
* @param reader composite reader to wrap
* @return MultiReader with bin-aware leaves
@@ -78,8 +76,8 @@ public final class BinScoreUtil {
}
/**
- * Wraps a single LeafReader with bin-aware scoring logic, collecting any
- * compound readers created for later cleanup.
+ * Wraps a single LeafReader with bin-aware scoring logic, collecting any compound readers created
+ * for later cleanup.
*
* @param reader the input LeafReader
* @param closables list to track compound readers for external cleanup
@@ -110,8 +108,10 @@ public final class BinScoreUtil {
return reader;
}
- String suffix = binmapFile.substring(name.length() + 1, binmapFile.length() - ".binmap".length());
- SegmentReadState state = new SegmentReadState(dir, info, sr.getFieldInfos(), IOContext.READONCE, suffix);
+ String suffix =
+ binmapFile.substring(name.length() + 1, binmapFile.length() - ".binmap".length());
+ SegmentReadState state =
+ new SegmentReadState(dir, info, sr.getFieldInfos(), IOContext.READONCE, suffix);
BinMapReader binMap = null;
try {
@@ -130,8 +130,8 @@ public final class BinScoreUtil {
}
/**
- * Wraps a LeafReader and tracks compound files internally.
- * This method is intended for direct wrapping of single-segment readers.
+ * Wraps a LeafReader and tracks compound files internally. This method is intended for direct
+ * wrapping of single-segment readers.
*
* @param reader input reader
* @return bin-aware reader or original
@@ -141,18 +141,14 @@ public final class BinScoreUtil {
return wrap(reader, new ArrayList<>());
}
- /**
- * Extracts the SegmentReader from the given reader if available.
- */
+ /** Extracts the SegmentReader from the given reader if available. */
private static SegmentReader getSegmentReader(LeafReader reader) {
if (reader instanceof SegmentReader) return (SegmentReader) reader;
LeafReader unwrapped = FilterLeafReader.unwrap(reader);
return (unwrapped instanceof SegmentReader) ? (SegmentReader) unwrapped : null;
}
- /**
- * Finds a binmap file in the given directory for the specified segment.
- */
+ /** Finds a binmap file in the given directory for the specified segment. */
private static String findBinmapFile(Directory dir, String segmentName) throws IOException {
for (String file : dir.listAll()) {
if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
@@ -180,8 +176,8 @@ public final class BinScoreUtil {
}
/**
- * Closes all resources (e.g., compound readers) associated with the wrapped IndexReader.
- * This method must be invoked before or during IndexReader#close to ensure full cleanup.
+ * Closes all resources (e.g., compound readers) associated with the wrapped IndexReader. This
+ * method must be invoked before or during IndexReader#close to ensure full cleanup.
*
* @param reader the top-level MultiReader returned by {@link #wrap(IndexReader)}
* @throws IOException on error
@@ -194,4 +190,4 @@ public final class BinScoreUtil {
}
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
index 40f946f4347..8221a8ed657 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Locale;
-
import org.apache.lucene.index.BinScoreLeafReader;
import org.apache.lucene.index.BinScoreReader;
import org.apache.lucene.index.LeafReader;
@@ -81,7 +80,8 @@ public final class AnytimeRankingCollector extends TopDocsCollector<ScoreDoc> {
: null;
final int interval =
- Math.max(MIN_CHECK_INTERVAL, Math.min(MAX_CHECK_INTERVAL, reader.numDocs() / DOCS_PER_CHECK));
+ Math.max(
+ MIN_CHECK_INTERVAL, Math.min(MAX_CHECK_INTERVAL, reader.numDocs() / DOCS_PER_CHECK));
return new LeafCollector() {
private Scorable scorer;
@@ -109,8 +109,11 @@ public final class AnytimeRankingCollector extends TopDocsCollector<ScoreDoc> {
System.out.printf(
Locale.ROOT,
"doc=%d raw=%.4f boost=%.2f bin=%d final=%.4f%n",
- doc + base, rawScore, boost, bin, finalScore
- );
+ doc + base,
+ rawScore,
+ boost,
+ bin,
+ finalScore);
if (finalScore <= 0 || Float.isNaN(finalScore)) {
return;
@@ -131,4 +134,4 @@ public final class AnytimeRankingCollector extends TopDocsCollector<ScoreDoc> {
public ScoreMode scoreMode() {
return ScoreMode.TOP_SCORES;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java
index 7b675073b44..ff486a7fd7c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceSearch.java
@@ -178,15 +178,15 @@ public class TestAnytimeRankingRelevanceSearch extends LuceneTestCase {
}
IndexReader reader = DirectoryReader.open(dir);
- assertTrue("Should have multiple segments", reader.leaves().size() > 1);
+ assertTrue("Should have multiple segments", reader.leaves().size() > 1);
- try (AnytimeRankingSearcher anytimeSearcher =
- new AnytimeRankingSearcher(reader, 10, 50, "content")) {
- TopDocs topDocs = anytimeSearcher.search(new TermQuery(new Term("content", "lucene")));
+ try (AnytimeRankingSearcher anytimeSearcher =
+ new AnytimeRankingSearcher(reader, 10, 50, "content")) {
+ TopDocs topDocs = anytimeSearcher.search(new TermQuery(new Term("content", "lucene")));
- assertNotNull("TopDocs should not be null", topDocs);
- assertTrue("Should return some results", topDocs.scoreDocs.length > 0);
- }
+ assertNotNull("TopDocs should not be null", topDocs);
+ assertTrue("Should return some results", topDocs.scoreDocs.length > 0);
+ }
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
index 2010bf8e1de..13867d34baa 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
@@ -32,15 +32,12 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.StoredField;
-import org.apache.lucene.index.BinScoreReader;
import org.apache.lucene.index.BinScoreUtil;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
index 598112bc21a..cd975f934e5 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
@@ -121,8 +121,9 @@ public class TestBinningRelevanceBoost extends LuceneTestCase {
}
}
-assertTrue("ARCANE hits=" + anytimeRankingHits + " baseline=" + baselineHits,
- anytimeRankingHits >= baselineHits.get() - 1); // allow small variance
+ assertTrue(
+ "ARCANE hits=" + anytimeRankingHits + " baseline=" + baselineHits,
+ anytimeRankingHits >= baselineHits.get() - 1); // allow small variance
reader.close();
dir.close();