You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by at...@apache.org on 2025/04/17 18:28:19 UTC
(lucene) branch new_index_query_include updated (e9508ac5073 -> 915c76816c0)
This is an automated email from the ASF dual-hosted git repository.
atri pushed a change to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
from e9508ac5073 Update benchmarks, introduce approx and add near duplicate detection
new d4c8927faab Tidy
new b56a562c871 Add more benchmarks and relevance tests
new 88f60460257 Remove redundant benchmarks
new 0bd1a10f073 Cleanup, tidy output and add javadocs
new dd15761120a More cleanups
new 794bec105d2 More cleanups
new d5da46ab01d more cleanup
new e25e6f70938 And more cleanup
new 64093d6869b More cleanup
new 66ef86399fb Continued cleanups
new c9be5c0d303 More cleanup
new 9febffa50e4 cleanup and add licenses
new 727bd7678e8 More cleanup
new d66c37d6993 More cleanup and tidy output
new ab2b96d0756 More cleanup
new 5ba124b661c Do not close twice
new 921ff070331 Do not close if prior write failed
new 915c76816c0 intermittent commit
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/java/DocBinningCostBenchmark.java | 60 +-
.../jmh/AnytimeMultiSegmentBenchmark.java | 160 +++++
...a => AnytimeQueryLatencyVarianceBenchmark.java} | 131 ++--
.../benchmark/jmh/AnytimeRankingBenchmark.java | 66 +-
.../jmh/BinningIndexingLatencyBenchmark.java | 61 +-
.../benchmark/jmh/IndexingLatencyBenchmark.java | 149 -----
.../apache/lucene/codecs/ApproximateDocBinner.java | 10 +-
.../lucene/codecs/ApproximateDocGraphBuilder.java | 24 +-
.../org/apache/lucene/codecs/BinMapReader.java | 53 +-
.../org/apache/lucene/codecs/BinMapWriter.java | 50 +-
.../apache/lucene/codecs/BinQualityMetrics.java | 30 +-
.../org/apache/lucene/codecs/BinningSupport.java | 7 -
.../java/org/apache/lucene/codecs/BinningUtil.java | 40 --
.../lucene/codecs/DocBinningGraphBuilder.java | 56 +-
.../org/apache/lucene/codecs/DocGraphBuilder.java | 20 +-
.../org/apache/lucene/codecs/FieldsLeafReader.java | 6 +
.../lucene/codecs/InMemorySparseEdgeGraph.java | 4 +-
.../org/apache/lucene/codecs/SparseEdgeGraph.java | 40 +-
.../codecs/lucene101/Lucene101PostingsFormat.java | 569 ++++++++---------
.../codecs/lucene101/Lucene101PostingsWriter.java | 2 -
.../apache/lucene/index/BinScoreLeafReader.java | 1 +
.../java/org/apache/lucene/index/BinScoreUtil.java | 30 +-
.../lucene/search/AnytimeRankingCollector.java | 16 +
.../search/AnytimeRankingCollectorManager.java | 16 +
.../lucene/search/AnytimeRankingSearcher.java | 16 +
.../apache/lucene/search/BinBoostCalculator.java | 5 +-
.../org/apache/lucene/search/DuplicateCluster.java | 28 -
.../apache/lucene/search/NearDuplicateFinder.java | 130 ----
.../lucene/codecs/TestBinQualityMetrics.java | 22 +-
.../lucene/codecs/TestDocBinningGraphBuilder.java | 16 +
.../apache/lucene/codecs/TestDocGraphBuilder.java | 16 +
.../lucene101/TestLucene101PostingsFormat.java | 709 ++++++++++-----------
.../org/apache/lucene/index/TestBinScoreUtil.java | 6 +-
.../lucene/search/TestAnytimeRankingNDCG.java | 134 ++--
.../search/TestAnytimeRankingRelevanceMatch.java | 121 ++++
.../lucene/search/TestBinningRelevanceBoost.java | 135 ++++
.../lucene/search/TestNearDuplicateFinder.java | 111 ----
37 files changed, 1631 insertions(+), 1419 deletions(-)
create mode 100644 lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
rename lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/{AnytimeSLARelevanceBenchmark.java => AnytimeQueryLatencyVarianceBenchmark.java} (50%)
delete mode 100644 lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/BinningSupport.java
delete mode 100644 lucene/core/src/java/org/apache/lucene/codecs/BinningUtil.java
delete mode 100644 lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
delete mode 100644 lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
create mode 100644 lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
delete mode 100644 lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
(lucene) 01/18: Tidy
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit d4c8927faab63bf0b3ecc73305e136d717fbfa89
Author: Atri Sharma <at...@Atris-MacBook-Pro.local>
AuthorDate: Fri Apr 11 01:52:21 2025 +0530
Tidy
---
.../src/java/DocBinningCostBenchmark.java | 60 +-
.../benchmark/jmh/AnytimeRankingBenchmark.java | 50 +-
.../jmh/AnytimeSLARelevanceBenchmark.java | 49 +-
.../jmh/BinningIndexingLatencyBenchmark.java | 61 +-
.../benchmark/jmh/IndexingLatencyBenchmark.java | 62 +-
.../apache/lucene/codecs/ApproximateDocBinner.java | 10 +-
.../lucene/codecs/ApproximateDocGraphBuilder.java | 6 +-
.../apache/lucene/codecs/BinQualityMetrics.java | 14 +-
.../lucene/codecs/DocBinningGraphBuilder.java | 9 +-
.../org/apache/lucene/codecs/DocGraphBuilder.java | 15 +-
.../codecs/lucene101/Lucene101PostingsFormat.java | 563 ++++++++--------
.../java/org/apache/lucene/index/BinScoreUtil.java | 14 +-
.../org/apache/lucene/search/DuplicateCluster.java | 10 +-
.../apache/lucene/search/NearDuplicateFinder.java | 9 +-
.../lucene/codecs/TestBinQualityMetrics.java | 6 +-
.../lucene101/TestLucene101PostingsFormat.java | 709 ++++++++++-----------
.../lucene/search/TestAnytimeRankingNDCG.java | 134 ++--
.../lucene/search/TestNearDuplicateFinder.java | 7 +-
18 files changed, 865 insertions(+), 923 deletions(-)
diff --git a/lucene/benchmark-jmh/src/java/DocBinningCostBenchmark.java b/lucene/benchmark-jmh/src/java/DocBinningCostBenchmark.java
index 7a646ae44d3..409dfa9be0b 100644
--- a/lucene/benchmark-jmh/src/java/DocBinningCostBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/DocBinningCostBenchmark.java
@@ -20,9 +20,7 @@ package org.apache.lucene.benchmark.jmh;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.Random;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -30,6 +28,7 @@ import org.apache.lucene.codecs.ApproximateDocBinner;
import org.apache.lucene.codecs.ApproximateDocGraphBuilder;
import org.apache.lucene.codecs.DocBinningGraphBuilder;
import org.apache.lucene.codecs.DocGraphBuilder;
+import org.apache.lucene.codecs.FieldsLeafReader;
import org.apache.lucene.codecs.SparseEdgeGraph;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
@@ -37,8 +36,11 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.openjdk.jmh.annotations.Benchmark;
@@ -54,12 +56,6 @@ import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.codecs.FieldsLeafReader;
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@@ -121,7 +117,8 @@ public class DocBinningCostBenchmark {
@Benchmark
public int[] approximateGraphBinning() throws IOException {
- ApproximateDocGraphBuilder builder = new ApproximateDocGraphBuilder(FIELD, DocGraphBuilder.DEFAULT_MAX_EDGES);
+ ApproximateDocGraphBuilder builder =
+ new ApproximateDocGraphBuilder(FIELD, DocGraphBuilder.DEFAULT_MAX_EDGES);
SparseEdgeGraph graph = builder.build(leafReader);
return ApproximateDocBinner.assign(graph, docCount, binCount);
}
@@ -132,32 +129,31 @@ public class DocBinningCostBenchmark {
directory.close();
}
- /**
- * Minimal synthetic analyzer that emits a fixed token — avoids analyzer module dependency.
- */
+ /** Minimal synthetic analyzer that emits a fixed token — avoids analyzer module dependency. */
private static final class SingleTokenAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new Tokenizer() {
- private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) return false;
- clearAttributes();
- attr.append(TOKEN);
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ attr.append(TOKEN);
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
return new TokenStreamComponents(tokenizer);
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
index 651bd65961e..297bec90bb7 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
@@ -2,11 +2,13 @@ package org.apache.lucene.benchmark.jmh;
import java.io.IOException;
import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -24,13 +26,8 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-
import org.openjdk.jmh.annotations.*;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
@State(Scope.Benchmark)
@@ -109,25 +106,26 @@ public class AnytimeRankingBenchmark {
private static final class SingleTokenAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean done = false;
-
- @Override
- public boolean incrementToken() {
- if (done) return false;
- clearAttributes();
- termAttr.append("lucene"); // emit single fixed token
- done = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- done = false;
- }
- };
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean done = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (done) return false;
+ clearAttributes();
+ termAttr.append("lucene"); // emit single fixed token
+ done = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ done = false;
+ }
+ };
return new TokenStreamComponents(tokenizer);
}
@@ -136,4 +134,4 @@ public class AnytimeRankingBenchmark {
return reader;
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java
index 0e0f413d710..519b7399d22 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java
@@ -5,7 +5,6 @@ import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -26,7 +25,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-
import org.openjdk.jmh.annotations.*;
@BenchmarkMode(Mode.Throughput)
@@ -46,7 +44,7 @@ public class AnytimeSLARelevanceBenchmark {
@Param({"1000", "10000"})
private int docCount;
- @Param({"1", "5", "10"})
+ @Param({"1", "5", "10"})
private int slaScoreBudget;
@Setup
@@ -114,31 +112,30 @@ public class AnytimeSLARelevanceBenchmark {
directory.close();
}
- /**
- * Analyzer that emits a single fixed token to eliminate dependency on analyzer modules.
- */
+ /** Analyzer that emits a single fixed token to eliminate dependency on analyzer modules. */
private static final class SingleTokenAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) return false;
- clearAttributes();
- termAttr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
return new TokenStreamComponents(tokenizer);
}
@@ -147,4 +144,4 @@ public class AnytimeSLARelevanceBenchmark {
return reader;
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
index a4f8340344a..4da55ce9026 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
@@ -23,7 +23,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -37,7 +36,6 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@@ -49,7 +47,6 @@ import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
import org.openjdk.jmh.annotations.Warmup;
@BenchmarkMode(Mode.AverageTime)
@@ -108,40 +105,42 @@ public class BinningIndexingLatencyBenchmark {
} finally {
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
- .forEach(path -> {
- try {
- Files.deleteIfExists(path);
- } catch (IOException ignored) {
- // best-effort cleanup
- }
- });
+ .forEach(
+ path -> {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException ignored) {
+ // best-effort cleanup
+ }
+ });
}
}
private static final class SingleTokenAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new Tokenizer() {
- private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) {
- return false;
- }
- clearAttributes();
- attr.append("lucene");
- emitted = true;
- return true;
- }
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ attr.append("lucene");
+ emitted = true;
+ return true;
+ }
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
return new TokenStreamComponents(tokenizer);
}
@@ -150,4 +149,4 @@ public class BinningIndexingLatencyBenchmark {
return reader;
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
index 616d3dde98a..3fe148e84e7 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
@@ -22,7 +22,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -36,7 +35,6 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
-
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
@@ -101,43 +99,43 @@ public class IndexingLatencyBenchmark {
} finally {
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
- .forEach(path -> {
- try {
- Files.deleteIfExists(path);
- } catch (IOException ignored) {
- // best-effort cleanup
- }
- });
+ .forEach(
+ path -> {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException ignored) {
+ // best-effort cleanup
+ }
+ });
}
}
- /**
- * Analyzer that emits a single token to avoid dependency on analyzer modules.
- */
+ /** Analyzer that emits a single token to avoid dependency on analyzer modules. */
private static final class SingleTokenAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new Tokenizer() {
- private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
- @Override
- public boolean incrementToken() {
- if (emitted) {
- return false;
- }
- clearAttributes();
- attr.append("lucene");
- emitted = true;
- return true;
- }
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ attr.append("lucene");
+ emitted = true;
+ return true;
+ }
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
return new TokenStreamComponents(tokenizer);
}
@@ -146,4 +144,4 @@ public class IndexingLatencyBenchmark {
return reader;
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocBinner.java b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocBinner.java
index cbdfac8e199..ac889998d8c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocBinner.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocBinner.java
@@ -18,9 +18,7 @@ package org.apache.lucene.codecs;
import java.io.IOException;
-/**
- * Approximate binning for documents using locality-sensitive hashing over sparse edge graph.
- */
+/** Approximate binning for documents using locality-sensitive hashing over sparse edge graph. */
public final class ApproximateDocBinner {
private ApproximateDocBinner() {}
@@ -28,8 +26,8 @@ public final class ApproximateDocBinner {
/**
* Assign documents to bins approximately based on edge distribution.
*
- * @param graph sparse edge graph
- * @param maxDoc number of documents
+ * @param graph sparse edge graph
+ * @param maxDoc number of documents
* @param numBins number of bins (must be power of 2)
* @return bin assignments for each document
*/
@@ -60,4 +58,4 @@ public final class ApproximateDocBinner {
h *= 0x5bd1e995;
return h;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
index b4d721de8b4..9a6cb45de47 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
@@ -88,7 +88,9 @@ public final class ApproximateDocGraphBuilder {
}
int added = 0;
- for (int docB = candidates.nextSetBit(0); docB >= 0 && added < maxEdgesPerDoc; docB = candidates.nextSetBit(docB + 1)) {
+ for (int docB = candidates.nextSetBit(0);
+ docB >= 0 && added < maxEdgesPerDoc;
+ docB = candidates.nextSetBit(docB + 1)) {
if (docA == docB) {
continue;
}
@@ -115,4 +117,4 @@ public final class ApproximateDocGraphBuilder {
return graph;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java b/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
index cac85cf938e..8c4233233b3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
@@ -2,16 +2,12 @@ package org.apache.lucene.codecs;
import java.util.Arrays;
-/**
- * Computes bin quality metrics like balance, skewness, and Gini index.
- */
+/** Computes bin quality metrics like balance, skewness, and Gini index. */
public final class BinQualityMetrics {
private BinQualityMetrics() {}
- /**
- * Computes standard deviation of document counts across bins.
- */
+ /** Computes standard deviation of document counts across bins. */
public static double computeStdDev(int[] docCounts) {
double mean = Arrays.stream(docCounts).average().orElse(0.0);
double variance = 0.0;
@@ -22,9 +18,7 @@ public final class BinQualityMetrics {
return Math.sqrt(variance / docCounts.length);
}
- /**
- * Computes the Gini index to measure bin inequality.
- */
+ /** Computes the Gini index to measure bin inequality. */
public static double computeGiniIndex(int[] docCounts) {
int n = docCounts.length;
if (n == 0) return 0.0;
@@ -38,4 +32,4 @@ public final class BinQualityMetrics {
if (total == 0.0) return 0.0;
return (2.0 * cumulative) / (n * total) - (n + 1.0) / n;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
index 9d54a9693fe..a8bc9206c7c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
@@ -4,14 +4,13 @@ import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
-/**
- * Non-recursive document binner using iterative graph bisection.
- */
+/** Non-recursive document binner using iterative graph bisection. */
public final class DocBinningGraphBuilder {
private DocBinningGraphBuilder() {}
- public static int[] computeBins(SparseEdgeGraph graph, int maxDoc, int numBins) throws IOException {
+ public static int[] computeBins(SparseEdgeGraph graph, int maxDoc, int numBins)
+ throws IOException {
if (maxDoc <= 0 || Integer.bitCount(numBins) != 1) {
throw new IllegalArgumentException("maxDoc must be > 0 and numBins must be a power of 2");
}
@@ -99,4 +98,4 @@ public final class DocBinningGraphBuilder {
this.binCount = binCount;
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
index f3254c856b0..aea98964069 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
@@ -22,7 +22,6 @@ import java.util.HashMap;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.Set;
-
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Terms;
@@ -30,9 +29,7 @@ import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.BytesRef;
-/**
- * Builds a sparse document similarity graph from postings lists.
- */
+/** Builds a sparse document similarity graph from postings lists. */
public final class DocGraphBuilder {
/** Default maximum outgoing edges per document */
@@ -55,9 +52,7 @@ public final class DocGraphBuilder {
this.maxEdgesPerDoc = maxEdgesPerDoc;
}
- /**
- * Constructs the sparse similarity graph using term overlap.
- */
+ /** Constructs the sparse similarity graph using term overlap. */
public SparseEdgeGraph build(LeafReader reader) throws IOException {
final int maxDoc = reader.maxDoc();
final SparseEdgeGraph graph = new InMemorySparseEdgeGraph();
@@ -163,9 +158,7 @@ public final class DocGraphBuilder {
}
}
- /**
- * Lightweight term frequency vector with string keys and int values.
- */
+ /** Lightweight term frequency vector with string keys and int values. */
private static final class IntVector {
private final Map<String, Integer> tf = new HashMap<>();
@@ -189,4 +182,4 @@ public final class DocGraphBuilder {
return tf.values();
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
index 3751d3d6354..9f9f627e4f0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
@@ -17,8 +17,6 @@
package org.apache.lucene.codecs.lucene101;
import java.io.IOException;
-import java.util.Map;
-
import org.apache.lucene.codecs.ApproximateDocBinner;
import org.apache.lucene.codecs.ApproximateDocGraphBuilder;
import org.apache.lucene.codecs.BinMapWriter;
@@ -40,10 +38,8 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.ImpactsEnum;
-import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.TermState;
@@ -338,314 +334,307 @@ import org.apache.lucene.util.packed.PackedInts;
*/
public final class Lucene101PostingsFormat extends PostingsFormat {
- /**
- * Filename extension for some small metadata about how postings are encoded.
- */
- public static final String META_EXTENSION = "psm";
-
- /**
- * Filename extension for document number, frequencies, and skip data. See chapter: <a
- * href="#Frequencies">Frequencies and Skip Data</a>
- */
- public static final String DOC_EXTENSION = "doc";
-
- /**
- * Filename extension for positions. See chapter: <a href="#Positions">Positions</a>
- */
- public static final String POS_EXTENSION = "pos";
-
- /**
- * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
- * Offsets</a>
- */
- public static final String PAY_EXTENSION = "pay";
-
- /**
- * Size of blocks.
- */
- public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
-
- public static final int BLOCK_MASK = BLOCK_SIZE - 1;
-
- /**
- * We insert skip data on every block and every SKIP_FACTOR=32 blocks.
- */
- public static final int LEVEL1_FACTOR = 32;
-
- /**
- * Total number of docs covered by level 1 skip data: 32 * 128 = 4,096
- */
- public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
-
- public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
- static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
- static final String META_CODEC = "Lucene101PostingsWriterMeta";
- static final String DOC_CODEC = "Lucene101PostingsWriterDoc";
- static final String POS_CODEC = "Lucene101PostingsWriterPos";
- static final String PAY_CODEC = "Lucene101PostingsWriterPay";
- static final int VERSION_START = 0;
- /**
- * Version that started encoding dense blocks as bit sets. Note: the old format is a subset of the
- * new format, so Lucene101PostingsReader is able to read the old format without checking the
- * version.
- */
- static final int VERSION_DENSE_BLOCKS_AS_BITSETS = 1;
- static final int VERSION_CURRENT = VERSION_DENSE_BLOCKS_AS_BITSETS;
- private static final int MAX_DOC_FOR_EXACT_BINNING = 10;
- private final int version;
- private final int minTermBlockSize;
- private final int maxTermBlockSize;
+ /** Filename extension for some small metadata about how postings are encoded. */
+ public static final String META_EXTENSION = "psm";
+
+ /**
+ * Filename extension for document number, frequencies, and skip data. See chapter: <a
+ * href="#Frequencies">Frequencies and Skip Data</a>
+ */
+ public static final String DOC_EXTENSION = "doc";
+
+ /** Filename extension for positions. See chapter: <a href="#Positions">Positions</a> */
+ public static final String POS_EXTENSION = "pos";
+
+ /**
+ * Filename extension for payloads and offsets. See chapter: <a href="#Payloads">Payloads and
+ * Offsets</a>
+ */
+ public static final String PAY_EXTENSION = "pay";
+
+ /** Size of blocks. */
+ public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
+
+ public static final int BLOCK_MASK = BLOCK_SIZE - 1;
+
+ /** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */
+ public static final int LEVEL1_FACTOR = 32;
+
+ /** Total number of docs covered by level 1 skip data: 32 * 128 = 4,096 */
+ public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
+
+ public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
+ static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
+ static final String META_CODEC = "Lucene101PostingsWriterMeta";
+ static final String DOC_CODEC = "Lucene101PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene101PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene101PostingsWriterPay";
+ static final int VERSION_START = 0;
+
+ /**
+ * Version that started encoding dense blocks as bit sets. Note: the old format is a subset of the
+ * new format, so Lucene101PostingsReader is able to read the old format without checking the
+ * version.
+ */
+ static final int VERSION_DENSE_BLOCKS_AS_BITSETS = 1;
+
+ static final int VERSION_CURRENT = VERSION_DENSE_BLOCKS_AS_BITSETS;
+ private static final int MAX_DOC_FOR_EXACT_BINNING = 10;
+ private final int version;
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene101PostingsFormat} with default settings. */
+ public Lucene101PostingsFormat() {
+ this(
+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ }
+
+ /**
+ * Creates {@code Lucene101PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * maxBlockSize} passed to block terms dictionary.
+ *
+ * @see Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,
+ * PostingsWriterBase, int, int)
+ */
+ public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows setting the version. */
+ public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
+ super("Lucene101");
+ if (version < VERSION_START || version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version out of range: " + version);
+ }
+ this.version = version;
+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
+ }
+
+ /**
+ * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is
+ * internally used to help the JVM make good inlining decisions.
+ *
+ * @lucene.internal
+ */
+ public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {
+ return Lucene101PostingsReader.BlockPostingsEnum.class;
+ }
+
+ static void maybeWriteDocBinning(SegmentWriteState state) throws IOException {
+ final int maxDoc = state.segmentInfo.maxDoc();
+ if (maxDoc == 0) {
+ return;
+ }
- /**
- * Creates {@code Lucene101PostingsFormat} with default settings.
- */
- public Lucene101PostingsFormat() {
- this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ final FieldInfos fieldInfos = state.fieldInfos;
+ String binningField = null;
+ String graphBuilderType = null;
+ int binCount = -1;
+
+ for (FieldInfo fi : fieldInfos) {
+ if ("true".equalsIgnoreCase(fi.getAttribute("doBinning"))) {
+ binningField = fi.name;
+ String binCountAttr = fi.getAttribute("bin.count");
+ binCount =
+ (binCountAttr != null)
+ ? Integer.parseInt(binCountAttr)
+ : Math.max(1, Integer.highestOneBit(maxDoc >>> 4));
+ String graphBuilderAttr = fi.getAttribute("bin.builder");
+ graphBuilderType = (graphBuilderAttr != null) ? graphBuilderAttr : "auto";
+ break;
+ }
}
- /**
- * Creates {@code Lucene101PostingsFormat} with custom values for {@code minBlockSize} and {@code
- * maxBlockSize} passed to block terms dictionary.
- *
- * @see Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState, PostingsWriterBase, int, int)
- */
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
- this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
+ if (binningField == null || binCount <= 0) {
+ return;
}
- /**
- * Expert constructor that allows setting the version.
- */
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
- super("Lucene101");
- if (version < VERSION_START || version > VERSION_CURRENT) {
- throw new IllegalArgumentException("Version out of range: " + version);
+ final PostingsFormat format = new Lucene101PostingsFormat();
+ try (FieldsProducer fields =
+ format.fieldsProducer(
+ new SegmentReadState(
+ state.directory,
+ state.segmentInfo,
+ state.fieldInfos,
+ state.context,
+ state.segmentSuffix))) {
+
+ Terms terms = fields.terms(binningField);
+ if (terms == null) {
+ return;
+ }
+
+ try (LeafReader reader =
+ new FieldsLeafReader(java.util.Collections.singletonMap(binningField, terms), maxDoc)) {
+
+ final int[] docToBin;
+
+ final SparseEdgeGraph graph;
+
+ if ("approx".equalsIgnoreCase(graphBuilderType)) {
+ ApproximateDocGraphBuilder builder =
+ new ApproximateDocGraphBuilder(
+ binningField, ApproximateDocGraphBuilder.DEFAULT_MAX_EDGES);
+ graph = builder.build(reader);
+ docToBin = ApproximateDocBinner.assign(graph, maxDoc, binCount);
+ } else if ("exact".equalsIgnoreCase(graphBuilderType)) {
+ DocGraphBuilder builder =
+ new DocGraphBuilder(binningField, DocGraphBuilder.DEFAULT_MAX_EDGES);
+ graph = builder.build(reader);
+ docToBin = DocBinningGraphBuilder.computeBins(graph, maxDoc, binCount);
+ } else {
+ // default: auto fallback based on doc count
+ if (maxDoc > MAX_DOC_FOR_EXACT_BINNING) {
+ ApproximateDocGraphBuilder builder =
+ new ApproximateDocGraphBuilder(
+ binningField, ApproximateDocGraphBuilder.DEFAULT_MAX_EDGES);
+ graph = builder.build(reader);
+ docToBin = ApproximateDocBinner.assign(graph, maxDoc, binCount);
+ } else {
+ DocGraphBuilder builder =
+ new DocGraphBuilder(binningField, DocGraphBuilder.DEFAULT_MAX_EDGES);
+ graph = builder.build(reader);
+ docToBin = DocBinningGraphBuilder.computeBins(graph, maxDoc, binCount);
+ }
}
- this.version = version;
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
- this.minTermBlockSize = minTermBlockSize;
- this.maxTermBlockSize = maxTermBlockSize;
- }
- /**
- * Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is
- * internally used to help the JVM make good inlining decisions.
- *
- * @lucene.internal
- */
- public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {
- return Lucene101PostingsReader.BlockPostingsEnum.class;
+ BinMapWriter writer = new BinMapWriter(state.directory, state, docToBin, binCount);
+ writer.close();
+ }
}
+ }
- static void maybeWriteDocBinning(SegmentWriteState state) throws IOException {
- final int maxDoc = state.segmentInfo.maxDoc();
- if (maxDoc == 0) {
- return;
- }
+ @Override
+ public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
+ final Lucene101PostingsWriter postingsWriter = new Lucene101PostingsWriter(state, version);
+
+ final FieldsConsumer delegate =
+ new Lucene90BlockTreeTermsWriter(state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- final FieldInfos fieldInfos = state.fieldInfos;
- String binningField = null;
- String graphBuilderType = null;
- int binCount = -1;
-
- for (FieldInfo fi : fieldInfos) {
- if ("true".equalsIgnoreCase(fi.getAttribute("doBinning"))) {
- binningField = fi.name;
- String binCountAttr = fi.getAttribute("bin.count");
- binCount = (binCountAttr != null) ? Integer.parseInt(binCountAttr)
- : Math.max(1, Integer.highestOneBit(maxDoc >>> 4));
- String graphBuilderAttr = fi.getAttribute("bin.builder");
- graphBuilderType = (graphBuilderAttr != null) ? graphBuilderAttr : "auto";
- break;
- }
+ return new FieldsConsumer() {
+ @Override
+ public void write(Fields fields, NormsProducer normsProducer) throws IOException {
+ delegate.write(fields, normsProducer);
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException prior = null;
+ try {
+ delegate.close(); // flushes .psm/.doc/.pos/.pay
+ } catch (IOException e) {
+ prior = e;
}
- if (binningField == null || binCount <= 0) {
- return;
+ try {
+ // Now that all outputs are flushed, perform binning safely
+ maybeWriteDocBinning(state);
+ } catch (IOException e) {
+ if (prior != null) {
+ prior.addSuppressed(e);
+ throw prior;
+ } else {
+ throw e;
+ }
}
- final PostingsFormat format = new Lucene101PostingsFormat();
- try (FieldsProducer fields = format.fieldsProducer(
- new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, state.context, state.segmentSuffix))) {
-
- Terms terms = fields.terms(binningField);
- if (terms == null) {
- return;
- }
-
- try (LeafReader reader = new FieldsLeafReader(
- java.util.Collections.singletonMap(binningField, terms), maxDoc)) {
-
-
- final int[] docToBin;
-
- final SparseEdgeGraph graph;
-
- if ("approx".equalsIgnoreCase(graphBuilderType)) {
- ApproximateDocGraphBuilder builder = new ApproximateDocGraphBuilder(binningField, ApproximateDocGraphBuilder.DEFAULT_MAX_EDGES);
- graph = builder.build(reader);
- docToBin = ApproximateDocBinner.assign(graph, maxDoc, binCount);
- } else if ("exact".equalsIgnoreCase(graphBuilderType)) {
- DocGraphBuilder builder = new DocGraphBuilder(binningField, DocGraphBuilder.DEFAULT_MAX_EDGES);
- graph = builder.build(reader);
- docToBin = DocBinningGraphBuilder.computeBins(graph, maxDoc, binCount);
- } else {
- // default: auto fallback based on doc count
- if (maxDoc > MAX_DOC_FOR_EXACT_BINNING) {
- ApproximateDocGraphBuilder builder = new ApproximateDocGraphBuilder(binningField, ApproximateDocGraphBuilder.DEFAULT_MAX_EDGES);
- graph = builder.build(reader);
- docToBin = ApproximateDocBinner.assign(graph, maxDoc, binCount);
- } else {
- DocGraphBuilder builder = new DocGraphBuilder(binningField, DocGraphBuilder.DEFAULT_MAX_EDGES);
- graph = builder.build(reader);
- docToBin = DocBinningGraphBuilder.computeBins(graph, maxDoc, binCount);
- }
- }
-
- BinMapWriter writer = new BinMapWriter(state.directory, state, docToBin, binCount);
- writer.close();
- }
+ if (prior != null) {
+ throw prior;
}
+ }
+ };
+ }
+
+ @Override
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ boolean success = false;
+ try {
+ FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(postingsReader);
+ }
}
+ }
+ /**
+ * Holds all state required for {@link Lucene101PostingsReader} to produce a {@link
+ * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
+ *
+ * @lucene.internal
+ */
+ public static final class IntBlockTermState extends BlockTermState {
+ /** file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file */
+ public long docStartFP;
- @Override
- public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- final Lucene101PostingsWriter postingsWriter = new Lucene101PostingsWriter(state, version);
-
- final FieldsConsumer delegate =
- new Lucene90BlockTreeTermsWriter(state, postingsWriter, minTermBlockSize, maxTermBlockSize);
-
- return new FieldsConsumer() {
- @Override
- public void write(Fields fields, NormsProducer normsProducer) throws IOException {
- delegate.write(fields, normsProducer);
- }
-
- @Override
- public void close() throws IOException {
- IOException prior = null;
- try {
- delegate.close(); // flushes .psm/.doc/.pos/.pay
- } catch (IOException e) {
- prior = e;
- }
-
- try {
- // Now that all outputs are flushed, perform binning safely
- maybeWriteDocBinning(state);
- } catch (IOException e) {
- if (prior != null) {
- prior.addSuppressed(e);
- throw prior;
- } else {
- throw e;
- }
- }
-
- if (prior != null) {
- throw prior;
- }
- }
- };
- }
+ /** file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file */
+ public long posStartFP;
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
- boolean success = false;
- try {
- FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
+ /** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
+ public long payStartFP;
/**
- * Holds all state required for {@link Lucene101PostingsReader} to produce a {@link
- * org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
+ * file offset for the last position in the last block, if there are more than {@link
+ * ForUtil#BLOCK_SIZE} positions; otherwise -1
*
- * @lucene.internal
+ * <p>One might think to use total term frequency to track how many positions are left to read
+ * as we decode the blocks, and decode the last block differently when num_left_positions <
+ * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip
+ * blocks as the skipper will only tell us new position offset (start of block) and number of
+ * positions to skip for that block, without telling us how many positions it has skipped.
*/
- public static final class IntBlockTermState extends BlockTermState {
- /**
- * file pointer to the start of the doc ids enumeration, in {@link #DOC_EXTENSION} file
- */
- public long docStartFP;
-
- /**
- * file pointer to the start of the positions enumeration, in {@link #POS_EXTENSION} file
- */
- public long posStartFP;
-
- /**
- * file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file
- */
- public long payStartFP;
-
- /**
- * file offset for the last position in the last block, if there are more than {@link
- * ForUtil#BLOCK_SIZE} positions; otherwise -1
- *
- * <p>One might think to use total term frequency to track how many positions are left to read
- * as we decode the blocks, and decode the last block differently when num_left_positions <
- * BLOCK_SIZE. Unfortunately this won't work since the tracking will be messed up when we skip
- * blocks as the skipper will only tell us new position offset (start of block) and number of
- * positions to skip for that block, without telling us how many positions it has skipped.
- */
- public long lastPosBlockOffset;
-
- /**
- * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
- * totalTermFreq in this case.
- */
- public int singletonDocID;
-
- /**
- * Sole constructor.
- */
- public IntBlockTermState() {
- lastPosBlockOffset = -1;
- singletonDocID = -1;
- }
+ public long lastPosBlockOffset;
- @Override
- public IntBlockTermState clone() {
- IntBlockTermState other = new IntBlockTermState();
- other.copyFrom(this);
- return other;
- }
+ /**
+ * docid when there is a single pulsed posting, otherwise -1. freq is always implicitly
+ * totalTermFreq in this case.
+ */
+ public int singletonDocID;
- @Override
- public void copyFrom(TermState _other) {
- super.copyFrom(_other);
- IntBlockTermState other = (IntBlockTermState) _other;
- docStartFP = other.docStartFP;
- posStartFP = other.posStartFP;
- payStartFP = other.payStartFP;
- lastPosBlockOffset = other.lastPosBlockOffset;
- singletonDocID = other.singletonDocID;
- }
+ /** Sole constructor. */
+ public IntBlockTermState() {
+ lastPosBlockOffset = -1;
+ singletonDocID = -1;
+ }
- @Override
- public String toString() {
- return super.toString()
- + " docStartFP="
- + docStartFP
- + " posStartFP="
- + posStartFP
- + " payStartFP="
- + payStartFP
- + " lastPosBlockOffset="
- + lastPosBlockOffset
- + " singletonDocID="
- + singletonDocID;
- }
+ @Override
+ public IntBlockTermState clone() {
+ IntBlockTermState other = new IntBlockTermState();
+ other.copyFrom(this);
+ return other;
+ }
+
+ @Override
+ public void copyFrom(TermState _other) {
+ super.copyFrom(_other);
+ IntBlockTermState other = (IntBlockTermState) _other;
+ docStartFP = other.docStartFP;
+ posStartFP = other.posStartFP;
+ payStartFP = other.payStartFP;
+ lastPosBlockOffset = other.lastPosBlockOffset;
+ singletonDocID = other.singletonDocID;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString()
+ + " docStartFP="
+ + docStartFP
+ + " posStartFP="
+ + posStartFP
+ + " payStartFP="
+ + payStartFP
+ + " lastPosBlockOffset="
+ + lastPosBlockOffset
+ + " singletonDocID="
+ + singletonDocID;
}
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
index 45b9f1c7dec..ab8f0c3af5c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
@@ -4,7 +4,6 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.lucene.codecs.BinMapReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -31,7 +30,8 @@ public final class BinScoreUtil {
for (LeafReaderContext ctx : reader.leaves()) {
wrapped.add(wrap(ctx.reader()));
}
- // This MultiReader just wraps the readers — responsibility for cleanup is delegated to individual leaves.
+ // This MultiReader just wraps the readers — responsibility for cleanup is delegated to
+ // individual leaves.
return new MultiReader(wrapped.toArray(new LeafReader[0]), true);
}
@@ -62,8 +62,10 @@ public final class BinScoreUtil {
return reader;
}
- String suffix = binmapFile.substring(name.length() + 1, binmapFile.length() - ".binmap".length());
- SegmentReadState state = new SegmentReadState(dir, info, sr.getFieldInfos(), IOContext.READONCE, suffix);
+ String suffix =
+ binmapFile.substring(name.length() + 1, binmapFile.length() - ".binmap".length());
+ SegmentReadState state =
+ new SegmentReadState(dir, info, sr.getFieldInfos(), IOContext.READONCE, suffix);
BinMapReader binMap = null;
try {
@@ -75,7 +77,7 @@ public final class BinScoreUtil {
@Override
protected void doClose() throws IOException {
IOUtils.close(closeOnClose); // closes compoundReader if present
- super.doClose(); // closes binMap, etc.
+ super.doClose(); // closes binMap, etc.
}
};
} catch (Throwable t) {
@@ -112,4 +114,4 @@ public final class BinScoreUtil {
}
return null;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java b/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
index 99e24f24b8b..df8e4338e51 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
@@ -3,9 +3,7 @@ package org.apache.lucene.search;
import java.util.Collections;
import java.util.Set;
-/**
- * Represents a group of near-duplicate documents.
- */
+/** Represents a group of near-duplicate documents. */
public final class DuplicateCluster {
private final Set<Integer> docIDs;
@@ -14,9 +12,7 @@ public final class DuplicateCluster {
this.docIDs = Collections.unmodifiableSet(docIDs);
}
- /**
- * Returns the set of global document IDs in this cluster.
- */
+ /** Returns the set of global document IDs in this cluster. */
public Set<Integer> docIds() {
return docIDs;
}
@@ -25,4 +21,4 @@ public final class DuplicateCluster {
public String toString() {
return "DuplicateCluster" + docIDs;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java b/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
index d3baa521f38..179b6e8a9be 100644
--- a/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
+++ b/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
@@ -4,16 +4,15 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.HashSet;
-
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.util.BytesRef;
/**
@@ -27,7 +26,7 @@ public final class NearDuplicateFinder {
/**
* Create a new finder.
*
- * @param field the field to analyze
+ * @param field the field to analyze
* @param threshold similarity threshold between 0 and 1
*/
public NearDuplicateFinder(String field, float threshold) {
@@ -127,4 +126,4 @@ public final class NearDuplicateFinder {
int union = a.size() + b.size() - intersection;
return union == 0 ? 0.0f : (float) intersection / union;
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java b/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
index 6f84d5468eb..f44898ee7ea 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
@@ -3,9 +3,7 @@ package org.apache.lucene.codecs;
import java.util.Random;
import org.apache.lucene.tests.util.LuceneTestCase;
-/**
- * Unit tests for verifying bin balance and quality.
- */
+/** Unit tests for verifying bin balance and quality. */
public class TestBinQualityMetrics extends LuceneTestCase {
public void testStandardDeviationAndGini() {
@@ -33,4 +31,4 @@ public class TestBinQualityMetrics extends LuceneTestCase {
double gini = BinQualityMetrics.computeGiniIndex(counts);
assertTrue("Gini index must be in range [0,1]", gini >= 0.0 && gini <= 1.0);
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
index e33ff39d9d0..9953238e541 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
@@ -20,9 +20,6 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.codecs.BinMapReader;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
@@ -33,18 +30,12 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.BinScoreReader;
-import org.apache.lucene.index.BinScoreUtil;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.Impact;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.NoMergePolicy;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentInfos;
@@ -56,7 +47,6 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
@@ -65,377 +55,372 @@ import org.apache.lucene.util.Version;
public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
- public static SegmentReadState fakeSegmentReadState(Directory dir, String segmentName, int maxDoc)
- throws IOException {
- SegmentInfo info =
- new SegmentInfo(
- dir,
- Version.LATEST,
- Version.LATEST,
- segmentName,
- maxDoc,
- false,
- false,
- Codec.getDefault(),
- Collections.emptyMap(),
- StringHelper.randomId(),
- Collections.emptyMap(),
- null);
-
- return new SegmentReadState(dir, info, null, IOContext.READONCE);
+ public static SegmentReadState fakeSegmentReadState(Directory dir, String segmentName, int maxDoc)
+ throws IOException {
+ SegmentInfo info =
+ new SegmentInfo(
+ dir,
+ Version.LATEST,
+ Version.LATEST,
+ segmentName,
+ maxDoc,
+ false,
+ false,
+ Codec.getDefault(),
+ Collections.emptyMap(),
+ StringHelper.randomId(),
+ Collections.emptyMap(),
+ null);
+
+ return new SegmentReadState(dir, info, null, IOContext.READONCE);
+ }
+
+ @Override
+ protected Codec getCodec() {
+ return TestUtil.alwaysPostingsFormat(new Lucene101PostingsFormat());
+ }
+
+ public void testVInt15() throws IOException {
+ byte[] bytes = new byte[5];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (int i : new int[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene101PostingsWriter.writeVInt15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene101PostingsReader.readVInt15(in));
+ assertEquals(out.getPosition(), in.getPosition());
}
-
- @Override
- protected Codec getCodec() {
- return TestUtil.alwaysPostingsFormat(new Lucene101PostingsFormat());
+ }
+
+ public void testVLong15() throws IOException {
+ byte[] bytes = new byte[9];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (long i : new long[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene101PostingsWriter.writeVLong15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene101PostingsReader.readVLong15(in));
+ assertEquals(out.getPosition(), in.getPosition());
}
-
- public void testVInt15() throws IOException {
- byte[] bytes = new byte[5];
- ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
- ByteArrayDataInput in = new ByteArrayDataInput();
- for (int i : new int[]{0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {
- out.reset(bytes);
- Lucene101PostingsWriter.writeVInt15(out, i);
- in.reset(bytes, 0, out.getPosition());
- assertEquals(i, Lucene101PostingsReader.readVInt15(in));
- assertEquals(out.getPosition(), in.getPosition());
- }
+ }
+
+ /** Make sure the final sub-block(s) are not skipped. */
+ public void testFinalBlock() throws Exception {
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
+ for (int i = 0; i < 25; i++) {
+ Document doc = new Document();
+ doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));
+ doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));
+ w.addDocument(doc);
}
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ assertEquals(1, r.leaves().size());
+ FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");
+ // We should see exactly two blocks: one root block (prefix empty string) and one block for z*
+ // terms (prefix z):
+ Stats stats = field.getStats();
+ assertEquals(0, stats.floorBlockCount);
+ assertEquals(2, stats.nonFloorBlockCount);
+ r.close();
+ w.close();
+ d.close();
+ }
+
+ public void testImpactSerialization() throws IOException {
+ // omit norms and omit freqs
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
+
+ // omit freqs
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
+ // omit freqs with very large norms
+ doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
+
+ // omit norms
+ doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
+ // omit norms with large freq
+ doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
+
+ // freqs and norms, basic
+ doTestImpactSerialization(
+ Arrays.asList(
+ new Impact(1, 7L),
+ new Impact(3, 9L),
+ new Impact(7, 10L),
+ new Impact(15, 11L),
+ new Impact(20, 13L),
+ new Impact(28, 14L)));
+
+ // freqs and norms, high values
+ doTestImpactSerialization(
+ Arrays.asList(
+ new Impact(2, 2L),
+ new Impact(10, 10L),
+ new Impact(12, 50L),
+ new Impact(50, -100L),
+ new Impact(1000, -80L),
+ new Impact(1005, -3L)));
+ }
+
+ private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
+ CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
+ for (Impact impact : impacts) {
+ acc.add(impact.freq, impact.norm);
+ }
+ try (Directory dir = newDirectory()) {
+ try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
+ Lucene101PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
+ }
+ try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
+ byte[] b = new byte[Math.toIntExact(in.length())];
+ in.readBytes(b, 0, b.length);
+ List<Impact> impacts2 =
+ Lucene101PostingsReader.readImpacts(
+ new ByteArrayDataInput(b),
+ new MutableImpactList(impacts.size() + random().nextInt(3)));
+ assertEquals(impacts, impacts2);
+ }
+ }
+ }
+
+ public void testBinMapIsWrittenOnFlush() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(getCodec());
+ iwc.setUseCompoundFile(false);
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < 3; i++) {
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.putAttribute("doBinning", "true");
+ Field field = new Field("field", "value" + i, ft);
+ doc.add(field);
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+ SegmentCommitInfo info = infos.info(0);
+ String segmentName = info.info.name;
+ String segmentSuffix = null;
+ String binmapFile = null;
+
+ for (String file : dir.listAll()) {
+ if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
+ binmapFile = file;
+ segmentSuffix =
+ file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
+ break;
+ }
+ }
+
+ assertNotNull("binmap file should exist", binmapFile);
- public void testVLong15() throws IOException {
- byte[] bytes = new byte[9];
- ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
- ByteArrayDataInput in = new ByteArrayDataInput();
- for (long i : new long[]{0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {
- out.reset(bytes);
- Lucene101PostingsWriter.writeVLong15(out, i);
- in.reset(bytes, 0, out.getPosition());
- assertEquals(i, Lucene101PostingsReader.readVLong15(in));
- assertEquals(out.getPosition(), in.getPosition());
+ SegmentReadState readState =
+ new SegmentReadState(dir, info.info, null, newIOContext(random()), segmentSuffix);
+
+ try (BinMapReader binMap = new BinMapReader(dir, readState)) {
+ assertEquals(3, binMap.getBinArrayCopy().length);
+ for (int i = 0; i < 3; i++) {
+ int bin = binMap.getBin(i);
+ assertTrue("Bin should be non-negative", bin >= 0);
}
+ }
}
+ }
+
+ public void testBinMapWithCompoundFileFormat() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(getCodec());
+ iwc.setUseCompoundFile(true);
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < 4; i++) {
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.putAttribute("doBinning", "true");
+ Field field = new Field("field", "term" + i, ft);
+ doc.add(field);
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+ SegmentCommitInfo info = infos.info(0);
+ String segmentName = info.info.name;
+
+ try (Directory cfsDir =
+ info.info.getCodec().compoundFormat().getCompoundReader(dir, info.info)) {
+ String binmapFile = null;
+ String segmentSuffix = null;
+ for (String file : cfsDir.listAll()) {
+ if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
+ binmapFile = file;
+ segmentSuffix =
+ file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
+ break;
+ }
+ }
+
+ assertNotNull("compound file should contain binmap", binmapFile);
+
+ SegmentReadState readState =
+ new SegmentReadState(cfsDir, info.info, null, newIOContext(random()), segmentSuffix);
- /**
- * Make sure the final sub-block(s) are not skipped.
- */
- public void testFinalBlock() throws Exception {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
- for (int i = 0; i < 25; i++) {
- Document doc = new Document();
- doc.add(newStringField("field", Character.toString((char) (97 + i)), Field.Store.NO));
- doc.add(newStringField("field", "z" + Character.toString((char) (97 + i)), Field.Store.NO));
- w.addDocument(doc);
+ try (BinMapReader binMap = new BinMapReader(cfsDir, readState)) {
+ assertEquals(4, binMap.getBinArrayCopy().length);
+ for (int i = 0; i < 4; i++) {
+ int bin = binMap.getBin(i);
+ assertTrue("Bin should be non-negative", bin >= 0);
+ }
}
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- assertEquals(1, r.leaves().size());
- FieldReader field = (FieldReader) r.leaves().get(0).reader().terms("field");
- // We should see exactly two blocks: one root block (prefix empty string) and one block for z*
- // terms (prefix z):
- Stats stats = field.getStats();
- assertEquals(0, stats.floorBlockCount);
- assertEquals(2, stats.nonFloorBlockCount);
- r.close();
- w.close();
- d.close();
+ }
}
-
- public void testImpactSerialization() throws IOException {
- // omit norms and omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
-
- // omit freqs
- doTestImpactSerialization(Collections.singletonList(new Impact(1, 42L)));
- // omit freqs with very large norms
- doTestImpactSerialization(Collections.singletonList(new Impact(1, -100L)));
-
- // omit norms
- doTestImpactSerialization(Collections.singletonList(new Impact(30, 1L)));
- // omit norms with large freq
- doTestImpactSerialization(Collections.singletonList(new Impact(500, 1L)));
-
- // freqs and norms, basic
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(1, 7L),
- new Impact(3, 9L),
- new Impact(7, 10L),
- new Impact(15, 11L),
- new Impact(20, 13L),
- new Impact(28, 14L)));
-
- // freqs and norms, high values
- doTestImpactSerialization(
- Arrays.asList(
- new Impact(2, 2L),
- new Impact(10, 10L),
- new Impact(12, 50L),
- new Impact(50, -100L),
- new Impact(1000, -80L),
- new Impact(1005, -3L)));
+ }
+
+ public void testBinMapBinningDisabled() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(getCodec());
+ iwc.setUseCompoundFile(false); // standalone files
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ Document doc = new Document();
+ doc.add(newTextField("field", "term value relevance", Field.Store.NO)); // no binning
+ writer.addDocument(doc);
+ writer.commit();
+ }
+
+ for (String file : dir.listAll()) {
+ assertFalse(
+ "binmap should not exist when binning is not enabled", file.endsWith(".binmap"));
+ }
}
-
- private void doTestImpactSerialization(List<Impact> impacts) throws IOException {
- CompetitiveImpactAccumulator acc = new CompetitiveImpactAccumulator();
- for (Impact impact : impacts) {
- acc.add(impact.freq, impact.norm);
+ }
+
+ public void testExplicitBinCountIsHonored() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(getCodec());
+ iwc.setUseCompoundFile(false);
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < 12; i++) {
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(true);
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ Field field = new Field("field", "lucene term doc", ft);
+ doc.add(field);
+ writer.addDocument(doc);
}
- try (Directory dir = newDirectory()) {
- try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
- Lucene101PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
- }
- try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
- byte[] b = new byte[Math.toIntExact(in.length())];
- in.readBytes(b, 0, b.length);
- List<Impact> impacts2 =
- Lucene101PostingsReader.readImpacts(
- new ByteArrayDataInput(b),
- new MutableImpactList(impacts.size() + random().nextInt(3)));
- assertEquals(impacts, impacts2);
- }
+ writer.commit();
+ }
+
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
+ SegmentCommitInfo info = infos.info(0);
+
+ String segmentName = info.info.name;
+ String segmentSuffix = null;
+ String binmapFile = null;
+
+ for (String file : dir.listAll()) {
+ if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
+ binmapFile = file;
+ segmentSuffix =
+ file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
+ break;
}
- }
+ }
- public void testBinMapIsWrittenOnFlush() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(getCodec());
- iwc.setUseCompoundFile(false);
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 3; i++) {
- Document doc = new Document();
- FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.putAttribute("doBinning", "true");
- Field field = new Field("field", "value" + i, ft);
- doc.add(field);
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
- SegmentCommitInfo info = infos.info(0);
- String segmentName = info.info.name;
- String segmentSuffix = null;
- String binmapFile = null;
-
- for (String file : dir.listAll()) {
- if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
- binmapFile = file;
- segmentSuffix =
- file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
- break;
- }
- }
-
- assertNotNull("binmap file should exist", binmapFile);
-
- SegmentReadState readState =
- new SegmentReadState(dir, info.info, null, newIOContext(random()), segmentSuffix);
-
- try (BinMapReader binMap = new BinMapReader(dir, readState)) {
- assertEquals(3, binMap.getBinArrayCopy().length);
- for (int i = 0; i < 3; i++) {
- int bin = binMap.getBin(i);
- assertTrue("Bin should be non-negative", bin >= 0);
- }
- }
- }
- }
+ assertNotNull("binmap file should exist", binmapFile);
- public void testBinMapWithCompoundFileFormat() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(getCodec());
- iwc.setUseCompoundFile(true);
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 4; i++) {
- Document doc = new Document();
- FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.putAttribute("doBinning", "true");
- Field field = new Field("field", "term" + i, ft);
- doc.add(field);
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
- SegmentCommitInfo info = infos.info(0);
- String segmentName = info.info.name;
-
- try (Directory cfsDir =
- info.info.getCodec().compoundFormat().getCompoundReader(dir, info.info)) {
- String binmapFile = null;
- String segmentSuffix = null;
- for (String file : cfsDir.listAll()) {
- if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
- binmapFile = file;
- segmentSuffix =
- file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
- break;
- }
- }
-
- assertNotNull("compound file should contain binmap", binmapFile);
-
- SegmentReadState readState =
- new SegmentReadState(cfsDir, info.info, null, newIOContext(random()), segmentSuffix);
-
- try (BinMapReader binMap = new BinMapReader(cfsDir, readState)) {
- assertEquals(4, binMap.getBinArrayCopy().length);
- for (int i = 0; i < 4; i++) {
- int bin = binMap.getBin(i);
- assertTrue("Bin should be non-negative", bin >= 0);
- }
- }
- }
- }
- }
+ SegmentReadState readState =
+ new SegmentReadState(dir, info.info, null, newIOContext(random()), segmentSuffix);
- public void testBinMapBinningDisabled() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(getCodec());
- iwc.setUseCompoundFile(false); // standalone files
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- Document doc = new Document();
- doc.add(newTextField("field", "term value relevance", Field.Store.NO)); // no binning
- writer.addDocument(doc);
- writer.commit();
- }
-
- for (String file : dir.listAll()) {
- assertFalse(
- "binmap should not exist when binning is not enabled", file.endsWith(".binmap"));
- }
+ try (BinMapReader binMap = new BinMapReader(dir, readState)) {
+ assertEquals("Expected 4 bins", 4, binMap.getBinCount());
+ int[] bins = binMap.getBinArrayCopy();
+ assertEquals(12, bins.length);
+ for (int bin : bins) {
+ assertTrue("bin id should be in [0, 3]", bin >= 0 && bin < 4);
}
+ }
}
-
- public void testExplicitBinCountIsHonored() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(getCodec());
- iwc.setUseCompoundFile(false);
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 12; i++) {
- Document doc = new Document();
- FieldType ft = new FieldType(TextField.TYPE_STORED);
- ft.setStoreTermVectors(true);
- ft.putAttribute("doBinning", "true");
- ft.putAttribute("bin.count", "4");
- Field field = new Field("field", "lucene term doc", ft);
- doc.add(field);
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
- SegmentCommitInfo info = infos.info(0);
-
- String segmentName = info.info.name;
- String segmentSuffix = null;
- String binmapFile = null;
-
- for (String file : dir.listAll()) {
- if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
- binmapFile = file;
- segmentSuffix =
- file.substring(segmentName.length() + 1, file.length() - ".binmap".length());
- break;
- }
- }
-
- assertNotNull("binmap file should exist", binmapFile);
-
- SegmentReadState readState =
- new SegmentReadState(dir, info.info, null, newIOContext(random()), segmentSuffix);
-
- try (BinMapReader binMap = new BinMapReader(dir, readState)) {
- assertEquals("Expected 4 bins", 4, binMap.getBinCount());
- int[] bins = binMap.getBinArrayCopy();
- assertEquals(12, bins.length);
- for (int bin : bins) {
- assertTrue("bin id should be in [0, 3]", bin >= 0 && bin < 4);
- }
- }
+ }
+
+ public void testBinMapIsPreservedAndUsableAfterMerge() throws Exception {
+ try (Directory dir = newDirectory()) {
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(getCodec());
+ iwc.setUseCompoundFile(random().nextBoolean());
+ iwc.setMaxBufferedDocs(2);
+
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setStoreTermVectors(false);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < 24; i++) {
+ Document doc = new Document();
+ doc.add(new Field("field", "lucene merge binning test", ft));
+ writer.addDocument(doc);
}
- }
+ writer.commit();
+ writer.forceMerge(1);
+ }
+
+ // Confirm single merged segment exists
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals("Expected single merged segment", 1, reader.leaves().size());
+
+ LeafReader leaf = reader.leaves().get(0).reader();
+ SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(leaf);
+ SegmentCommitInfo info = segmentReader.getSegmentInfo();
+ String segmentName = info.info.name;
+
+ // Extract actual binmap file from Directory
+ String binmapFile = null;
+ for (String file : dir.listAll()) {
+ if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
+ binmapFile = file;
+ break;
+ }
+ }
+
+ assertNotNull("Expected .binmap file after merge", binmapFile);
+
+ String suffix =
+ binmapFile.substring(
+ segmentName.length() + 1, binmapFile.length() - ".binmap".length());
+
+ SegmentReadState readState =
+ new SegmentReadState(
+ dir, info.info, segmentReader.getFieldInfos(), newIOContext(random()), suffix);
- public void testBinMapIsPreservedAndUsableAfterMerge() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(getCodec());
- iwc.setUseCompoundFile(random().nextBoolean());
- iwc.setMaxBufferedDocs(2);
-
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- ft.setStoreTermVectors(false);
- ft.putAttribute("postingsFormat", "Lucene101");
- ft.putAttribute("doBinning", "true");
- ft.putAttribute("bin.count", "4");
- ft.freeze();
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 24; i++) {
- Document doc = new Document();
- doc.add(new Field("field", "lucene merge binning test", ft));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- }
-
- // Confirm single merged segment exists
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- assertEquals("Expected single merged segment", 1, reader.leaves().size());
-
- LeafReader leaf = reader.leaves().get(0).reader();
- SegmentReader segmentReader = (SegmentReader) FilterLeafReader.unwrap(leaf);
- SegmentCommitInfo info = segmentReader.getSegmentInfo();
- String segmentName = info.info.name;
-
- // Extract actual binmap file from Directory
- String binmapFile = null;
- for (String file : dir.listAll()) {
- if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
- binmapFile = file;
- break;
- }
- }
-
- assertNotNull("Expected .binmap file after merge", binmapFile);
-
- String suffix = binmapFile.substring(
- segmentName.length() + 1,
- binmapFile.length() - ".binmap".length());
-
- SegmentReadState readState = new SegmentReadState(
- dir,
- info.info,
- segmentReader.getFieldInfos(),
- newIOContext(random()),
- suffix);
-
- try (BinMapReader binMap = new BinMapReader(dir, readState)) {
- assertEquals("Expected 4 bins", 4, binMap.getBinCount());
- int[] bins = binMap.getBinArrayCopy();
- assertEquals("Unexpected number of docs", 24, bins.length);
- for (int bin : bins) {
- assertTrue("Invalid bin ID", bin >= 0 && bin < 4);
- }
- }
- }
+ try (BinMapReader binMap = new BinMapReader(dir, readState)) {
+ assertEquals("Expected 4 bins", 4, binMap.getBinCount());
+ int[] bins = binMap.getBinArrayCopy();
+ assertEquals("Unexpected number of docs", 24, bins.length);
+ for (int bin : bins) {
+ assertTrue("Invalid bin ID", bin >= 0 && bin < 4);
+ }
}
+ }
}
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
index 824d55e45a5..34318429dcb 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
@@ -17,10 +17,8 @@
package org.apache.lucene.search;
-import java.io.IOException;
import java.util.Arrays;
import java.util.Locale;
-
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -40,82 +38,82 @@ import org.apache.lucene.tests.util.TestUtil;
public class TestAnytimeRankingNDCG extends LuceneTestCase {
- public void testNDCGDoesNotDiverge() throws Exception {
- MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(new Lucene101Codec());
-
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- ft.setTokenized(true);
- ft.putAttribute("postingsFormat", "Lucene101");
- ft.putAttribute("doBinning", "true");
- ft.putAttribute("bin.count", "4");
- ft.freeze();
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 500; i++) {
- Document doc = new Document();
- StringBuilder content = new StringBuilder();
-
- if (i % 50 == 0) {
- content.append("lucene high quality scoring search");
- } else {
- content.append("random filler ").append(TestUtil.randomSimpleString(random()));
- }
-
- doc.add(new Field("field", content.toString(), ft));
- writer.addDocument(doc);
- }
- writer.commit();
+ public void testNDCGDoesNotDiverge() throws Exception {
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ iwc.setCodec(new Lucene101Codec());
+
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.setTokenized(true);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < 500; i++) {
+ Document doc = new Document();
+ StringBuilder content = new StringBuilder();
+
+ if (i % 50 == 0) {
+ content.append("lucene high quality scoring search");
+ } else {
+ content.append("random filler ").append(TestUtil.randomSimpleString(random()));
}
- DirectoryReader reader = DirectoryReader.open(dir);
- IndexSearcher baseline = new IndexSearcher(reader);
- baseline.setSimilarity(new BM25Similarity());
+ doc.add(new Field("field", content.toString(), ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
- AnytimeRankingSearcher anytime =
- new AnytimeRankingSearcher(baseline, 10, 5, "field");
+ DirectoryReader reader = DirectoryReader.open(dir);
+ IndexSearcher baseline = new IndexSearcher(reader);
+ baseline.setSimilarity(new BM25Similarity());
- TopDocs baselineDocs = baseline.search(new TermQuery(new Term("field", "lucene")), 10);
- TopDocs anytimeDocs = anytime.search(new TermQuery(new Term("field", "lucene")));
+ AnytimeRankingSearcher anytime = new AnytimeRankingSearcher(baseline, 10, 5, "field");
- double baselineNDCG = computeNDCG(baselineDocs);
- double anytimeNDCG = computeNDCG(anytimeDocs);
+ TopDocs baselineDocs = baseline.search(new TermQuery(new Term("field", "lucene")), 10);
+ TopDocs anytimeDocs = anytime.search(new TermQuery(new Term("field", "lucene")));
- System.out.printf(Locale.ROOT, "baselineNDCG=%.3f anytimeNDCG=%.3f%n", baselineNDCG, anytimeNDCG);
+ double baselineNDCG = computeNDCG(baselineDocs);
+ double anytimeNDCG = computeNDCG(anytimeDocs);
- // Ensure that NDCG does not diverge by more than 0.1
- assertEquals("NDCG values should be close", baselineNDCG, anytimeNDCG, 0.1);
+ System.out.printf(
+ Locale.ROOT, "baselineNDCG=%.3f anytimeNDCG=%.3f%n", baselineNDCG, anytimeNDCG);
- reader.close();
- dir.close();
- }
+ // Ensure that NDCG does not diverge by more than 0.1
+ assertEquals("NDCG values should be close", baselineNDCG, anytimeNDCG, 0.1);
- private double computeNDCG(TopDocs topDocs) {
- int[] gains = new int[topDocs.scoreDocs.length];
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- gains[i] = (topDocs.scoreDocs[i].doc % 50 == 0) ? 3 : 0;
- }
+ reader.close();
+ dir.close();
+ }
- double dcg = 0.0;
- for (int i = 0; i < gains.length; i++) {
- int rel = gains[i];
- if (rel > 0) {
- dcg += (Math.pow(2, rel) - 1) / (Math.log(i + 2) / Math.log(2));
- }
- }
+ private double computeNDCG(TopDocs topDocs) {
+ int[] gains = new int[topDocs.scoreDocs.length];
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ gains[i] = (topDocs.scoreDocs[i].doc % 50 == 0) ? 3 : 0;
+ }
- Arrays.sort(gains);
- double idcg = 0.0;
- for (int i = gains.length - 1; i >= 0; i--) {
- int rel = gains[i];
- if (rel > 0) {
- int rank = gains.length - i;
- idcg += (Math.pow(2, rel) - 1) / (Math.log(rank + 1) / Math.log(2));
- }
- }
+ double dcg = 0.0;
+ for (int i = 0; i < gains.length; i++) {
+ int rel = gains[i];
+ if (rel > 0) {
+ dcg += (Math.pow(2, rel) - 1) / (Math.log(i + 2) / Math.log(2));
+ }
+ }
- return idcg == 0.0 ? 0.0 : dcg / idcg;
+ Arrays.sort(gains);
+ double idcg = 0.0;
+ for (int i = gains.length - 1; i >= 0; i--) {
+ int rel = gains[i];
+ if (rel > 0) {
+ int rank = gains.length - i;
+ idcg += (Math.pow(2, rel) - 1) / (Math.log(rank + 1) / Math.log(2));
+ }
}
-}
\ No newline at end of file
+
+ return idcg == 0.0 ? 0.0 : dcg / idcg;
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java b/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
index 1a69bad9c39..9cf41640632 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
@@ -9,7 +9,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.List;
import java.util.Set;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -36,7 +35,9 @@ public class TestNearDuplicateFinder extends LuceneTestCase {
}
private Directory buildIndex(String... docs) throws IOException {
- Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("TestNearDuplicateFinder")));
+ Directory dir =
+ new MockDirectoryWrapper(
+ random(), new MMapDirectory(createTempDir("TestNearDuplicateFinder")));
IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
try (IndexWriter writer = new IndexWriter(dir, config)) {
FieldType fieldType = buildTextType();
@@ -108,4 +109,4 @@ public class TestNearDuplicateFinder extends LuceneTestCase {
assertEquals(0, clusters.size());
}
}
-}
\ No newline at end of file
+}
(lucene) 16/18: Do not close twice
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 5ba124b661cd19c69b12331ff96c96618ddf25f0
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 19:04:01 2025 +0530
Do not close twice
---
.../org/apache/lucene/codecs/BinMapWriter.java | 138 +++++++++++----------
1 file changed, 75 insertions(+), 63 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
index fea1907feac..35ba8690380 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
@@ -17,6 +17,7 @@
package org.apache.lucene.codecs;
import java.io.IOException;
+
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.Directory;
@@ -31,73 +32,84 @@ import org.apache.lucene.util.IOUtils;
*/
public final class BinMapWriter implements AutoCloseable {
- /** File extension for binmap files. */
- public static final String EXTENSION = "binmap";
+ /**
+ * File extension for binmap files.
+ */
+ public static final String EXTENSION = "binmap";
+
+ /**
+ * Initial file format version.
+ */
+ public static final int VERSION_START = 0;
- /** Initial file format version. */
- public static final int VERSION_START = 0;
+ private static final String CODEC_NAME = "BinMap";
- private static final String CODEC_NAME = "BinMap";
+ private final Directory directory;
+ private final SegmentWriteState state;
+ private final int maxDoc;
+ private final int binCount;
+ private final int[] bins;
- private final Directory directory;
- private final SegmentWriteState state;
- private final int maxDoc;
- private final int binCount;
- private final int[] bins;
+ /**
+ * Creates a new {@link BinMapWriter} instance.
+ *
+ * @param directory output directory
+ * @param state current segment write state
+ * @param docToBin mapping from docID to bin ID
+ * @param binCount total number of bins
+ */
+ public BinMapWriter(Directory directory, SegmentWriteState state, int[] docToBin, int binCount) {
+ this.directory = directory;
+ this.state = state;
+ this.maxDoc = state.segmentInfo.maxDoc();
+ this.binCount = binCount;
+ this.bins = docToBin;
+ }
- /**
- * Creates a new {@link BinMapWriter} instance.
- *
- * @param directory output directory
- * @param state current segment write state
- * @param docToBin mapping from docID to bin ID
- * @param binCount total number of bins
- */
- public BinMapWriter(Directory directory, SegmentWriteState state, int[] docToBin, int binCount) {
- this.directory = directory;
- this.state = state;
- this.maxDoc = state.segmentInfo.maxDoc();
- this.binCount = binCount;
- this.bins = docToBin;
- }
+ /**
+ * Writes the binmap file to the index output directory.
+ *
+ * <p>The format is:
+ *
+ * <ul>
+ * <li>Header (versioned with {@link org.apache.lucene.codecs.CodecUtil})
+ * <li>maxDoc (int)
+ * <li>binCount (int)
+ * <li>{@code maxDoc} bin IDs, one int per document
+ * <li>Footer
+ * </ul>
+ */
+ @Override
+ public void close() throws IOException {
+ final String fileName =
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- /**
- * Writes the binmap file to the index output directory.
- *
- * <p>The format is:
- *
- * <ul>
- * <li>Header (versioned with {@link org.apache.lucene.codecs.CodecUtil})
- * <li>maxDoc (int)
- * <li>binCount (int)
- * <li>{@code maxDoc} bin IDs, one int per document
- * <li>Footer
- * </ul>
- */
- @Override
- public void close() throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- IndexOutput out = null;
- boolean success = false;
- try {
- out = directory.createOutput(fileName, state.context);
- CodecUtil.writeIndexHeader(
- out, CODEC_NAME, VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
- out.writeInt(maxDoc);
- out.writeInt(binCount);
- for (int i = 0; i < maxDoc; i++) {
- out.writeInt(bins[i]);
- }
- CodecUtil.writeFooter(out);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(out);
- } else {
- IOUtils.closeWhileHandlingException(out);
- IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
- }
+ // Prevent double-write in case close() is called multiple times
+ for (String existing : directory.listAll()) {
+ if (existing.equals(fileName)) {
+ return; // already written
+ }
+ }
+ IndexOutput out = null;
+ boolean success = false;
+ try {
+ out = directory.createOutput(fileName, state.context);
+ CodecUtil.writeIndexHeader(
+ out, CODEC_NAME, VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
+ out.writeInt(maxDoc);
+ out.writeInt(binCount);
+ for (int i = 0; i < maxDoc; i++) {
+ out.writeInt(bins[i]);
+ }
+ CodecUtil.writeFooter(out);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(out);
+ } else {
+ IOUtils.closeWhileHandlingException(out);
+ IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
+ }
+ }
}
- }
}
(lucene) 08/18: And more cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit e25e6f7093861f517d2ae8aa9cbd8b45786dff95
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:24:51 2025 +0530
And more cleanup
---
.../apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
index 4da55ce9026..852d3ecc4eb 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/BinningIndexingLatencyBenchmark.java
@@ -109,7 +109,7 @@ public class BinningIndexingLatencyBenchmark {
path -> {
try {
Files.deleteIfExists(path);
- } catch (IOException ignored) {
+ } catch (IOException _) {
// best-effort cleanup
}
});
(lucene) 03/18: Remove redundant benchmarks
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 88f6046025772ea21c7ef9c645d9d4f53735b52a
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 01:46:03 2025 +0530
Remove redundant benchmarks
---
.../jmh/AnytimeRankingRecallStressBenchmark.java | 217 ---------------------
.../jmh/AnytimeSLARelevanceBenchmark.java | 147 --------------
2 files changed, 364 deletions(-)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
deleted file mode 100644
index d51a6d3582f..00000000000
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.benchmark.jmh;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.AnytimeRankingSearcher;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.similarities.BM25Similarity;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.TearDown;
-import org.openjdk.jmh.annotations.Warmup;
-
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@State(Scope.Benchmark)
-@Warmup(iterations = 1)
-@Measurement(iterations = 3)
-@Fork(1)
-public class AnytimeRankingRecallStressBenchmark {
-
- private Directory directory;
- private IndexReader reader;
- private IndexSearcher baselineSearcher;
- private AnytimeRankingSearcher anytimeSearcher;
- private TermQuery query;
-
- @Param({"20000"})
- private int docCount;
-
- @Param({"1", "3"})
- private int sla;
-
- @Setup
- public void setup() throws Exception {
- Path path = Files.createTempDirectory("recall-stress");
- directory = new MMapDirectory(path);
- IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
- config.setCodec(new Lucene101Codec());
- config.setUseCompoundFile(false);
- config.setMaxBufferedDocs(100);
-
- FieldType ft = new FieldType();
- ft.setTokenized(true);
- ft.setStored(false);
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- ft.putAttribute("postingsFormat", "Lucene101");
- ft.putAttribute("doBinning", "true");
- ft.putAttribute("bin.count", "4");
- ft.freeze();
-
- try (IndexWriter writer = new IndexWriter(directory, config)) {
- for (int i = 0; i < docCount; i++) {
- Document doc = new Document();
- String content = (i % 200 == 0) ? "lucene precision recall document" : "random junk text";
- doc.add(new Field("field", content, ft));
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- reader = DirectoryReader.open(directory);
- baselineSearcher = new IndexSearcher(reader);
- baselineSearcher.setSimilarity(new BM25Similarity());
- anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, sla, "field");
- query = new TermQuery(new Term("field", "lucene"));
- }
-
- @Benchmark
- public float baselineRecall() throws IOException {
- TopDocs topDocs = baselineSearcher.search(query, 10);
- return countRelevant(topDocs) / 10.0f;
- }
-
- @Benchmark
- public float anytimeRecall() throws IOException {
- TopDocs topDocs = anytimeSearcher.search(query);
- return countRelevant(topDocs) / 10.0f;
- }
-
- @Benchmark
- public float baselineMRR() throws IOException {
- TopDocs topDocs = baselineSearcher.search(query, 10);
- return computeMRR(topDocs);
- }
-
- @Benchmark
- public float anytimeMRR() throws IOException {
- TopDocs topDocs = anytimeSearcher.search(query);
- return computeMRR(topDocs);
- }
-
- @Benchmark
- public float baselineNDCG() throws IOException {
- TopDocs topDocs = baselineSearcher.search(query, 10);
- return computeNDCG(topDocs);
- }
-
- @Benchmark
- public float anytimeNDCG() throws IOException {
- TopDocs topDocs = anytimeSearcher.search(query);
- return computeNDCG(topDocs);
- }
-
- private int countRelevant(TopDocs topDocs) {
- int relevant = 0;
- for (ScoreDoc doc : topDocs.scoreDocs) {
- if (doc.doc % 200 == 0) {
- relevant++;
- }
- }
- return relevant;
- }
-
- private float computeMRR(TopDocs topDocs) {
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- if (topDocs.scoreDocs[i].doc % 200 == 0) {
- return 1.0f / (i + 1);
- }
- }
- return 0f;
- }
-
- private float computeNDCG(TopDocs topDocs) {
- float dcg = 0f;
- float idcg = 0f;
- int count = 0;
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- int rank = i + 1;
- if (topDocs.scoreDocs[i].doc % 200 == 0) {
- dcg += 1.0 / (Math.log(rank + 1) / Math.log(2));
- count++;
- }
- }
- for (int j = 1; j <= count; j++) {
- idcg += 1.0 / (Math.log(j + 1) / Math.log(2));
- }
- return idcg > 0 ? dcg / idcg : 0f;
- }
-
- @TearDown
- public void tearDown() throws IOException {
- reader.close();
- directory.close();
- }
-
- private static final class SingleTokenAnalyzer extends Analyzer {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer =
- new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) return false;
- clearAttributes();
- termAttr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
- return new TokenStreamComponents(tokenizer);
- }
- }
-}
\ No newline at end of file
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java
deleted file mode 100644
index 519b7399d22..00000000000
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeSLARelevanceBenchmark.java
+++ /dev/null
@@ -1,147 +0,0 @@
-package org.apache.lucene.benchmark.jmh;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.concurrent.TimeUnit;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.AnytimeRankingSearcher;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.search.similarities.BM25Similarity;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import org.openjdk.jmh.annotations.*;
-
-@BenchmarkMode(Mode.Throughput)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@State(Scope.Benchmark)
-@Warmup(iterations = 2)
-@Measurement(iterations = 5)
-@Fork(1)
-public class AnytimeSLARelevanceBenchmark {
-
- private Directory directory;
- private IndexReader reader;
- private IndexSearcher baselineSearcher;
- private AnytimeRankingSearcher anytimeSearcher;
- private TermQuery query;
-
- @Param({"1000", "10000"})
- private int docCount;
-
- @Param({"1", "5", "10"})
- private int slaScoreBudget;
-
- @Setup
- public void setup() throws Exception {
- Path tempDir = Files.createTempDirectory("sla-relevance-benchmark");
- directory = new MMapDirectory(tempDir);
- IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
- config.setCodec(new Lucene101Codec());
- config.setUseCompoundFile(false);
-
- FieldType fieldType = new FieldType();
- fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- fieldType.setTokenized(true);
- fieldType.setStored(false);
- fieldType.putAttribute("postingsFormat", "Lucene101");
- fieldType.putAttribute("doBinning", "true");
- fieldType.putAttribute("bin.count", "4");
- fieldType.freeze();
-
- try (IndexWriter writer = new IndexWriter(directory, config)) {
- for (int i = 0; i < docCount; i++) {
- Document doc = new Document();
- // Every 200th document is a high-quality relevant one
- String content = (i % 200 == 0) ? "lucene high quality content" : "filler junk text";
- doc.add(new Field("field", content, fieldType));
- writer.addDocument(doc);
- }
- writer.commit();
- writer.forceMerge(1);
- }
-
- reader = DirectoryReader.open(directory);
- baselineSearcher = new IndexSearcher(reader);
- baselineSearcher.setSimilarity(new BM25Similarity());
-
- anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, slaScoreBudget, "field");
- query = new TermQuery(new Term("field", "lucene"));
- }
-
- @Benchmark
- public int baselineRecallUnderSLA() throws IOException {
- TopDocs topDocs = baselineSearcher.search(query, 10);
- return countRelevant(topDocs);
- }
-
- @Benchmark
- public int anytimeRecallUnderSLA() throws IOException {
- TopDocs topDocs = anytimeSearcher.search(query);
- return countRelevant(topDocs);
- }
-
- private int countRelevant(TopDocs topDocs) {
- int relevant = 0;
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- if (topDocs.scoreDocs[i].doc % 200 == 0) {
- relevant++;
- }
- }
- return relevant;
- }
-
- @TearDown
- public void tearDown() throws IOException {
- reader.close();
- directory.close();
- }
-
- /** Analyzer that emits a single fixed token to eliminate dependency on analyzer modules. */
- private static final class SingleTokenAnalyzer extends Analyzer {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer =
- new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) return false;
- clearAttributes();
- termAttr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
- return new TokenStreamComponents(tokenizer);
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return reader;
- }
- }
-}
(lucene) 12/18: cleanup and add licenses
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 9febffa50e4dc35c01d792bd8f7e92bbe2182603
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 15:30:54 2025 +0530
cleanup and add licenses
---
.../apache/lucene/codecs/BinQualityMetrics.java | 16 +++++++++++++++
.../java/org/apache/lucene/index/BinScoreUtil.java | 16 +++++++++++++++
.../lucene/search/AnytimeRankingCollector.java | 16 +++++++++++++++
.../search/AnytimeRankingCollectorManager.java | 16 +++++++++++++++
.../lucene/search/AnytimeRankingSearcher.java | 16 +++++++++++++++
.../org/apache/lucene/search/DuplicateCluster.java | 24 ----------------------
.../lucene/codecs/TestBinQualityMetrics.java | 16 +++++++++++++++
.../lucene/codecs/TestDocBinningGraphBuilder.java | 16 +++++++++++++++
.../apache/lucene/codecs/TestDocGraphBuilder.java | 16 +++++++++++++++
9 files changed, 128 insertions(+), 24 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java b/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
index 8c4233233b3..42c3c7eb18b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinQualityMetrics.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.codecs;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
index ab8f0c3af5c..51fb1fba439 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.index;
import java.io.Closeable;
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
index 2c6b43f347f..dd16c1e77ea 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollector.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.search;
import java.io.IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java
index 9e00b6421c8..982fd7b62ca 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingCollectorManager.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.search;
import java.io.IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
index 5390bb05e04..25b557779de 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.search;
import java.io.IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java b/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
deleted file mode 100644
index df8e4338e51..00000000000
--- a/lucene/core/src/java/org/apache/lucene/search/DuplicateCluster.java
+++ /dev/null
@@ -1,24 +0,0 @@
-package org.apache.lucene.search;
-
-import java.util.Collections;
-import java.util.Set;
-
-/** Represents a group of near-duplicate documents. */
-public final class DuplicateCluster {
-
- private final Set<Integer> docIDs;
-
- public DuplicateCluster(Set<Integer> docIDs) {
- this.docIDs = Collections.unmodifiableSet(docIDs);
- }
-
- /** Returns the set of global document IDs in this cluster. */
- public Set<Integer> docIds() {
- return docIDs;
- }
-
- @Override
- public String toString() {
- return "DuplicateCluster" + docIDs;
- }
-}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java b/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
index f44898ee7ea..78c48d99711 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestBinQualityMetrics.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.codecs;
import java.util.Random;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestDocBinningGraphBuilder.java b/lucene/core/src/test/org/apache/lucene/codecs/TestDocBinningGraphBuilder.java
index d6f292c89a2..c9052a0eb20 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestDocBinningGraphBuilder.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestDocBinningGraphBuilder.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.codecs;
import java.io.IOException;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestDocGraphBuilder.java b/lucene/core/src/test/org/apache/lucene/codecs/TestDocGraphBuilder.java
index b67e2413233..3c7a3ab4586 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/TestDocGraphBuilder.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/TestDocGraphBuilder.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.codecs;
import java.io.IOException;
(lucene) 14/18: More cleanup and tidy output
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit d66c37d6993b39985f79b5be905cd7aafa3e58f6
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 15:34:21 2025 +0530
More cleanup and tidy output
---
.../org/apache/lucene/codecs/BinMapReader.java | 140 ++++++++++-----------
.../apache/lucene/search/BinBoostCalculator.java | 114 ++++++++---------
2 files changed, 121 insertions(+), 133 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
index 4d39c8d43cf..60fa9ad2873 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
@@ -12,7 +12,6 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
-
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
@@ -28,89 +27,84 @@ import org.apache.lucene.store.Directory;
*/
public final class BinMapReader implements Closeable {
- private static final String EXTENSION = "binmap";
- private static final int VERSION_START = 0;
- private static final String CODEC_NAME = "BinMap";
+ private static final String EXTENSION = "binmap";
+ private static final int VERSION_START = 0;
+ private static final String CODEC_NAME = "BinMap";
- private final int[] bins;
- private final int binCount;
+ private final int[] bins;
+ private final int binCount;
- /**
- * Loads the binmap file for the given segment.
- *
- * @param dir directory containing the segment files
- * @param state read state for the segment
- * @throws IOException if the file cannot be read or fails validation
- */
- public BinMapReader(Directory dir, SegmentReadState state) throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+ /**
+ * Loads the binmap file for the given segment.
+ *
+ * @param dir directory containing the segment files
+ * @param state read state for the segment
+ * @throws IOException if the file cannot be read or fails validation
+ */
+ public BinMapReader(Directory dir, SegmentReadState state) throws IOException {
+ final String fileName =
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- ChecksumIndexInput in = dir.openChecksumInput(fileName);
- boolean success = false;
- try {
- CodecUtil.checkIndexHeader(
- in,
- CODEC_NAME,
- VERSION_START,
- VERSION_START,
- state.segmentInfo.getId(),
- state.segmentSuffix);
+ ChecksumIndexInput in = dir.openChecksumInput(fileName);
+ boolean success = false;
+ try {
+ CodecUtil.checkIndexHeader(
+ in,
+ CODEC_NAME,
+ VERSION_START,
+ VERSION_START,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
- int maxDoc = in.readInt();
- this.binCount = in.readInt();
+ int maxDoc = in.readInt();
+ this.binCount = in.readInt();
- if (maxDoc != state.segmentInfo.maxDoc()) {
- throw new IOException(
- "maxDoc mismatch: expected=" + state.segmentInfo.maxDoc() + " actual=" + maxDoc);
- }
+ if (maxDoc != state.segmentInfo.maxDoc()) {
+ throw new IOException(
+ "maxDoc mismatch: expected=" + state.segmentInfo.maxDoc() + " actual=" + maxDoc);
+ }
- this.bins = new int[maxDoc];
- for (int i = 0; i < maxDoc; i++) {
- bins[i] = in.readInt();
- }
+ this.bins = new int[maxDoc];
+ for (int i = 0; i < maxDoc; i++) {
+ bins[i] = in.readInt();
+ }
- CodecUtil.checkFooter(in);
- success = true;
- } finally {
- if (success == false) {
- try {
- in.close();
- } catch (Throwable _) {
- }
- } else {
- in.close();
- }
+ CodecUtil.checkFooter(in);
+ success = true;
+ } finally {
+ if (success == false) {
+ try {
+ in.close();
+ } catch (Throwable _) {
}
+ } else {
+ in.close();
+ }
}
+ }
- /**
- * Returns the bin ID assigned to the given document.
- *
- * @param docID document ID
- * @return the bin ID
- */
- public int getBin(int docID) {
- return bins[docID];
- }
+ /**
+ * Returns the bin ID assigned to the given document.
+ *
+ * @param docID document ID
+ * @return the bin ID
+ */
+ public int getBin(int docID) {
+ return bins[docID];
+ }
- /**
- * Returns the total number of bins used in this segment.
- */
- public int getBinCount() {
- return binCount;
- }
+ /** Returns the total number of bins used in this segment. */
+ public int getBinCount() {
+ return binCount;
+ }
- /**
- * Returns a copy of the full bin assignment array.
- */
- public int[] getBinArrayCopy() {
- int[] copy = new int[bins.length];
- System.arraycopy(bins, 0, copy, 0, bins.length);
- return copy;
- }
+ /** Returns a copy of the full bin assignment array. */
+ public int[] getBinArrayCopy() {
+ int[] copy = new int[bins.length];
+ System.arraycopy(bins, 0, copy, 0, bins.length);
+ return copy;
+ }
- @Override
- public void close() {
- }
+ @Override
+ public void close() {}
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
index da0657eedf2..f1a9e98a32a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
@@ -17,7 +17,6 @@
package org.apache.lucene.search;
import java.io.IOException;
-
import org.apache.lucene.index.BinScoreLeafReader;
import org.apache.lucene.index.BinScoreReader;
import org.apache.lucene.index.IndexReader;
@@ -25,76 +24,71 @@ import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
-/**
- * Computes per-bin score boosts using inverse frequency heuristics.
- */
+/** Computes per-bin score boosts using inverse frequency heuristics. */
final class BinBoostCalculator {
- private BinBoostCalculator() {
- }
+ private BinBoostCalculator() {}
- /**
- * Computes bin-level boosts using inverse frequency weighting with log scaling.
- */
- static float[] compute(IndexReader reader) throws IOException {
- int[] binCounts = new int[8]; // start small, grow as needed
- int totalDocs = 0;
- int maxBin = -1;
+ /** Computes bin-level boosts using inverse frequency weighting with log scaling. */
+ static float[] compute(IndexReader reader) throws IOException {
+ int[] binCounts = new int[8]; // start small, grow as needed
+ int totalDocs = 0;
+ int maxBin = -1;
- for (LeafReaderContext ctx : reader.leaves()) {
- LeafReader leaf = ctx.reader();
- BinScoreReader binReader =
- (leaf instanceof BinScoreLeafReader)
- ? ((BinScoreLeafReader) leaf).getBinScoreReader()
- : null;
- if (binReader == null) {
- continue;
- }
+ for (LeafReaderContext ctx : reader.leaves()) {
+ LeafReader leaf = ctx.reader();
+ BinScoreReader binReader =
+ (leaf instanceof BinScoreLeafReader)
+ ? ((BinScoreLeafReader) leaf).getBinScoreReader()
+ : null;
+ if (binReader == null) {
+ continue;
+ }
- for (int docID = 0; docID < leaf.maxDoc(); docID++) {
- int bin = binReader.getBinForDoc(docID);
- if (bin >= binCounts.length) {
- int newLength = Math.max(bin + 1, binCounts.length * 2);
- binCounts = ArrayUtil.growExact(binCounts, newLength);
- }
- binCounts[bin]++;
- totalDocs++;
- maxBin = Math.max(maxBin, bin);
- }
+ for (int docID = 0; docID < leaf.maxDoc(); docID++) {
+ int bin = binReader.getBinForDoc(docID);
+ if (bin >= binCounts.length) {
+ int newLength = Math.max(bin + 1, binCounts.length * 2);
+ binCounts = ArrayUtil.growExact(binCounts, newLength);
}
+ binCounts[bin]++;
+ totalDocs++;
+ maxBin = Math.max(maxBin, bin);
+ }
+ }
- if (totalDocs == 0 || maxBin < 0) {
- return new float[0];
- }
+ if (totalDocs == 0 || maxBin < 0) {
+ return new float[0];
+ }
- float[] boosts = new float[maxBin + 1];
- float maxBoost = 0f;
+ float[] boosts = new float[maxBin + 1];
+ float maxBoost = 0f;
- for (int bin = 0; bin <= maxBin; bin++) {
- int count = binCounts[bin];
- if (count == 0) {
- boosts[bin] = 0f;
- continue;
- }
+ for (int bin = 0; bin <= maxBin; bin++) {
+ int count = binCounts[bin];
+ if (count == 0) {
+ boosts[bin] = 0f;
+ continue;
+ }
- // Boost is proportional to inverse frequency
- float freq = (float) count / totalDocs;
- float boost = (float) Math.log((1.0f + 1.0f / freq)); // avoid log(0)
- boosts[bin] = boost;
- maxBoost = Math.max(maxBoost, boost);
- }
+ // Boost is proportional to inverse frequency
+ float freq = (float) count / totalDocs;
+ float boost = (float) Math.log((1.0f + 1.0f / freq)); // avoid log(0)
+ boosts[bin] = boost;
+ maxBoost = Math.max(maxBoost, boost);
+ }
- // Normalize boosts to the range [1.0, max], where 1.0 is neutral
- if (maxBoost > 0f) {
- for (int i = 0; i < boosts.length; i++) {
- if (boosts[i] == 0f) {
- continue;
- }
- // Scale so the most underrepresented bin gets higher score
- boosts[i] = 1.0f + (boosts[i] / maxBoost); // final boost ∈ [1.0, 2.0]
- }
+ // Normalize boosts to the range [1.0, max], where 1.0 is neutral
+ if (maxBoost > 0f) {
+ for (int i = 0; i < boosts.length; i++) {
+ if (boosts[i] == 0f) {
+ continue;
}
-
- return boosts;
+ // Scale so the most underrepresented bin gets higher score
+ boosts[i] = 1.0f + (boosts[i] / maxBoost); // final boost ∈ [1.0, 2.0]
+ }
}
+
+ return boosts;
+ }
}
(lucene) 09/18: More cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 64093d6869b02e43e38cf59ab28bd0a57427102f
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:38:38 2025 +0530
More cleanup
---
.../org/apache/lucene/codecs/BinMapReader.java | 139 +++++++++++----------
.../apache/lucene/search/BinBoostCalculator.java | 114 +++++++++--------
.../lucene/search/TestNearDuplicateFinder.java | 112 -----------------
3 files changed, 135 insertions(+), 230 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
index ae2b0f265ab..908b391291f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
@@ -13,10 +13,12 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
+
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.ArrayUtil;
/**
* Reads the binmap file for a segment and exposes per-document bin assignments.
@@ -28,82 +30,89 @@ import org.apache.lucene.store.Directory;
*/
public final class BinMapReader implements Closeable {
- private static final String EXTENSION = "binmap";
- private static final int VERSION_START = 0;
- private static final String CODEC_NAME = "BinMap";
+ private static final String EXTENSION = "binmap";
+ private static final int VERSION_START = 0;
+ private static final String CODEC_NAME = "BinMap";
- private final int[] bins;
- private final int binCount;
+ private final int[] bins;
+ private final int binCount;
- /**
- * Loads the binmap file for the given segment.
- *
- * @param dir directory containing the segment files
- * @param state read state for the segment
- * @throws IOException if the file cannot be read or fails validation
- */
- public BinMapReader(Directory dir, SegmentReadState state) throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+ /**
+ * Loads the binmap file for the given segment.
+ *
+ * @param dir directory containing the segment files
+ * @param state read state for the segment
+ * @throws IOException if the file cannot be read or fails validation
+ */
+ public BinMapReader(Directory dir, SegmentReadState state) throws IOException {
+ final String fileName =
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- ChecksumIndexInput in = dir.openChecksumInput(fileName);
- boolean success = false;
- try {
- CodecUtil.checkIndexHeader(
- in,
- CODEC_NAME,
- VERSION_START,
- VERSION_START,
- state.segmentInfo.getId(),
- state.segmentSuffix);
+ ChecksumIndexInput in = dir.openChecksumInput(fileName);
+ boolean success = false;
+ try {
+ CodecUtil.checkIndexHeader(
+ in,
+ CODEC_NAME,
+ VERSION_START,
+ VERSION_START,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
- int maxDoc = in.readInt();
- this.binCount = in.readInt();
+ int maxDoc = in.readInt();
+ this.binCount = in.readInt();
- if (maxDoc != state.segmentInfo.maxDoc()) {
- throw new IOException(
- "maxDoc mismatch: expected=" + state.segmentInfo.maxDoc() + " actual=" + maxDoc);
- }
+ if (maxDoc != state.segmentInfo.maxDoc()) {
+ throw new IOException(
+ "maxDoc mismatch: expected=" + state.segmentInfo.maxDoc() + " actual=" + maxDoc);
+ }
- this.bins = new int[maxDoc];
- for (int i = 0; i < maxDoc; i++) {
- bins[i] = in.readInt();
- }
+ this.bins = new int[maxDoc];
+ for (int i = 0; i < maxDoc; i++) {
+ bins[i] = in.readInt();
+ }
- CodecUtil.checkFooter(in);
- success = true;
- } finally {
- if (success == false) {
- try {
- in.close();
- } catch (Throwable _) {
+ CodecUtil.checkFooter(in);
+ success = true;
+ } finally {
+ if (success == false) {
+ try {
+ in.close();
+ } catch (Throwable _) {
+ }
+ } else {
+ in.close();
+ }
}
- } else {
- in.close();
- }
}
- }
- /**
- * Returns the bin ID assigned to the given document.
- *
- * @param docID document ID
- * @return the bin ID
- */
- public int getBin(int docID) {
- return bins[docID];
- }
+ /**
+ * Returns the bin ID assigned to the given document.
+ *
+ * @param docID document ID
+ * @return the bin ID
+ */
+ public int getBin(int docID) {
+ return bins[docID];
+ }
- /** Returns the total number of bins used in this segment. */
- public int getBinCount() {
- return binCount;
- }
+ /**
+ * Returns the total number of bins used in this segment.
+ */
+ public int getBinCount() {
+ return binCount;
+ }
- /** Returns a copy of the full bin assignment array. */
- public int[] getBinArrayCopy() {
- return Arrays.copyOf(bins, bins.length);
- }
+ /**
+ * Returns a copy of the full bin assignment array.
+ */
+ public int[] getBinArrayCopy() {
+ int[] copy = new int[bins.length];
+ System.arraycopy(bins, 0, copy, 0, bins.length);
+ return copy;
+ }
- @Override
- public void close() {}
+ @Override
+ public void close() {
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
index d3dd5a0d7c6..210438e8c42 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
@@ -18,76 +18,84 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Arrays;
+
import org.apache.lucene.index.BinScoreLeafReader;
import org.apache.lucene.index.BinScoreReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.util.ArrayUtil;
-/** Computes per-bin score boosts using inverse frequency heuristics. */
+/**
+ * Computes per-bin score boosts using inverse frequency heuristics.
+ */
final class BinBoostCalculator {
- private BinBoostCalculator() {}
+ private BinBoostCalculator() {
+ }
- /** Computes bin-level boosts using inverse frequency weighting with log scaling. */
- static float[] compute(IndexReader reader) throws IOException {
- int[] binCounts = new int[8]; // start small, grow as needed
- int totalDocs = 0;
- int maxBin = -1;
+ /**
+ * Computes bin-level boosts using inverse frequency weighting with log scaling.
+ */
+ static float[] compute(IndexReader reader) throws IOException {
+ int[] binCounts = new int[8]; // start small, grow as needed
+ int totalDocs = 0;
+ int maxBin = -1;
- for (LeafReaderContext ctx : reader.leaves()) {
- LeafReader leaf = ctx.reader();
- BinScoreReader binReader =
- (leaf instanceof BinScoreLeafReader)
- ? ((BinScoreLeafReader) leaf).getBinScoreReader()
- : null;
- if (binReader == null) {
- continue;
- }
+ for (LeafReaderContext ctx : reader.leaves()) {
+ LeafReader leaf = ctx.reader();
+ BinScoreReader binReader =
+ (leaf instanceof BinScoreLeafReader)
+ ? ((BinScoreLeafReader) leaf).getBinScoreReader()
+ : null;
+ if (binReader == null) {
+ continue;
+ }
- for (int docID = 0; docID < leaf.maxDoc(); docID++) {
- int bin = binReader.getBinForDoc(docID);
- if (bin >= binCounts.length) {
- binCounts = Arrays.copyOf(binCounts, Math.max(bin + 1, binCounts.length * 2));
+ for (int docID = 0; docID < leaf.maxDoc(); docID++) {
+ int bin = binReader.getBinForDoc(docID);
+ if (bin >= binCounts.length) {
+ int newLength = Math.max(bin + 1, binCounts.length * 2);
+ binCounts = ArrayUtil.growExact(binCounts, newLength);
+ }
+ binCounts[bin]++;
+ totalDocs++;
+ maxBin = Math.max(maxBin, bin);
+ }
}
- binCounts[bin]++;
- totalDocs++;
- maxBin = Math.max(maxBin, bin);
- }
- }
- if (totalDocs == 0 || maxBin < 0) {
- return new float[0];
- }
+ if (totalDocs == 0 || maxBin < 0) {
+ return new float[0];
+ }
- float[] boosts = new float[maxBin + 1];
- float maxBoost = 0f;
+ float[] boosts = new float[maxBin + 1];
+ float maxBoost = 0f;
- for (int bin = 0; bin <= maxBin; bin++) {
- int count = binCounts[bin];
- if (count == 0) {
- boosts[bin] = 0f;
- continue;
- }
+ for (int bin = 0; bin <= maxBin; bin++) {
+ int count = binCounts[bin];
+ if (count == 0) {
+ boosts[bin] = 0f;
+ continue;
+ }
- // Boost is proportional to inverse frequency
- float freq = (float) count / totalDocs;
- float boost = (float) Math.log((1.0f + 1.0f / freq)); // avoid log(0)
- boosts[bin] = boost;
- maxBoost = Math.max(maxBoost, boost);
- }
+ // Boost is proportional to inverse frequency
+ float freq = (float) count / totalDocs;
+ float boost = (float) Math.log((1.0f + 1.0f / freq)); // avoid log(0)
+ boosts[bin] = boost;
+ maxBoost = Math.max(maxBoost, boost);
+ }
- // Normalize boosts to the range [1.0, max], where 1.0 is neutral
- if (maxBoost > 0f) {
- for (int i = 0; i < boosts.length; i++) {
- if (boosts[i] == 0f) {
- continue;
+ // Normalize boosts to the range [1.0, max], where 1.0 is neutral
+ if (maxBoost > 0f) {
+ for (int i = 0; i < boosts.length; i++) {
+ if (boosts[i] == 0f) {
+ continue;
+ }
+ // Scale so the most underrepresented bin gets higher score
+ boosts[i] = 1.0f + (boosts[i] / maxBoost); // final boost ∈ [1.0, 2.0]
+ }
}
- // Scale so the most underrepresented bin gets higher score
- boosts[i] = 1.0f + (boosts[i] / maxBoost); // final boost ∈ [1.0, 2.0]
- }
- }
- return boosts;
- }
+ return boosts;
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java b/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
deleted file mode 100644
index 9cf41640632..00000000000
--- a/lucene/core/src/test/org/apache/lucene/search/TestNearDuplicateFinder.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0.
- */
-package org.apache.lucene.search;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.LuceneTestCase;
-
-public class TestNearDuplicateFinder extends LuceneTestCase {
-
- private FieldType buildTextType() {
- FieldType type = new FieldType(TextField.TYPE_STORED);
- type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- type.setStoreTermVectors(true);
- type.setStoreTermVectorPositions(true);
- type.freeze();
- return type;
- }
-
- private Directory buildIndex(String... docs) throws IOException {
- Directory dir =
- new MockDirectoryWrapper(
- random(), new MMapDirectory(createTempDir("TestNearDuplicateFinder")));
- IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
- try (IndexWriter writer = new IndexWriter(dir, config)) {
- FieldType fieldType = buildTextType();
- for (String content : docs) {
- Document doc = new Document();
- doc.add(new Field("body", content, fieldType));
- writer.addDocument(doc);
- }
- }
- return dir;
- }
-
- public void testExactDuplicates() throws IOException {
- Directory dir = buildIndex("alpha beta gamma", "alpha beta gamma", "delta epsilon");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finder = new NearDuplicateFinder("body", 0.95f);
- List<DuplicateCluster> clusters = finder.findNearDuplicates(reader);
- assertEquals(1, clusters.size());
- Set<Integer> dup = clusters.get(0).docIds();
- assertTrue(dup.contains(0));
- assertTrue(dup.contains(1));
- }
- }
-
- public void testPartialOverlap() throws IOException {
- Directory dir = buildIndex("quick brown fox", "quick brown dog", "lazy dog sleeps");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finder = new NearDuplicateFinder("body", 0.5f);
- List<DuplicateCluster> clusters = finder.findNearDuplicates(reader);
- assertEquals(1, clusters.size());
- }
- }
-
- public void testThresholdSensitivity() throws IOException {
- Directory dir = buildIndex("a b c d e", "a b c", "x y z");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finderHigh = new NearDuplicateFinder("body", 0.9f);
- NearDuplicateFinder finderLow = new NearDuplicateFinder("body", 0.3f);
- List<DuplicateCluster> high = finderHigh.findNearDuplicates(reader);
- List<DuplicateCluster> low = finderLow.findNearDuplicates(reader);
- assertEquals(0, high.size());
- assertTrue(low.size() >= 1);
- }
- }
-
- public void testEmptyDocs() throws IOException {
- Directory dir = buildIndex("", "", "filled text content");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finder = new NearDuplicateFinder("body", 0.9f);
- List<DuplicateCluster> clusters = finder.findNearDuplicates(reader);
- assertEquals(1, clusters.size());
- }
- }
-
- public void testMultipleClusters() throws IOException {
- Directory dir = buildIndex("apple orange", "apple orange", "car bus", "car bus", "train");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finder = new NearDuplicateFinder("body", 0.95f);
- List<DuplicateCluster> clusters = finder.findNearDuplicates(reader);
- assertEquals(2, clusters.size());
- }
- }
-
- public void testNoDuplicates() throws IOException {
- Directory dir = buildIndex("one", "two", "three", "four");
- try (DirectoryReader reader = DirectoryReader.open(dir)) {
- NearDuplicateFinder finder = new NearDuplicateFinder("body", 0.95f);
- List<DuplicateCluster> clusters = finder.findNearDuplicates(reader);
- assertEquals(0, clusters.size());
- }
- }
-}
(lucene) 10/18: Continued cleanups
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 66ef86399fb28a93bbbef3a49eb1a07b38fbae04
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:40:42 2025 +0530
Continued cleanups
---
.../benchmark/jmh/IndexingLatencyBenchmark.java | 147 ---------------------
.../org/apache/lucene/codecs/BinMapReader.java | 2 -
.../apache/lucene/search/BinBoostCalculator.java | 1 -
3 files changed, 150 deletions(-)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
deleted file mode 100644
index 3fe148e84e7..00000000000
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/IndexingLatencyBenchmark.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.benchmark.jmh;
-
-import java.io.IOException;
-import java.io.Reader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Comparator;
-import java.util.concurrent.TimeUnit;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MMapDirectory;
-import org.openjdk.jmh.annotations.Benchmark;
-import org.openjdk.jmh.annotations.BenchmarkMode;
-import org.openjdk.jmh.annotations.Fork;
-import org.openjdk.jmh.annotations.Level;
-import org.openjdk.jmh.annotations.Measurement;
-import org.openjdk.jmh.annotations.Mode;
-import org.openjdk.jmh.annotations.OutputTimeUnit;
-import org.openjdk.jmh.annotations.Param;
-import org.openjdk.jmh.annotations.Scope;
-import org.openjdk.jmh.annotations.Setup;
-import org.openjdk.jmh.annotations.State;
-import org.openjdk.jmh.annotations.Warmup;
-
-@BenchmarkMode(Mode.AverageTime)
-@OutputTimeUnit(TimeUnit.MILLISECONDS)
-@State(Scope.Thread)
-@Warmup(iterations = 2)
-@Measurement(iterations = 5)
-@Fork(1)
-public class IndexingLatencyBenchmark {
-
- @Param({"true", "false"})
- private String binningFlag;
-
- @Param({"1000", "10000"})
- private int docCount;
-
- private boolean binningEnabled;
- private FieldType fieldType;
- private Path tempDir;
-
- @Setup(Level.Trial)
- public void setup() {
- binningEnabled = Boolean.parseBoolean(binningFlag);
- fieldType = new FieldType(TextField.TYPE_NOT_STORED);
- fieldType.setTokenized(true);
- fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- if (binningEnabled) {
- fieldType.putAttribute("postingsFormat", "Lucene101");
- fieldType.putAttribute("doBinning", "true");
- fieldType.putAttribute("bin.count", "4");
- }
- fieldType.freeze();
- }
-
- @Benchmark
- public void indexBatch() throws IOException {
- tempDir = Files.createTempDirectory("lucene-indexing-benchmark");
- try (Directory dir = new MMapDirectory(tempDir)) {
- IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
- config.setCodec(new Lucene101Codec());
- config.setUseCompoundFile(false);
- try (IndexWriter writer = new IndexWriter(dir, config)) {
- for (int i = 0; i < docCount; i++) {
- Document doc = new Document();
- String content = (i % 200 == 0) ? "lucene high quality text" : "noise filler token " + i;
- doc.add(new Field("field", content, fieldType));
- writer.addDocument(doc);
- }
- writer.commit();
- }
- } finally {
- Files.walk(tempDir)
- .sorted(Comparator.reverseOrder())
- .forEach(
- path -> {
- try {
- Files.deleteIfExists(path);
- } catch (IOException ignored) {
- // best-effort cleanup
- }
- });
- }
- }
-
- /** Analyzer that emits a single token to avoid dependency on analyzer modules. */
- private static final class SingleTokenAnalyzer extends Analyzer {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer =
- new Tokenizer() {
- private final CharTermAttribute attr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) {
- return false;
- }
- clearAttributes();
- attr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
- return new TokenStreamComponents(tokenizer);
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return reader;
- }
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
index 908b391291f..4d39c8d43cf 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
@@ -12,13 +12,11 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
-import java.util.Arrays;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.ArrayUtil;
/**
* Reads the binmap file for a segment and exposes per-document bin assignments.
diff --git a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
index 210438e8c42..da0657eedf2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BinBoostCalculator.java
@@ -17,7 +17,6 @@
package org.apache.lucene.search;
import java.io.IOException;
-import java.util.Arrays;
import org.apache.lucene.index.BinScoreLeafReader;
import org.apache.lucene.index.BinScoreReader;
(lucene) 06/18: More cleanups
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 794bec105d22eabfdbf57c5d342d6639e16278af
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:20:00 2025 +0530
More cleanups
---
.../lucene/benchmark/jmh/AnytimeRankingBenchmark.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
index 297bec90bb7..4bfd408c13a 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingBenchmark.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.benchmark.jmh;
import java.io.IOException;
(lucene) 04/18: Cleanup, tidy output and add javadocs
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 0bd1a10f07394a5402aa52f4aede519071513c0e
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:13:01 2025 +0530
Cleanup, tidy output and add javadocs
---
.../jmh/AnytimeMultiSegmentBenchmark.java | 3 +-
.../jmh/AnytimeQueryLatencyVarianceBenchmark.java | 183 ++++++++++-----------
.../lucene/codecs/ApproximateDocGraphBuilder.java | 19 ++-
.../org/apache/lucene/codecs/BinMapReader.java | 37 ++++-
.../org/apache/lucene/codecs/BinMapWriter.java | 37 ++++-
.../org/apache/lucene/codecs/BinningSupport.java | 7 -
.../java/org/apache/lucene/codecs/BinningUtil.java | 40 -----
.../lucene/codecs/DocBinningGraphBuilder.java | 53 ++++--
.../org/apache/lucene/codecs/FieldsLeafReader.java | 6 +
.../org/apache/lucene/codecs/SparseEdgeGraph.java | 40 ++++-
.../apache/lucene/index/BinScoreLeafReader.java | 1 +
.../search/TestAnytimeRankingRelevanceMatch.java | 15 +-
.../lucene/search/TestArcaneRelevanceBoost.java | 65 ++++----
13 files changed, 306 insertions(+), 200 deletions(-)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
index 09c3884b23c..d79683f4c14 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -158,4 +157,4 @@ public class AnytimeMultiSegmentBenchmark {
return new TokenStreamComponents(tokenizer);
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
index 0e70b3f0f8f..1a90fe85952 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
@@ -21,7 +21,6 @@ import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.concurrent.TimeUnit;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -64,101 +63,101 @@ import org.openjdk.jmh.annotations.Warmup;
@Fork(1)
public class AnytimeQueryLatencyVarianceBenchmark {
- private Directory directory;
- private IndexReader reader;
- private IndexSearcher baselineSearcher;
- private AnytimeRankingSearcher anytimeSearcher;
- private TermQuery query;
-
- @Param({"1000", "10000"})
- private int docCount;
-
- @Param({"true", "false"})
- private boolean binningEnabled;
-
- @Setup
- public void setup() throws Exception {
- Path tempDir = Files.createTempDirectory("query-latency-variance");
- directory = new MMapDirectory(tempDir);
- IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
- config.setCodec(new Lucene101Codec());
- config.setUseCompoundFile(false);
- config.setMaxBufferedDocs(64);
-
- FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
- fieldType.setTokenized(true);
- fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- if (binningEnabled) {
- fieldType.putAttribute("doBinning", "true");
- fieldType.putAttribute("bin.count", "4");
- }
- fieldType.freeze();
-
- try (IndexWriter writer = new IndexWriter(directory, config)) {
- for (int i = 0; i < docCount; i++) {
- Document doc = new Document();
- String content = (i % 3 == 0) ? "lucene latency benchmark" : "random filler content";
- doc.add(new Field("field", content, fieldType));
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- reader = DirectoryReader.open(directory);
- baselineSearcher = new IndexSearcher(reader);
- baselineSearcher.setSimilarity(new BM25Similarity());
-
- anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
- query = new TermQuery(new Term("field", "lucene"));
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Setup
+ public void setup() throws Exception {
+ Path tempDir = Files.createTempDirectory("query-latency-variance");
+ directory = new MMapDirectory(tempDir);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(64);
+
+ FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binningEnabled) {
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
}
-
- @Benchmark
- public TopDocs baselineQueryLatency() throws IOException {
- return baselineSearcher.search(query, 10);
+ fieldType.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene latency benchmark" : "random filler content";
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
}
- @Benchmark
- public TopDocs anytimeQueryLatency() throws IOException {
- return anytimeSearcher.search(query);
- }
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineQueryLatency() throws IOException {
+ return baselineSearcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeQueryLatency() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
- @TearDown
- public void tearDown() throws IOException {
- reader.close();
- directory.close();
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
}
- private static final class SingleTokenAnalyzer extends Analyzer {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer =
- new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) {
- return false;
- }
- clearAttributes();
- termAttr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
- return new TokenStreamComponents(tokenizer);
- }
-
- @Override
- protected Reader initReader(String fieldName, Reader reader) {
- return reader;
- }
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
}
-}
\ No newline at end of file
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
index e60b08fc062..33a13cb3310 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
@@ -29,17 +29,34 @@ import org.apache.lucene.util.BytesRef;
*/
public final class ApproximateDocGraphBuilder {
- /* Default max number of edges per doc */
+ /** Default number of edges retained per document in the similarity graph. */
public static final int DEFAULT_MAX_EDGES = 10;
private final String field;
private final int maxEdgesPerDoc;
+ /**
+ * Creates a new graph builder for the specified field.
+ *
+ * @param field the indexed field to use for term overlap computation
+ * @param maxEdgesPerDoc maximum number of outgoing edges to retain per document
+ */
public ApproximateDocGraphBuilder(String field, int maxEdgesPerDoc) {
this.field = field;
this.maxEdgesPerDoc = maxEdgesPerDoc;
}
+ /**
+ * Constructs the similarity graph for documents in the provided reader using term co-occurrence.
+ *
+ * <p>Documents that share terms in the specified field are connected with an edge, with edge
+ * weights based on normalized overlap count. The number of outgoing edges per document is limited
+ * by {@code maxEdgesPerDoc}.
+ *
+ * @param reader a leaf reader over a single Lucene segment
+ * @return a sparse document similarity graph based on token overlap
+ * @throws IOException if an I/O error occurs during term enumeration or postings access
+ */
public SparseEdgeGraph build(LeafReader reader) throws IOException {
final int maxDoc = reader.maxDoc();
final InMemorySparseEdgeGraph graph = new InMemorySparseEdgeGraph();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
index f256bedcc25..ae2b0f265ab 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
@@ -1,3 +1,13 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
package org.apache.lucene.codecs;
import java.io.Closeable;
@@ -8,7 +18,14 @@ import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
-/** Reads bin ID assignments per doc from binmap file. */
+/**
+ * Reads the binmap file for a segment and exposes per-document bin assignments.
+ *
+ * <p>Each document is assigned to a bin during indexing. The binmap file encodes a mapping from
+ * docID to binID and is used at search time to access bin-specific metadata such as scoring boosts.
+ *
+ * <p>This class validates file integrity using Lucene’s codec header/footer checks.
+ */
public final class BinMapReader implements Closeable {
private static final String EXTENSION = "binmap";
@@ -18,6 +35,13 @@ public final class BinMapReader implements Closeable {
private final int[] bins;
private final int binCount;
+ /**
+ * Loads the binmap file for the given segment.
+ *
+ * @param dir directory containing the segment files
+ * @param state read state for the segment
+ * @throws IOException if the file cannot be read or fails validation
+ */
public BinMapReader(Directory dir, SegmentReadState state) throws IOException {
final String fileName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
@@ -52,8 +76,7 @@ public final class BinMapReader implements Closeable {
if (success == false) {
try {
in.close();
- } catch (Throwable t) {
- // suppress
+ } catch (Throwable _) {
}
} else {
in.close();
@@ -61,14 +84,22 @@ public final class BinMapReader implements Closeable {
}
}
+ /**
+ * Returns the bin ID assigned to the given document.
+ *
+ * @param docID document ID
+ * @return the bin ID
+ */
public int getBin(int docID) {
return bins[docID];
}
+ /** Returns the total number of bins used in this segment. */
public int getBinCount() {
return binCount;
}
+ /** Returns a copy of the full bin assignment array. */
public int[] getBinArrayCopy() {
return Arrays.copyOf(bins, bins.length);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
index c6fa938f432..b3f68293055 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
@@ -23,11 +23,20 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.IOUtils;
-/** Writes a binmap file that maps docIDs to bin IDs. */
+/**
+ * Writes a binmap file for a segment.
+ *
+ * <p>The binmap assigns each document in the segment to a bin ID, typically used for scoring or
+ * early termination. The output is a binary file storing one integer bin ID per document.
+ */
public final class BinMapWriter implements AutoCloseable {
+ /** File extension for binmap files. */
public static final String EXTENSION = "binmap";
+
+ /** Initial file format version. */
public static final int VERSION_START = 0;
+
private static final String CODEC_NAME = "BinMap";
private final Directory directory;
@@ -36,6 +45,14 @@ public final class BinMapWriter implements AutoCloseable {
private final int binCount;
private final int[] bins;
+ /**
+ * Creates a new {@link BinMapWriter} instance.
+ *
+ * @param directory output directory
+ * @param state current segment write state
+ * @param docToBin mapping from docID to bin ID
+ * @param binCount total number of bins
+ */
public BinMapWriter(Directory directory, SegmentWriteState state, int[] docToBin, int binCount) {
this.directory = directory;
this.state = state;
@@ -44,11 +61,19 @@ public final class BinMapWriter implements AutoCloseable {
this.bins = docToBin;
}
- /** Returns the expected binmap file name for this writer's segment. */
- public String getBinMapFileName() {
- return IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- }
-
+ /**
+ * Writes the binmap file to the index output directory.
+ *
+ * <p>The format is:
+ *
+ * <ul>
+ * <li>Header (versioned with {@link org.apache.lucene.codecs.CodecUtil})
+ * <li>maxDoc (int)
+ * <li>binCount (int)
+ * <li>{@code maxDoc} bin IDs, one int per document
+ * <li>Footer
+ * </ul>
+ */
@Override
public void close() throws IOException {
final String fileName =
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinningSupport.java b/lucene/core/src/java/org/apache/lucene/codecs/BinningSupport.java
deleted file mode 100644
index 99a27e655c8..00000000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinningSupport.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.apache.lucene.codecs;
-
-import java.io.IOException;
-
-public interface BinningSupport {
- void finalizeBinning() throws IOException;
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinningUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/BinningUtil.java
deleted file mode 100644
index b2b46a7d210..00000000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinningUtil.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.apache.lucene.codecs;
-
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.SegmentWriteState;
-
-public final class BinningUtil {
-
- private BinningUtil() {}
-
- public static boolean isBinningEnabled(SegmentWriteState state) {
- for (FieldInfo fi : state.fieldInfos) {
- if ("true".equalsIgnoreCase(fi.getAttribute("doBinning"))) {
- return true;
- }
- }
- return false;
- }
-
- public static String getBinningField(SegmentWriteState state) {
- for (FieldInfo fi : state.fieldInfos) {
- if ("true".equalsIgnoreCase(fi.getAttribute("doBinning"))) {
- return fi.name;
- }
- }
- return null;
- }
-
- public static int getBinCount(SegmentWriteState state, int maxDoc) {
- for (FieldInfo fi : state.fieldInfos) {
- if ("true".equalsIgnoreCase(fi.getAttribute("doBinning"))) {
- String attr = fi.getAttribute("bin.count");
- if (attr != null) {
- return Integer.parseInt(attr);
- }
- return Math.max(1, Integer.highestOneBit(maxDoc >>> 4));
- }
- }
- return -1;
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
index a8bc9206c7c..7773f302e99 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocBinningGraphBuilder.java
@@ -1,14 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.lucene.codecs;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
-/** Non-recursive document binner using iterative graph bisection. */
+/**
+ * Partitions documents into bins using iterative graph bisection.
+ *
+ * <p>This utility approximates a balanced clustering of documents using a sparse similarity graph.
+ * It uses fixed seed points and edge weights to recursively assign documents to bins, ensuring that
+ * similar documents are grouped together.
+ */
public final class DocBinningGraphBuilder {
private DocBinningGraphBuilder() {}
+ /**
+ * Assigns each document to a bin by performing recursive bisection over a similarity graph.
+ *
+ * @param graph the sparse edge graph of document similarities
+ * @param maxDoc total number of documents
+ * @param numBins number of bins (must be a power of 2)
+ * @return an array mapping docID to bin ID
+ * @throws IOException if an error occurs during graph access
+ */
public static int[] computeBins(SparseEdgeGraph graph, int maxDoc, int numBins)
throws IOException {
if (maxDoc <= 0 || Integer.bitCount(numBins) != 1) {
@@ -73,6 +104,11 @@ public final class DocBinningGraphBuilder {
return docToBin;
}
+ /**
+ * Computes similarity between two documents using the sparse graph.
+ *
+ * <p>The similarity is computed as the sum of edge weights from docA to docB.
+ */
private static float similarity(SparseEdgeGraph graph, int docA, int docB) {
int[] neighbors = graph.getNeighbors(docA);
float[] weights = graph.getWeights(docA);
@@ -85,17 +121,6 @@ public final class DocBinningGraphBuilder {
return score;
}
- private static final class PartitionTask {
- final int start;
- final int end;
- final int binBase;
- final int binCount;
-
- PartitionTask(int start, int end, int binBase, int binCount) {
- this.start = start;
- this.end = end;
- this.binBase = binBase;
- this.binCount = binCount;
- }
- }
+ /** Internal task used to represent a partition step. */
+ private record PartitionTask(int start, int end, int binBase, int binCount) {}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsLeafReader.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsLeafReader.java
index dfbd42fcc4a..98998b20713 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsLeafReader.java
@@ -43,6 +43,12 @@ public final class FieldsLeafReader extends LeafReader {
private final Map<String, Terms> fieldTerms;
private final int maxDoc;
+ /**
+ * Creates a {@code FieldsLeafReader} for the given field terms.
+ *
+ * @param fieldTerms mapping of field names to {@link Terms} enumerations
+ * @param maxDoc total number of documents in this reader
+ */
public FieldsLeafReader(Map<String, Terms> fieldTerms, int maxDoc) {
this.fieldTerms = fieldTerms;
this.maxDoc = maxDoc;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/SparseEdgeGraph.java b/lucene/core/src/java/org/apache/lucene/codecs/SparseEdgeGraph.java
index fcb16fadb2e..59f2d71730e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/SparseEdgeGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/SparseEdgeGraph.java
@@ -1,14 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
package org.apache.lucene.codecs;
-/** Sparse adjacency graph interface. */
+/**
+ * Interface for a sparse directed graph of documents. Used to represent approximate similarity
+ * relationships between documents.
+ */
public interface SparseEdgeGraph {
+ /**
+ * Adds a directed edge from {@code fromDocID} to {@code toDocID} with the given weight.
+ *
+ * @param fromDocID source document ID
+ * @param toDocID destination document ID
+ * @param weight edge weight representing similarity
+ */
void addEdge(int fromDocID, int toDocID, float weight);
+ /**
+ * Returns the list of neighboring document IDs for a given source document.
+ *
+ * @param docID source document ID
+ * @return array of neighboring document IDs
+ */
int[] getNeighbors(int docID);
+ /**
+ * Returns the edge weights associated with the neighbors of the given document.
+ *
+ * @param docID source document ID
+ * @return array of weights corresponding to {@link #getNeighbors(int)}
+ */
float[] getWeights(int docID);
+ /** Returns the number of vertices in the graph. */
int size();
+ /**
+ * Ensures internal data structures can accommodate the given document ID.
+ *
+ * @param docID document ID to reserve space for
+ */
void ensureVertex(int docID);
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
index 9b58197b870..fb28686c6ee 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
@@ -26,6 +26,7 @@ public class BinScoreLeafReader extends FilterLeafReader {
private final BinScoreReader binScore;
private final Closeable[] resources;
+ /** Default constructor */
public BinScoreLeafReader(LeafReader in, BinScoreReader binScore, Closeable... resources) {
super(in);
this.binScore = binScore;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
index f2be02f635d..007b8674830 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
@@ -52,7 +52,8 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
try (IndexWriter writer = new IndexWriter(dir, config)) {
for (int i = 0; i < DOC_COUNT; i++) {
Document doc = new Document();
- String content = (i % RELEVANT_FREQ == 0) ? "lucene precision recall document" : "random filler content";
+ String content =
+ (i % RELEVANT_FREQ == 0) ? "lucene precision recall document" : "random filler content";
doc.add(new Field("field", content, ft));
writer.addDocument(doc);
}
@@ -78,9 +79,15 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
float baselineNDCG = computeNDCG(baseline);
float anytimeNDCG = computeNDCG(anytimeResults);
- System.out.printf(Locale.ROOT,
+ System.out.printf(
+ Locale.ROOT,
"Recall: baseline=%.2f anytime=%.2f | MRR: baseline=%.2f anytime=%.2f | NDCG: baseline=%.2f anytime=%.2f%n",
- baselineRecall, anytimeRecall, baselineMRR, anytimeMRR, baselineNDCG, anytimeNDCG);
+ baselineRecall,
+ anytimeRecall,
+ baselineMRR,
+ anytimeMRR,
+ baselineNDCG,
+ anytimeNDCG);
assertTrue("Anytime recall should be >= baseline", anytimeRecall >= baselineRecall);
assertTrue("Anytime MRR should be >= baseline", anytimeMRR >= baselineMRR);
@@ -152,4 +159,4 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
return new TokenStreamComponents(tokenizer);
}
}
-}
\ No newline at end of file
+}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
index 5fe2bf188c3..932d960fd1d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
@@ -19,7 +19,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.Collection;
import java.util.concurrent.atomic.AtomicInteger;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
@@ -61,7 +60,8 @@ public class TestArcaneRelevanceBoost extends LuceneTestCase {
try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc)) {
for (int i = 0; i < totalDocs; i++) {
Document doc = new Document();
- String content = (i % relevantEvery == 0) ? "lucene highly relevant text" : "noise filler irrelevant";
+ String content =
+ (i % relevantEvery == 0) ? "lucene highly relevant text" : "noise filler irrelevant";
doc.add(new StoredField("docID", i));
doc.add(new Field("content", content, ft));
writer.addDocument(doc);
@@ -79,37 +79,39 @@ public class TestArcaneRelevanceBoost extends LuceneTestCase {
TermQuery query = new TermQuery(new Term("content", "lucene"));
AtomicInteger baselineHits = new AtomicInteger();
- baseline.search(query, new CollectorManager<SimpleCollector, Void>() {
- @Override
- public SimpleCollector newCollector() {
- return new SimpleCollector() {
- private int seen = 0;
-
+ baseline.search(
+ query,
+ new CollectorManager<SimpleCollector, Void>() {
@Override
- public void collect(int doc) throws IOException {
- if (doc % relevantEvery == 0) {
- baselineHits.incrementAndGet();
- }
- if (++seen >= truncationLimit) {
- throw new CollectionTerminatedException();
- }
+ public SimpleCollector newCollector() {
+ return new SimpleCollector() {
+ private int seen = 0;
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (doc % relevantEvery == 0) {
+ baselineHits.incrementAndGet();
+ }
+ if (++seen >= truncationLimit) {
+ throw new CollectionTerminatedException();
+ }
+ }
+
+ @Override
+ public void doSetNextReader(LeafReaderContext context) {}
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.COMPLETE_NO_SCORES;
+ }
+ };
}
@Override
- public void doSetNextReader(LeafReaderContext context) {}
-
- @Override
- public ScoreMode scoreMode() {
- return ScoreMode.COMPLETE_NO_SCORES;
+ public Void reduce(Collection<SimpleCollector> collectors) {
+ return null;
}
- };
- }
-
- @Override
- public Void reduce(Collection<SimpleCollector> collectors) {
- return null;
- }
- });
+ });
TopDocs arcaneResults = arcane.search(query);
int arcaneHits = 0;
@@ -121,10 +123,13 @@ public class TestArcaneRelevanceBoost extends LuceneTestCase {
assertTrue(
"ARCANE should retrieve more relevant results under truncation. "
- + "Baseline hits=" + baselineHits.get() + ", ARCANE hits=" + arcaneHits,
+ + "Baseline hits="
+ + baselineHits.get()
+ + ", ARCANE hits="
+ + arcaneHits,
arcaneHits > baselineHits.get());
reader.close();
dir.close();
}
-}
\ No newline at end of file
+}
(lucene) 11/18: More cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit c9be5c0d30363e1eb3504094b8e90679503f2bc6
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:42:07 2025 +0530
More cleanup
---
lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java b/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
index 0fc32f84cec..2094a817ea8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
@@ -37,7 +37,7 @@ public class TestBinScoreUtil extends LuceneTestCase {
private Directory dir;
private IndexWriter writer;
- @Before
+ @Override
public void setUp() throws Exception {
super.setUp();
Path path = createTempDir("binutil");
@@ -48,7 +48,7 @@ public class TestBinScoreUtil extends LuceneTestCase {
writer = new IndexWriter(dir, iwc);
}
- @After
+ @Override
public void tearDown() throws Exception {
super.tearDown();
(lucene) 02/18: Add more benchmarks and relevance tests
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit b56a562c87187739f637e753ec77b9a86bb12ac1
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Wed Apr 16 21:21:08 2025 +0530
Add more benchmarks and relevance tests
---
.../jmh/AnytimeMultiSegmentBenchmark.java | 161 +++++++++++++++
.../jmh/AnytimeQueryLatencyVarianceBenchmark.java | 164 ++++++++++++++++
.../jmh/AnytimeRankingRecallStressBenchmark.java | 217 +++++++++++++++++++++
.../lucene/codecs/ApproximateDocGraphBuilder.java | 1 +
.../search/TestAnytimeRankingRelevanceMatch.java | 155 +++++++++++++++
.../lucene/search/TestArcaneRelevanceBoost.java | 130 ++++++++++++
6 files changed, 828 insertions(+)
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
new file mode 100644
index 00000000000..09c3884b23c
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeMultiSegmentBenchmark.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmark for multi-segment bin-aware search with AnytimeRanking. */
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeMultiSegmentBenchmark {
+
+ private Directory dir;
+ private IndexReader reader;
+ private IndexSearcher searcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "5000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binning;
+
+ @Setup
+ public void setup() throws Exception {
+ Path path = Files.createTempDirectory("benchmark-segments");
+ dir = new MMapDirectory(path);
+ IndexWriterConfig iwc = new IndexWriterConfig(new FixedTokenAnalyzer());
+ iwc.setCodec(new Lucene101Codec());
+ iwc.setUseCompoundFile(false);
+ iwc.setMaxBufferedDocs(50); // force multiple flushes
+
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setTokenized(true);
+ type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binning) {
+ type.putAttribute("postingsFormat", "Lucene101");
+ type.putAttribute("doBinning", "true");
+ type.putAttribute("bin.count", "4");
+ }
+ type.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, iwc)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene fast ranking test" : "filler junk";
+ doc.add(new Field("field", content, type));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
+ searcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(searcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineSearch() throws IOException {
+ return searcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeSearch() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws Exception {
+ reader.close();
+ dir.close();
+ }
+
+ /** Simple analyzer that emits a single token. */
+ private static final class FixedTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
new file mode 100644
index 00000000000..0e70b3f0f8f
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeQueryLatencyVarianceBenchmark.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 2)
+@Measurement(iterations = 5)
+@Fork(1)
+public class AnytimeQueryLatencyVarianceBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"1000", "10000"})
+ private int docCount;
+
+ @Param({"true", "false"})
+ private boolean binningEnabled;
+
+ @Setup
+ public void setup() throws Exception {
+ Path tempDir = Files.createTempDirectory("query-latency-variance");
+ directory = new MMapDirectory(tempDir);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(64);
+
+ FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED);
+ fieldType.setTokenized(true);
+ fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ if (binningEnabled) {
+ fieldType.putAttribute("doBinning", "true");
+ fieldType.putAttribute("bin.count", "4");
+ }
+ fieldType.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 3 == 0) ? "lucene latency benchmark" : "random filler content";
+ doc.add(new Field("field", content, fieldType));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, 5, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public TopDocs baselineQueryLatency() throws IOException {
+ return baselineSearcher.search(query, 10);
+ }
+
+ @Benchmark
+ public TopDocs anytimeQueryLatency() throws IOException {
+ return anytimeSearcher.search(query);
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) {
+ return false;
+ }
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return reader;
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
new file mode 100644
index 00000000000..d51a6d3582f
--- /dev/null
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/AnytimeRankingRecallStressBenchmark.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.AnytimeRankingSearcher;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Warmup(iterations = 1)
+@Measurement(iterations = 3)
+@Fork(1)
+public class AnytimeRankingRecallStressBenchmark {
+
+ private Directory directory;
+ private IndexReader reader;
+ private IndexSearcher baselineSearcher;
+ private AnytimeRankingSearcher anytimeSearcher;
+ private TermQuery query;
+
+ @Param({"20000"})
+ private int docCount;
+
+ @Param({"1", "3"})
+ private int sla;
+
+ @Setup
+ public void setup() throws Exception {
+ Path path = Files.createTempDirectory("recall-stress");
+ directory = new MMapDirectory(path);
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(100);
+
+ FieldType ft = new FieldType();
+ ft.setTokenized(true);
+ ft.setStored(false);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(directory, config)) {
+ for (int i = 0; i < docCount; i++) {
+ Document doc = new Document();
+ String content = (i % 200 == 0) ? "lucene precision recall document" : "random junk text";
+ doc.add(new Field("field", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ reader = DirectoryReader.open(directory);
+ baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+ anytimeSearcher = new AnytimeRankingSearcher(baselineSearcher, 10, sla, "field");
+ query = new TermQuery(new Term("field", "lucene"));
+ }
+
+ @Benchmark
+ public float baselineRecall() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return countRelevant(topDocs) / 10.0f;
+ }
+
+ @Benchmark
+ public float anytimeRecall() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return countRelevant(topDocs) / 10.0f;
+ }
+
+ @Benchmark
+ public float baselineMRR() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return computeMRR(topDocs);
+ }
+
+ @Benchmark
+ public float anytimeMRR() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return computeMRR(topDocs);
+ }
+
+ @Benchmark
+ public float baselineNDCG() throws IOException {
+ TopDocs topDocs = baselineSearcher.search(query, 10);
+ return computeNDCG(topDocs);
+ }
+
+ @Benchmark
+ public float anytimeNDCG() throws IOException {
+ TopDocs topDocs = anytimeSearcher.search(query);
+ return computeNDCG(topDocs);
+ }
+
+ private int countRelevant(TopDocs topDocs) {
+ int relevant = 0;
+ for (ScoreDoc doc : topDocs.scoreDocs) {
+ if (doc.doc % 200 == 0) {
+ relevant++;
+ }
+ }
+ return relevant;
+ }
+
+ private float computeMRR(TopDocs topDocs) {
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ if (topDocs.scoreDocs[i].doc % 200 == 0) {
+ return 1.0f / (i + 1);
+ }
+ }
+ return 0f;
+ }
+
+ private float computeNDCG(TopDocs topDocs) {
+ float dcg = 0f;
+ float idcg = 0f;
+ int count = 0;
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ int rank = i + 1;
+ if (topDocs.scoreDocs[i].doc % 200 == 0) {
+ dcg += 1.0 / (Math.log(rank + 1) / Math.log(2));
+ count++;
+ }
+ }
+ for (int j = 1; j <= count; j++) {
+ idcg += 1.0 / (Math.log(j + 1) / Math.log(2));
+ }
+ return idcg > 0 ? dcg / idcg : 0f;
+ }
+
+ @TearDown
+ public void tearDown() throws IOException {
+ reader.close();
+ directory.close();
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
index 9a6cb45de47..e60b08fc062 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/ApproximateDocGraphBuilder.java
@@ -29,6 +29,7 @@ import org.apache.lucene.util.BytesRef;
*/
public final class ApproximateDocGraphBuilder {
+ /* Default max number of edges per doc */
public static final int DEFAULT_MAX_EDGES = 10;
private final String field;
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
new file mode 100644
index 00000000000..f2be02f635d
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Locale;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.junit.Test;
+
+public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
+
+ private static final int DOC_COUNT = 20000;
+ private static final int RELEVANT_FREQ = 200;
+
+ @Test
+ public void testRelevanceMetricsComparison() throws Exception {
+ Directory dir = new MMapDirectory(createTempDir("relevance-test"));
+ IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ config.setCodec(new Lucene101Codec());
+ config.setUseCompoundFile(false);
+ config.setMaxBufferedDocs(100);
+
+ FieldType ft = new FieldType();
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.setTokenized(true);
+ ft.setStored(false);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (IndexWriter writer = new IndexWriter(dir, config)) {
+ for (int i = 0; i < DOC_COUNT; i++) {
+ Document doc = new Document();
+ String content = (i % RELEVANT_FREQ == 0) ? "lucene precision recall document" : "random filler content";
+ doc.add(new Field("field", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher baselineSearcher = new IndexSearcher(reader);
+ baselineSearcher.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher anytime = new AnytimeRankingSearcher(baselineSearcher, 10, 3, "field");
+
+ TermQuery query = new TermQuery(new Term("field", "lucene"));
+ TopDocs baseline = baselineSearcher.search(query, 10);
+ TopDocs anytimeResults = anytime.search(query);
+
+ float baselineRecall = countRelevant(baseline) / 10.0f;
+ float anytimeRecall = countRelevant(anytimeResults) / 10.0f;
+
+ float baselineMRR = computeMRR(baseline);
+ float anytimeMRR = computeMRR(anytimeResults);
+
+ float baselineNDCG = computeNDCG(baseline);
+ float anytimeNDCG = computeNDCG(anytimeResults);
+
+ System.out.printf(Locale.ROOT,
+ "Recall: baseline=%.2f anytime=%.2f | MRR: baseline=%.2f anytime=%.2f | NDCG: baseline=%.2f anytime=%.2f%n",
+ baselineRecall, anytimeRecall, baselineMRR, anytimeMRR, baselineNDCG, anytimeNDCG);
+
+ assertTrue("Anytime recall should be >= baseline", anytimeRecall >= baselineRecall);
+ assertTrue("Anytime MRR should be >= baseline", anytimeMRR >= baselineMRR);
+ assertTrue("Anytime NDCG should be >= baseline", anytimeNDCG >= baselineNDCG);
+
+ reader.close();
+ dir.close();
+ }
+
+ private int countRelevant(TopDocs topDocs) {
+ int relevant = 0;
+ for (ScoreDoc doc : topDocs.scoreDocs) {
+ if (doc.doc % RELEVANT_FREQ == 0) {
+ relevant++;
+ }
+ }
+ return relevant;
+ }
+
+ private float computeMRR(TopDocs topDocs) {
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ if (topDocs.scoreDocs[i].doc % RELEVANT_FREQ == 0) {
+ return 1.0f / (i + 1);
+ }
+ }
+ return 0f;
+ }
+
+ private float computeNDCG(TopDocs topDocs) {
+ float dcg = 0f;
+ float idcg = 0f;
+ int count = 0;
+ for (int i = 0; i < topDocs.scoreDocs.length; i++) {
+ int rank = i + 1;
+ if (topDocs.scoreDocs[i].doc % RELEVANT_FREQ == 0) {
+ dcg += (float) (1.0 / (Math.log(rank + 1) / Math.log(2)));
+ count++;
+ }
+ }
+ for (int j = 1; j <= count; j++) {
+ idcg += (float) (1.0 / (Math.log(j + 1) / Math.log(2)));
+ }
+ return idcg > 0 ? dcg / idcg : 0f;
+ }
+
+ private static final class SingleTokenAnalyzer extends Analyzer {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer =
+ new Tokenizer() {
+ private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+ private boolean emitted = false;
+
+ @Override
+ public boolean incrementToken() {
+ if (emitted) return false;
+ clearAttributes();
+ termAttr.append("lucene");
+ emitted = true;
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ emitted = false;
+ }
+ };
+ return new TokenStreamComponents(tokenizer);
+ }
+ }
+}
\ No newline at end of file
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
new file mode 100644
index 00000000000..5fe2bf188c3
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.tests.util.TestUtil;
+
+public class TestArcaneRelevanceBoost extends LuceneTestCase {
+
+ public void testArcaneImprovesRecallUnderTruncation() throws Exception {
+ final int totalDocs = 5000;
+ final int relevantEvery = 200;
+ final int truncationLimit = 20;
+
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig();
+ iwc.setCodec(TestUtil.getDefaultCodec());
+ iwc.setMaxBufferedDocs(100);
+ iwc.setUseCompoundFile(false);
+
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ ft.setTokenized(true);
+ ft.putAttribute("postingsFormat", "Lucene101");
+ ft.putAttribute("doBinning", "true");
+ ft.putAttribute("bin.count", "4");
+ ft.freeze();
+
+ try (RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc)) {
+ for (int i = 0; i < totalDocs; i++) {
+ Document doc = new Document();
+ String content = (i % relevantEvery == 0) ? "lucene highly relevant text" : "noise filler irrelevant";
+ doc.add(new StoredField("docID", i));
+ doc.add(new Field("content", content, ft));
+ writer.addDocument(doc);
+ }
+ writer.commit();
+ }
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher baseline = newSearcher(reader);
+ baseline.setSimilarity(new BM25Similarity());
+
+ AnytimeRankingSearcher arcane =
+ new AnytimeRankingSearcher(baseline, 10, truncationLimit, "content");
+
+ TermQuery query = new TermQuery(new Term("content", "lucene"));
+
+ AtomicInteger baselineHits = new AtomicInteger();
+ baseline.search(query, new CollectorManager<SimpleCollector, Void>() {
+ @Override
+ public SimpleCollector newCollector() {
+ return new SimpleCollector() {
+ private int seen = 0;
+
+ @Override
+ public void collect(int doc) throws IOException {
+ if (doc % relevantEvery == 0) {
+ baselineHits.incrementAndGet();
+ }
+ if (++seen >= truncationLimit) {
+ throw new CollectionTerminatedException();
+ }
+ }
+
+ @Override
+ public void doSetNextReader(LeafReaderContext context) {}
+
+ @Override
+ public ScoreMode scoreMode() {
+ return ScoreMode.COMPLETE_NO_SCORES;
+ }
+ };
+ }
+
+ @Override
+ public Void reduce(Collection<SimpleCollector> collectors) {
+ return null;
+ }
+ });
+
+ TopDocs arcaneResults = arcane.search(query);
+ int arcaneHits = 0;
+ for (ScoreDoc sd : arcaneResults.scoreDocs) {
+ if (sd.doc % relevantEvery == 0) {
+ arcaneHits++;
+ }
+ }
+
+ assertTrue(
+ "ARCANE should retrieve more relevant results under truncation. "
+ + "Baseline hits=" + baselineHits.get() + ", ARCANE hits=" + arcaneHits,
+ arcaneHits > baselineHits.get());
+
+ reader.close();
+ dir.close();
+ }
+}
\ No newline at end of file
(lucene) 18/18: intermittent commit
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 915c76816c0f31f9755816c59575870467919e5d
Author: Atri Sharma <at...@192.168.1.25>
AuthorDate: Thu Apr 17 23:56:37 2025 +0530
intermittent commit
---
.../org/apache/lucene/codecs/BinMapWriter.java | 148 ++++++++++-----------
.../codecs/lucene101/Lucene101PostingsFormat.java | 10 +-
2 files changed, 80 insertions(+), 78 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
index 35ba8690380..1eb3810d03a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
@@ -17,7 +17,6 @@
package org.apache.lucene.codecs;
import java.io.IOException;
-
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.Directory;
@@ -32,84 +31,85 @@ import org.apache.lucene.util.IOUtils;
*/
public final class BinMapWriter implements AutoCloseable {
- /**
- * File extension for binmap files.
- */
- public static final String EXTENSION = "binmap";
+ /** File extension for binmap files. */
+ public static final String EXTENSION = "binmap";
- /**
- * Initial file format version.
- */
- public static final int VERSION_START = 0;
+ /** Initial file format version. */
+ public static final int VERSION_START = 0;
- private static final String CODEC_NAME = "BinMap";
+ private static final String CODEC_NAME = "BinMap";
- private final Directory directory;
- private final SegmentWriteState state;
- private final int maxDoc;
- private final int binCount;
- private final int[] bins;
+ private final Directory directory;
+ private final SegmentWriteState state;
+ private final int maxDoc;
+ private final int binCount;
+ private final int[] bins;
- /**
- * Creates a new {@link BinMapWriter} instance.
- *
- * @param directory output directory
- * @param state current segment write state
- * @param docToBin mapping from docID to bin ID
- * @param binCount total number of bins
- */
- public BinMapWriter(Directory directory, SegmentWriteState state, int[] docToBin, int binCount) {
- this.directory = directory;
- this.state = state;
- this.maxDoc = state.segmentInfo.maxDoc();
- this.binCount = binCount;
- this.bins = docToBin;
- }
+ private boolean closed = false;
+
+ /**
+ * Creates a new {@link BinMapWriter} instance.
+ *
+ * @param directory output directory
+ * @param state current segment write state
+ * @param docToBin mapping from docID to bin ID
+ * @param binCount total number of bins
+ */
+ public BinMapWriter(Directory directory, SegmentWriteState state, int[] docToBin, int binCount) {
+ this.directory = directory;
+ this.state = state;
+ this.maxDoc = state.segmentInfo.maxDoc();
+ this.binCount = binCount;
+ this.bins = docToBin;
+ }
- /**
- * Writes the binmap file to the index output directory.
- *
- * <p>The format is:
- *
- * <ul>
- * <li>Header (versioned with {@link org.apache.lucene.codecs.CodecUtil})
- * <li>maxDoc (int)
- * <li>binCount (int)
- * <li>{@code maxDoc} bin IDs, one int per document
- * <li>Footer
- * </ul>
- */
- @Override
- public void close() throws IOException {
- final String fileName =
- IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
+ /**
+ * Writes the binmap file to the index output directory.
+ *
+ * <p>The format is:
+ *
+ * <ul>
+ * <li>Header (versioned with {@link org.apache.lucene.codecs.CodecUtil})
+ * <li>maxDoc (int)
+ * <li>binCount (int)
+ * <li>{@code maxDoc} bin IDs, one int per document
+ * <li>Footer
+ * </ul>
+ */
+ @Override
+ public void close() throws IOException {
+ final String fileName =
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- // Prevent double-write in case close() is called multiple times
- for (String existing : directory.listAll()) {
- if (existing.equals(fileName)) {
- return; // already written
- }
- }
- IndexOutput out = null;
- boolean success = false;
- try {
- out = directory.createOutput(fileName, state.context);
- CodecUtil.writeIndexHeader(
- out, CODEC_NAME, VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
- out.writeInt(maxDoc);
- out.writeInt(binCount);
- for (int i = 0; i < maxDoc; i++) {
- out.writeInt(bins[i]);
- }
- CodecUtil.writeFooter(out);
- success = true;
- } finally {
- if (success) {
- IOUtils.close(out);
- } else {
- IOUtils.closeWhileHandlingException(out);
- IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
- }
- }
+ if (closed) return;
+ closed = true;
+
+ // Prevent double-write in case close() is called multiple times
+ for (String existing : directory.listAll()) {
+ if (existing.equals(fileName)) {
+ return; // already written
+ }
+ }
+ IndexOutput out = null;
+ boolean success = false;
+ try {
+ out = directory.createOutput(fileName, state.context);
+ CodecUtil.writeIndexHeader(
+ out, CODEC_NAME, VERSION_START, state.segmentInfo.getId(), state.segmentSuffix);
+ out.writeInt(maxDoc);
+ out.writeInt(binCount);
+ for (int i = 0; i < maxDoc; i++) {
+ out.writeInt(bins[i]);
+ }
+ CodecUtil.writeFooter(out);
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(out);
+ } else {
+ IOUtils.closeWhileHandlingException(out);
+ IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
+ }
}
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
index 55d39307ea3..2031a93e522 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
@@ -502,8 +502,10 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
}
}
- BinMapWriter writer = new BinMapWriter(state.directory, state, docToBin, binCount);
- writer.close();
+ try (BinMapWriter writer = new BinMapWriter(state.directory, state, docToBin, binCount)) {
+ // Trigger side-effect and keep compiler happy
+ assert writer != null;
+ }
}
}
}
@@ -530,7 +532,7 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
prior = e;
}
- try {
+ /*try {
if (prior == null) {
// Now that all outputs are flushed, perform binning safely
maybeWriteDocBinning(state);
@@ -542,7 +544,7 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
} else {
throw e;
}
- }
+ }*/
if (prior != null) {
throw prior;
(lucene) 13/18: More cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 727bd7678e8c617fb037d358896eba20be3a8c5b
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 15:32:43 2025 +0530
More cleanup
---
lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java b/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
index 2094a817ea8..079bc5ab002 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestBinScoreUtil.java
@@ -27,8 +27,6 @@ import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
-import org.junit.After;
-import org.junit.Before;
public class TestBinScoreUtil extends LuceneTestCase {
(lucene) 15/18: More cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit ab2b96d075651497235b4bed48e262b9b0fcd278
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 16:05:37 2025 +0530
More cleanup
---
.../src/java/org/apache/lucene/codecs/BinMapReader.java | 13 ++++++-------
.../src/java/org/apache/lucene/codecs/BinMapWriter.java | 1 +
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
index 60fa9ad2873..26e914ba8cd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapReader.java
@@ -16,6 +16,7 @@ import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
/**
* Reads the binmap file for a segment and exposes per-document bin assignments.
@@ -45,9 +46,10 @@ public final class BinMapReader implements Closeable {
final String fileName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
- ChecksumIndexInput in = dir.openChecksumInput(fileName);
+ ChecksumIndexInput in = null;
boolean success = false;
try {
+ in = dir.openChecksumInput(fileName);
CodecUtil.checkIndexHeader(
in,
CODEC_NAME,
@@ -72,13 +74,10 @@ public final class BinMapReader implements Closeable {
CodecUtil.checkFooter(in);
success = true;
} finally {
- if (success == false) {
- try {
- in.close();
- } catch (Throwable _) {
- }
+ if (success) {
+ IOUtils.close(in);
} else {
- in.close();
+ IOUtils.closeWhileHandlingException(in);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
index b3f68293055..fea1907feac 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/BinMapWriter.java
@@ -96,6 +96,7 @@ public final class BinMapWriter implements AutoCloseable {
IOUtils.close(out);
} else {
IOUtils.closeWhileHandlingException(out);
+ IOUtils.deleteFilesIgnoringExceptions(directory, fileName);
}
}
}
(lucene) 07/18: more cleanup
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit d5da46ab01d338aa9eaa40679200e34986385e05
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:22:37 2025 +0530
more cleanup
---
.../org/apache/lucene/codecs/DocGraphBuilder.java | 5 +-
.../lucene/codecs/InMemorySparseEdgeGraph.java | 4 +-
.../codecs/lucene101/Lucene101PostingsWriter.java | 2 -
.../apache/lucene/search/NearDuplicateFinder.java | 129 ---------------------
4 files changed, 4 insertions(+), 136 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
index aea98964069..aa499a7d549 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/DocGraphBuilder.java
@@ -54,7 +54,6 @@ public final class DocGraphBuilder {
/** Constructs the sparse similarity graph using term overlap. */
public SparseEdgeGraph build(LeafReader reader) throws IOException {
- final int maxDoc = reader.maxDoc();
final SparseEdgeGraph graph = new InMemorySparseEdgeGraph();
final Map<String, ArrayList<Integer>> termToDocList = new HashMap<>();
@@ -75,8 +74,8 @@ public final class DocGraphBuilder {
int docID = postings.docID();
int freq = postings.freq();
- docTermVectors.computeIfAbsent(docID, k -> new IntVector()).add(termStr, freq);
- termToDocList.computeIfAbsent(termStr, k -> new ArrayList<>()).add(docID);
+ docTermVectors.computeIfAbsent(docID, _ -> new IntVector()).add(termStr, freq);
+ termToDocList.computeIfAbsent(termStr, _ -> new ArrayList<>()).add(docID);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/InMemorySparseEdgeGraph.java b/lucene/core/src/java/org/apache/lucene/codecs/InMemorySparseEdgeGraph.java
index 30b8202e83a..f0a26a67682 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/InMemorySparseEdgeGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/InMemorySparseEdgeGraph.java
@@ -67,8 +67,8 @@ public final class InMemorySparseEdgeGraph implements SparseEdgeGraph {
@Override
public void ensureVertex(int docID) {
- neighbors.computeIfAbsent(docID, k -> new int[0]);
- weights.computeIfAbsent(docID, k -> new float[0]);
+ neighbors.computeIfAbsent(docID, _ -> new int[0]);
+ weights.computeIfAbsent(docID, _ -> new float[0]);
}
/** Returns the number of vertices in the graph with at least one edge. */
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
index 52ca8203c1f..851e7183994 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
@@ -122,7 +122,6 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
private int maxImpactNumBytesAtLevel0;
private int maxNumImpactsAtLevel1;
private int maxImpactNumBytesAtLevel1;
- private SegmentWriteState segmentWriteState;
/** Sole public constructor. */
public Lucene101PostingsWriter(SegmentWriteState state) throws IOException {
@@ -203,7 +202,6 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
docDeltaBuffer = new int[BLOCK_SIZE];
freqBuffer = new int[BLOCK_SIZE];
- segmentWriteState = state;
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java b/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
deleted file mode 100644
index 179b6e8a9be..00000000000
--- a/lucene/core/src/java/org/apache/lucene/search/NearDuplicateFinder.java
+++ /dev/null
@@ -1,129 +0,0 @@
-package org.apache.lucene.search;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * Identifies clusters of near-duplicate documents based on term overlap using Jaccard similarity.
- */
-public final class NearDuplicateFinder {
-
- private final String field;
- private final float threshold;
-
- /**
- * Create a new finder.
- *
- * @param field the field to analyze
- * @param threshold similarity threshold between 0 and 1
- */
- public NearDuplicateFinder(String field, float threshold) {
- this.field = field;
- this.threshold = threshold;
- }
-
- /**
- * Computes near-duplicate clusters across all segments using global docIDs.
- *
- * @param reader top-level reader
- * @return list of duplicate clusters
- */
- public List<DuplicateCluster> findNearDuplicates(DirectoryReader reader) throws IOException {
- final int maxDoc = reader.maxDoc();
- final Map<Integer, Set<String>> docToTerms = new HashMap<>();
- final Map<String, Set<Integer>> termToDocs = new HashMap<>();
-
- int docBase = 0;
- for (LeafReaderContext ctx : reader.leaves()) {
- Terms terms = ctx.reader().terms(field);
- if (terms == null) {
- docBase += ctx.reader().maxDoc();
- continue;
- }
-
- TermsEnum termsEnum = terms.iterator();
- BytesRef term;
- while ((term = termsEnum.next()) != null) {
- PostingsEnum postings = termsEnum.postings(null, PostingsEnum.NONE);
- if (postings == null) {
- continue;
- }
-
- String termStr = term.utf8ToString();
- while (postings.nextDoc() != PostingsEnum.NO_MORE_DOCS) {
- int globalDocID = docBase + postings.docID();
- docToTerms.computeIfAbsent(globalDocID, k -> new HashSet<>()).add(termStr);
- termToDocs.computeIfAbsent(termStr, k -> new HashSet<>()).add(globalDocID);
- }
- }
-
- docBase += ctx.reader().maxDoc();
- }
-
- final BitSet visited = new BitSet(maxDoc);
- final List<DuplicateCluster> clusters = new ArrayList<>();
-
- for (Map.Entry<Integer, Set<String>> entry : docToTerms.entrySet()) {
- final int docID = entry.getKey();
- if (visited.get(docID)) {
- continue;
- }
-
- final Set<Integer> cluster = new HashSet<>();
- final Set<String> termsA = entry.getValue();
- cluster.add(docID);
- visited.set(docID);
-
- // Include all documents with no terms as potential duplicates of this one
- for (Map.Entry<Integer, Set<String>> otherEntry : docToTerms.entrySet()) {
- final int candidate = otherEntry.getKey();
- if (candidate == docID || visited.get(candidate)) {
- continue;
- }
-
- final Set<String> termsB = otherEntry.getValue();
-
- if (termsA.isEmpty() && termsB.isEmpty()) {
- cluster.add(candidate);
- visited.set(candidate);
- continue;
- }
-
- float score = jaccard(termsA, termsB);
- if (score >= threshold) {
- cluster.add(candidate);
- visited.set(candidate);
- }
- }
-
- if (cluster.size() > 1) {
- clusters.add(new DuplicateCluster(cluster));
- }
- }
-
- return clusters;
- }
-
- private static float jaccard(Set<String> a, Set<String> b) {
- int intersection = 0;
- for (String s : a) {
- if (b.contains(s)) {
- intersection++;
- }
- }
- int union = a.size() + b.size() - intersection;
- return union == 0 ? 0.0f : (float) intersection / union;
- }
-}
(lucene) 05/18: More cleanups
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit dd15761120a52725bb455017b26d0cda779173bc
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 02:17:47 2025 +0530
More cleanups
---
.../lucene/search/TestAnytimeRankingNDCG.java | 4 --
.../search/TestAnytimeRankingRelevanceMatch.java | 45 +---------------------
...ceBoost.java => TestBinningRelevanceBoost.java} | 22 +++++------
3 files changed, 13 insertions(+), 58 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
index 34318429dcb..d3ade51bdba 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
@@ -18,7 +18,6 @@
package org.apache.lucene.search;
import java.util.Arrays;
-import java.util.Locale;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -80,9 +79,6 @@ public class TestAnytimeRankingNDCG extends LuceneTestCase {
double baselineNDCG = computeNDCG(baselineDocs);
double anytimeNDCG = computeNDCG(anytimeDocs);
- System.out.printf(
- Locale.ROOT, "baselineNDCG=%.3f anytimeNDCG=%.3f%n", baselineNDCG, anytimeNDCG);
-
// Ensure that NDCG does not diverge by more than 0.1
assertEquals("NDCG values should be close", baselineNDCG, anytimeNDCG, 0.1);
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
index 007b8674830..80d17887f3d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingRelevanceMatch.java
@@ -6,11 +6,6 @@
*/
package org.apache.lucene.search;
-import java.io.IOException;
-import java.util.Locale;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.codecs.lucene101.Lucene101Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -24,6 +19,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.junit.Test;
@@ -35,7 +31,7 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
@Test
public void testRelevanceMetricsComparison() throws Exception {
Directory dir = new MMapDirectory(createTempDir("relevance-test"));
- IndexWriterConfig config = new IndexWriterConfig(new SingleTokenAnalyzer());
+ IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
config.setCodec(new Lucene101Codec());
config.setUseCompoundFile(false);
config.setMaxBufferedDocs(100);
@@ -79,16 +75,6 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
float baselineNDCG = computeNDCG(baseline);
float anytimeNDCG = computeNDCG(anytimeResults);
- System.out.printf(
- Locale.ROOT,
- "Recall: baseline=%.2f anytime=%.2f | MRR: baseline=%.2f anytime=%.2f | NDCG: baseline=%.2f anytime=%.2f%n",
- baselineRecall,
- anytimeRecall,
- baselineMRR,
- anytimeMRR,
- baselineNDCG,
- anytimeNDCG);
-
assertTrue("Anytime recall should be >= baseline", anytimeRecall >= baselineRecall);
assertTrue("Anytime MRR should be >= baseline", anytimeMRR >= baselineMRR);
assertTrue("Anytime NDCG should be >= baseline", anytimeNDCG >= baselineNDCG);
@@ -132,31 +118,4 @@ public class TestAnytimeRankingRelevanceMatch extends LuceneTestCase {
}
return idcg > 0 ? dcg / idcg : 0f;
}
-
- private static final class SingleTokenAnalyzer extends Analyzer {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer =
- new Tokenizer() {
- private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
- private boolean emitted = false;
-
- @Override
- public boolean incrementToken() {
- if (emitted) return false;
- clearAttributes();
- termAttr.append("lucene");
- emitted = true;
- return true;
- }
-
- @Override
- public void reset() throws IOException {
- super.reset();
- emitted = false;
- }
- };
- return new TokenStreamComponents(tokenizer);
- }
- }
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java b/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
similarity index 87%
rename from lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
rename to lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
index 932d960fd1d..52f3b07e8f3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestArcaneRelevanceBoost.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBinningRelevanceBoost.java
@@ -36,9 +36,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
-public class TestArcaneRelevanceBoost extends LuceneTestCase {
+public class TestBinningRelevanceBoost extends LuceneTestCase {
- public void testArcaneImprovesRecallUnderTruncation() throws Exception {
+ public void testBinningImprovesRecallUnderTruncation() throws Exception {
final int totalDocs = 5000;
final int relevantEvery = 200;
final int truncationLimit = 20;
@@ -73,7 +73,7 @@ public class TestArcaneRelevanceBoost extends LuceneTestCase {
IndexSearcher baseline = newSearcher(reader);
baseline.setSimilarity(new BM25Similarity());
- AnytimeRankingSearcher arcane =
+ AnytimeRankingSearcher anytimeRankingSearcher =
new AnytimeRankingSearcher(baseline, 10, truncationLimit, "content");
TermQuery query = new TermQuery(new Term("content", "lucene"));
@@ -113,21 +113,21 @@ public class TestArcaneRelevanceBoost extends LuceneTestCase {
}
});
- TopDocs arcaneResults = arcane.search(query);
- int arcaneHits = 0;
- for (ScoreDoc sd : arcaneResults.scoreDocs) {
+ TopDocs anytimeRankingResults = anytimeRankingSearcher.search(query);
+ int anytimeRankingHits = 0;
+ for (ScoreDoc sd : anytimeRankingResults.scoreDocs) {
if (sd.doc % relevantEvery == 0) {
- arcaneHits++;
+ anytimeRankingHits++;
}
}
assertTrue(
- "ARCANE should retrieve more relevant results under truncation. "
+ "AnytimeRankingSearch should retrieve more relevant results under truncation. "
+ "Baseline hits="
+ baselineHits.get()
- + ", ARCANE hits="
- + arcaneHits,
- arcaneHits > baselineHits.get());
+ + ", AnytimeRankingSearch hits="
+ + anytimeRankingHits,
+ anytimeRankingHits > baselineHits.get());
reader.close();
dir.close();
(lucene) 17/18: Do not close if prior write failed
Posted by at...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch new_index_query_include
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 921ff07033142ce04c631e1b5f0259f871ecd532
Author: Atri Sharma <at...@192.168.1.75>
AuthorDate: Thu Apr 17 19:16:15 2025 +0530
Do not close if prior write failed
---
.../org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
index 9f9f627e4f0..55d39307ea3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
@@ -531,8 +531,10 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
}
try {
- // Now that all outputs are flushed, perform binning safely
- maybeWriteDocBinning(state);
+ if (prior == null) {
+ // Now that all outputs are flushed, perform binning safely
+ maybeWriteDocBinning(state);
+ }
} catch (IOException e) {
if (prior != null) {
prior.addSuppressed(e);