You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by at...@apache.org on 2025/04/18 10:09:23 UTC
(lucene) 09/09: Closing resources
This is an automated email from the ASF dual-hosted git repository.
atri pushed a commit to branch index_binning_changes
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 20a9240d92ecb5ee5f659b3b47f5a4936ddc650e
Author: Atri Sharma <at...@Atris-MacBook-Pro-3.local>
AuthorDate: Fri Apr 18 15:37:59 2025 +0530
Closing resources
---
.../apache/lucene/index/BinScoreLeafReader.java | 19 +++-
.../java/org/apache/lucene/index/BinScoreUtil.java | 111 +++++++++++++++++---
.../lucene/search/AnytimeRankingSearcher.java | 1 +
.../lucene/search/TestAnytimeRankingNDCG.java | 116 ---------------------
.../lucene/search/TestAnytimeRankingSearch.java | 63 -----------
5 files changed, 109 insertions(+), 201 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
index fb28686c6ee..83a76948c35 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreLeafReader.java
@@ -1,10 +1,10 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
+ * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
@@ -20,13 +20,22 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.util.IOUtils;
-/** Reader wrapper that exposes bin-aware scoring. */
+/**
+ * Reader wrapper that exposes bin-aware scoring.
+ * Supports resource tracking for clean closure of auxiliary components.
+ */
public class BinScoreLeafReader extends FilterLeafReader {
private final BinScoreReader binScore;
private final Closeable[] resources;
- /** Default constructor */
+ /**
+ * Constructs a bin-aware reader.
+ *
+ * @param in the original reader
+ * @param binScore Score reader
+ * @param resources Closeable resources
+ */
public BinScoreLeafReader(LeafReader in, BinScoreReader binScore, Closeable... resources) {
super(in);
this.binScore = binScore;
@@ -52,4 +61,4 @@ public class BinScoreLeafReader extends FilterLeafReader {
public CacheHelper getCoreCacheHelper() {
return in.getCoreCacheHelper();
}
-}
+}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
index cf1e859524a..dedaa75d65f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/index/BinScoreUtil.java
@@ -14,15 +14,28 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.IOUtils;
-/** Utilities for wrapping readers with binning support. */
+/**
+ * Utility methods for wrapping readers with bin-aware scoring support.
+ * Enables loading bin map metadata and injecting scoring logic at search time.
+ */
public final class BinScoreUtil {
+ /** Tracks compound file readers for cleanup after wrapping. */
private static final Map<IndexReader, List<Closeable>> compoundReaders = new WeakHashMap<>();
+
+ /** Shuffle seed to reduce leaf bias in multi-segment readers. */
private static final long SHUFFLE_SEED = 42L;
private BinScoreUtil() {}
- /** Wrap an IndexReader with bin support. */
+ /**
+ * Wraps an IndexReader with bin-aware scoring if binmap metadata is present.
+ * Supports both LeafReader and CompositeReader types.
+ *
+ * @param reader the input IndexReader
+ * @return wrapped reader if binmap is found; otherwise original reader
+ * @throws IOException on I/O error
+ */
public static IndexReader wrap(IndexReader reader) throws IOException {
if (reader instanceof LeafReader) {
return wrap((LeafReader) reader);
@@ -33,19 +46,47 @@ public final class BinScoreUtil {
return reader;
}
- /** Shuffle leaves before wrapping to reduce early-termination bias. */
+ /**
+ * Wraps a CompositeReader (e.g., DirectoryReader) with bin-aware logic.
+ * Leaf readers are shuffled to mitigate early termination bias, and each is
+ * wrapped individually. All associated compound readers are tracked and must
+ * be closed via {@link #closeResources}.
+ *
+ * @param reader composite reader to wrap
+ * @return MultiReader with bin-aware leaves
+ * @throws IOException on error
+ */
private static CompositeReader wrapComposite(CompositeReader reader) throws IOException {
List<LeafReaderContext> leaves = new ArrayList<>(reader.leaves());
Collections.shuffle(leaves, new Random(SHUFFLE_SEED));
List<LeafReader> wrapped = new ArrayList<>();
+ List<Closeable> toClose = new ArrayList<>();
+
for (LeafReaderContext ctx : leaves) {
- wrapped.add(wrap(ctx.reader()));
+ wrapped.add(wrap(ctx.reader(), toClose));
}
- return new MultiReader(wrapped.toArray(new LeafReader[0]), true);
+
+ MultiReader multiReader = new MultiReader(wrapped.toArray(new LeafReader[0]), true);
+
+ if (!toClose.isEmpty()) {
+ synchronized (compoundReaders) {
+ compoundReaders.put(multiReader, toClose);
+ }
+ }
+
+ return multiReader;
}
- /** Wrap a LeafReader with bin score logic if binmap file is found. */
- public static LeafReader wrap(LeafReader reader) throws IOException {
+ /**
+ * Wraps a single LeafReader with bin-aware scoring logic, collecting any
+ * compound readers created for later cleanup.
+ *
+ * @param reader the input LeafReader
+ * @param closables list to track compound readers for external cleanup
+ * @return wrapped reader if binmap is found; otherwise original reader
+ * @throws IOException on error
+ */
+ private static LeafReader wrap(LeafReader reader, List<Closeable> closables) throws IOException {
SegmentReader sr = getSegmentReader(reader);
if (sr == null) return reader;
@@ -76,27 +117,42 @@ public final class BinScoreUtil {
try {
binMap = new BinMapReader(dir, state);
BinScoreReader binScore = new BinScoreReader(binMap);
- final Closeable closeOnClose = compoundReader;
-
- return new BinScoreLeafReader(reader, binScore, binMap) {
- @Override
- protected void doClose() throws IOException {
- IOUtils.close(closeOnClose);
- super.doClose();
- }
- };
+
+ if (compoundReader != null) {
+ closables.add(compoundReader);
+ }
+
+ return new BinScoreLeafReader(reader, binScore, binMap, compoundReader);
} catch (Throwable t) {
IOUtils.closeWhileHandlingException(binMap, compoundReader);
throw t;
}
}
+ /**
+ * Wraps a LeafReader and tracks compound files internally.
+ * This method is intended for direct wrapping of single-segment readers.
+ *
+ * @param reader input reader
+ * @return bin-aware reader or original
+ * @throws IOException on error
+ */
+ public static LeafReader wrap(LeafReader reader) throws IOException {
+ return wrap(reader, new ArrayList<>());
+ }
+
+ /**
+ * Extracts the SegmentReader from the given reader if available.
+ */
private static SegmentReader getSegmentReader(LeafReader reader) {
if (reader instanceof SegmentReader) return (SegmentReader) reader;
LeafReader unwrapped = FilterLeafReader.unwrap(reader);
return (unwrapped instanceof SegmentReader) ? (SegmentReader) unwrapped : null;
}
+ /**
+ * Finds a binmap file in the given directory for the specified segment.
+ */
private static String findBinmapFile(Directory dir, String segmentName) throws IOException {
for (String file : dir.listAll()) {
if (file.startsWith(segmentName + "_") && file.endsWith(".binmap")) {
@@ -106,7 +162,12 @@ public final class BinScoreUtil {
return null;
}
- /** Return bin score reader if available. */
+ /**
+ * Returns the BinScoreReader associated with a bin-aware wrapped reader.
+ *
+ * @param reader the LeafReader
+ * @return BinScoreReader or null
+ */
public static BinScoreReader getBinScoreReader(LeafReader reader) {
if (reader instanceof BinScoreLeafReader) {
return ((BinScoreLeafReader) reader).getBinScoreReader();
@@ -117,4 +178,20 @@ public final class BinScoreUtil {
}
return null;
}
+
+ /**
+ * Closes all resources (e.g., compound readers) associated with the wrapped IndexReader.
+ * This method must be invoked before or during IndexReader#close to ensure full cleanup.
+ *
+ * @param reader the top-level MultiReader returned by {@link #wrap(IndexReader)}
+ * @throws IOException on error
+ */
+ public static void closeResources(IndexReader reader) throws IOException {
+ synchronized (compoundReaders) {
+ List<Closeable> list = compoundReaders.remove(reader);
+ if (list != null) {
+ IOUtils.close(list);
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
index 32bd223b346..8c7f07c2c18 100644
--- a/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
+++ b/lucene/core/src/java/org/apache/lucene/search/AnytimeRankingSearcher.java
@@ -56,6 +56,7 @@ public final class AnytimeRankingSearcher implements AutoCloseable {
@Override
public void close() throws IOException {
+ BinScoreUtil.closeResources(reader);
reader.close();
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
deleted file mode 100644
index 3e0e628189b..00000000000
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingNDCG.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.search;
-
-import java.util.Arrays;
-import org.apache.lucene.codecs.lucene103.Lucene103Codec;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.BM25Similarity;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.TestUtil;
-
-public class TestAnytimeRankingNDCG extends LuceneTestCase {
-
- public void testNDCGDoesNotDiverge() throws Exception {
- MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new ByteBuffersDirectory());
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- iwc.setCodec(new Lucene103Codec());
-
- FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
- ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- ft.setTokenized(true);
- ft.putAttribute("postingsFormat", "Lucene101");
- ft.putAttribute("doBinning", "true");
- ft.putAttribute("bin.count", "4");
- ft.freeze();
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 500; i++) {
- Document doc = new Document();
- StringBuilder content = new StringBuilder();
-
- if (i % 50 == 0) {
- content.append("lucene high quality scoring search");
- } else {
- content.append("random filler ").append(TestUtil.randomSimpleString(random()));
- }
-
- doc.add(new Field("field", content.toString(), ft));
- writer.addDocument(doc);
- }
- writer.commit();
- }
-
- DirectoryReader reader = DirectoryReader.open(dir);
- IndexSearcher baseline = new IndexSearcher(reader);
- baseline.setSimilarity(new BM25Similarity());
-
- try (AnytimeRankingSearcher anytime = new AnytimeRankingSearcher(reader, 10, 5, "field")) {
-
- TopDocs baselineDocs = baseline.search(new TermQuery(new Term("field", "lucene")), 10);
- TopDocs anytimeDocs = anytime.search(new TermQuery(new Term("field", "lucene")));
-
- double baselineNDCG = computeNDCG(baselineDocs);
- double anytimeNDCG = computeNDCG(anytimeDocs);
-
- // Ensure that NDCG does not diverge by more than 0.1
- assertEquals("NDCG values should be close", baselineNDCG, anytimeNDCG, 0.1);
-
- reader.close();
- dir.close();
- }
- }
-
- private double computeNDCG(TopDocs topDocs) {
- int[] gains = new int[topDocs.scoreDocs.length];
- for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- gains[i] = (topDocs.scoreDocs[i].doc % 50 == 0) ? 3 : 0;
- }
-
- double dcg = 0.0;
- for (int i = 0; i < gains.length; i++) {
- int rel = gains[i];
- if (rel > 0) {
- dcg += (Math.pow(2, rel) - 1) / (Math.log(i + 2) / Math.log(2));
- }
- }
-
- Arrays.sort(gains);
- double idcg = 0.0;
- for (int i = gains.length - 1; i >= 0; i--) {
- int rel = gains[i];
- if (rel > 0) {
- int rank = gains.length - i;
- idcg += (Math.pow(2, rel) - 1) / (Math.log(rank + 1) / Math.log(2));
- }
- }
-
- return idcg == 0.0 ? 0.0 : dcg / idcg;
- }
-}
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
index 869347f2d19..2010bf8e1de 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestAnytimeRankingSearch.java
@@ -77,69 +77,6 @@ public class TestAnytimeRankingSearch extends LuceneTestCase {
return doc;
}
- @Test
- public void testSparseBinDistribution() throws Exception {
- try (Directory dir = newDirectory()) {
- IndexWriterConfig iwc = new IndexWriterConfig(new StandardAnalyzer());
- iwc.setMaxBufferedDocs(5);
- iwc.setMergePolicy(NoMergePolicy.INSTANCE);
-
- FieldType ft = newBinningFieldType();
-
- try (IndexWriter writer = new IndexWriter(dir, iwc)) {
- for (int i = 0; i < 64; i++) {
- String content = (i % 8 == 0) ? "lucene boost" : "noise filler content";
- writer.addDocument(newDoc(i, content, ft));
- }
- writer.commit();
- }
-
- IndexReader base = DirectoryReader.open(dir);
- //IndexReader wrapped = BinScoreUtil.wrap(base);
-
- try {
- IndexSearcher searcher = newSearcher(base);
- searcher.setSimilarity(new BM25Similarity());
-
- try (AnytimeRankingSearcher rankingSearcher =
- new AnytimeRankingSearcher(base, 10, 100, "content")) {
-
- TopDocs topDocs = rankingSearcher.search(new TermQuery(new Term("content", "lucene")));
-
- assertNotNull(topDocs);
- assertTrue("Expected some results", topDocs.scoreDocs.length > 0);
-
- int bin0Hits = 0;
- int total = 0;
- IndexReader wrapped = rankingSearcher.getSearcher().getIndexReader();
- for (ScoreDoc sd : topDocs.scoreDocs) {
- int docID = sd.doc;
- for (LeafReaderContext ctx : wrapped.leaves()) {
- if (docID >= ctx.docBase && docID < ctx.docBase + ctx.reader().maxDoc()) {
- int segDoc = docID - ctx.docBase;
- BinScoreReader binReader = BinScoreUtil.getBinScoreReader(ctx.reader());
- if (binReader != null) {
- int bin = binReader.getBinForDoc(segDoc);
- System.out.println("doc" + segDoc + " bin " + bin);
- if (bin == 0) {
- bin0Hits++;
- }
- total++;
- }
- break;
- }
- }
- }
-
- assertTrue(
- "Bin 0 should dominate in sparse distribution, got " + bin0Hits + " of " + total,
- total > 0 && bin0Hits >= total / 2);
- }
- } finally {
- }
- }
- }
-
@Test
public void testSlaCutoffTriggersEarlyTermination() throws Exception {
Codec.setDefault(new Lucene103Codec());