You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2025/01/09 21:33:08 UTC
(lucene) branch branch_10x updated: gh-14123: Add null checks to SortingCodecReader (#14128)
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new bfc8859f442 gh-14123: Add null checks to SortingCodecReader (#14128)
bfc8859f442 is described below
commit bfc8859f4420dabb9dda11696fd337d02611d06b
Author: Michael Sokolov <so...@falutin.net>
AuthorDate: Thu Jan 9 16:33:03 2025 -0500
gh-14123: Add null checks to SortingCodecReader (#14128)
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/index/SortingCodecReader.java | 22 ++++++++++
.../lucene/index/TestSortingCodecReader.java | 47 +++++++++++++++++-----
3 files changed, 62 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index d8d3e51ed1e..15679c0d940 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -34,6 +34,8 @@ Bug Fixes
* GITHUB#14109: prefetch may select the wrong memory segment for
multi-segment slices. (Chris Hegarty)
+* GITHUB#14123: SortingCodecReader NPE when segment has no (points, vectors, etc...) (Mike Sokolov)
+
Other
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
index daec0c197d6..ab9964026ad 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
@@ -314,6 +314,7 @@ public final class SortingCodecReader extends FilterCodecReader {
SortingFloatVectorValues(FloatVectorValues delegate, Sorter.DocMap sortMap) throws IOException {
this.delegate = delegate;
+ assert delegate != null;
// SortingValuesIterator consumes the iterator and records the docs and ord mapping
iteratorSupplier = iteratorSupplier(delegate, sortMap);
}
@@ -446,6 +447,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public FieldsProducer getPostingsReader() {
FieldsProducer postingsReader = in.getPostingsReader();
+ if (postingsReader == null) {
+ return null;
+ }
return new FieldsProducer() {
@Override
public void close() throws IOException {
@@ -481,6 +485,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public StoredFieldsReader getFieldsReader() {
StoredFieldsReader delegate = in.getFieldsReader();
+ if (delegate == null) {
+ return null;
+ }
return newStoredFieldsReader(delegate);
}
@@ -526,6 +533,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public PointsReader getPointsReader() {
final PointsReader delegate = in.getPointsReader();
+ if (delegate == null) {
+ return null;
+ }
return new PointsReader() {
@Override
public void checkIntegrity() throws IOException {
@@ -551,6 +561,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public KnnVectorsReader getVectorReader() {
KnnVectorsReader delegate = in.getVectorReader();
+ if (delegate == null) {
+ return null;
+ }
return new KnnVectorsReader() {
@Override
public void checkIntegrity() throws IOException {
@@ -587,6 +600,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public NormsProducer getNormsReader() {
final NormsProducer delegate = in.getNormsReader();
+ if (delegate == null) {
+ return null;
+ }
return new NormsProducer() {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
@@ -609,6 +625,9 @@ public final class SortingCodecReader extends FilterCodecReader {
@Override
public DocValuesProducer getDocValuesReader() {
final DocValuesProducer delegate = in.getDocValuesReader();
+ if (delegate == null) {
+ return null;
+ }
return new DocValuesProducer() {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
@@ -710,6 +729,9 @@ public final class SortingCodecReader extends FilterCodecReader {
}
private TermVectorsReader newTermVectorsReader(TermVectorsReader delegate) {
+ if (delegate == null) {
+ return null;
+ }
return new TermVectorsReader() {
@Override
public void prefetch(int doc) throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
index 9663d676255..285296d55c1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
@@ -25,7 +25,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
+import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.codecs.hnsw.HnswGraphProvider;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -53,6 +55,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.hnsw.HnswGraph;
public class TestSortingCodecReader extends LuceneTestCase {
@@ -151,12 +154,16 @@ public class TestSortingCodecReader extends LuceneTestCase {
docIds.add(i);
}
Collections.shuffle(docIds, random());
+ // If true, index a vector and points for every doc
+ boolean dense = random().nextBoolean();
try (RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) {
for (int i = 0; i < numDocs; i++) {
int docId = docIds.get(i);
Document doc = new Document();
doc.add(new StringField("string_id", Integer.toString(docId), Field.Store.YES));
- doc.add(new LongPoint("point_id", docId));
+ if (dense || docId % 3 == 0) {
+ doc.add(new LongPoint("point_id", docId));
+ }
String s = RandomStrings.randomRealisticUnicodeOfLength(random(), 25);
doc.add(new TextField("text_field", s, Field.Store.YES));
doc.add(new BinaryDocValuesField("text_field", new BytesRef(s)));
@@ -168,7 +175,9 @@ public class TestSortingCodecReader extends LuceneTestCase {
doc.add(new BinaryDocValuesField("binary_dv", new BytesRef(Integer.toString(docId))));
doc.add(
new SortedSetDocValuesField("sorted_set_dv", new BytesRef(Integer.toString(docId))));
- doc.add(new KnnFloatVectorField("vector", new float[] {(float) docId}));
+ if (dense || docId % 2 == 0) {
+ doc.add(new KnnFloatVectorField("vector", new float[] {(float) docId}));
+ }
doc.add(new NumericDocValuesField("foo", random().nextInt(20)));
FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
@@ -239,6 +248,13 @@ public class TestSortingCodecReader extends LuceneTestCase {
SortedSetDocValues sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");
SortedDocValues binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");
FloatVectorValues vectorValues = leaf.getFloatVectorValues("vector");
+ KnnVectorsReader vectorsReader = ((CodecReader) leaf).getVectorReader();
+ HnswGraph graph;
+ if (vectorsReader instanceof HnswGraphProvider hnswGraphProvider) {
+ graph = hnswGraphProvider.getGraph("vector");
+ } else {
+ graph = null;
+ }
NumericDocValues ids = leaf.getNumericDocValues("id");
long prevValue = -1;
boolean usingAltIds = false;
@@ -264,7 +280,14 @@ public class TestSortingCodecReader extends LuceneTestCase {
assertTrue(sorted_numeric_dv.advanceExact(idNext));
assertTrue(sorted_set_dv.advanceExact(idNext));
assertTrue(binary_sorted_dv.advanceExact(idNext));
- assertEquals(idNext, valuesIterator.advance(idNext));
+ if (dense || prevValue % 2 == 0) {
+ assertEquals(idNext, valuesIterator.advance(idNext));
+ if (graph != null) {
+ graph.seek(0, valuesIterator.index());
+ assertNotEquals(DocIdSetIterator.NO_MORE_DOCS, graph.nextNeighbor());
+ }
+ }
+
assertEquals(new BytesRef(ids.longValue() + ""), binary_dv.binaryValue());
assertEquals(
new BytesRef(ids.longValue() + ""),
@@ -276,9 +299,11 @@ public class TestSortingCodecReader extends LuceneTestCase {
assertEquals(1, sorted_numeric_dv.docValueCount());
assertEquals(ids.longValue(), sorted_numeric_dv.nextValue());
- float[] vectorValue = vectorValues.vectorValue(valuesIterator.index());
- assertEquals(1, vectorValue.length);
- assertEquals((float) ids.longValue(), vectorValue[0], 0.001f);
+ if (dense || prevValue % 2 == 0) {
+ float[] vectorValue = vectorValues.vectorValue(valuesIterator.index());
+ assertEquals(1, vectorValue.length);
+ assertEquals((float) ids.longValue(), vectorValue[0], 0.001f);
+ }
Fields termVectors = leaf.termVectors().get(idNext);
assertTrue(
@@ -291,9 +316,13 @@ public class TestSortingCodecReader extends LuceneTestCase {
leaf.storedFields().document(idNext).get("string_id"));
IndexSearcher searcher = new IndexSearcher(r);
TopDocs result =
- searcher.search(LongPoint.newExactQuery("point_id", ids.longValue()), 1);
- assertEquals(1, result.totalHits.value());
- assertEquals(idNext, result.scoreDocs[0].doc);
+ searcher.search(LongPoint.newExactQuery("point_id", ids.longValue()), 10);
+ if (dense || ids.longValue() % 3 == 0) {
+ assertEquals(1, result.totalHits.value());
+ assertEquals(idNext, result.scoreDocs[0].doc);
+ } else {
+ assertEquals(0, result.totalHits.value());
+ }
result =
searcher.search(new TermQuery(new Term("string_id", "" + ids.longValue())), 1);