You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2020/10/26 12:01:06 UTC

[lucene-solr] branch master updated: LUCENE-9322: Make sure to account for vectors in SortingCodecReader. (#2028)

This is an automated email from the ASF dual-hosted git repository.

sokolov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git


The following commit(s) were added to refs/heads/master by this push:
     new 37c7d15  LUCENE-9322: Make sure to account for vectors in SortingCodecReader. (#2028)
37c7d15 is described below

commit 37c7d156ab54ce9baae08bebb76eebe4da2e5b81
Author: Julie Tibshirani <ju...@elastic.co>
AuthorDate: Mon Oct 26 05:00:26 2020 -0700

    LUCENE-9322: Make sure to account for vectors in SortingCodecReader. (#2028)
---
 .../apache/lucene/index/SortingCodecReader.java    | 27 ++++++++++++++++++++++
 .../apache/lucene/index/VectorValuesWriter.java    |  2 +-
 .../lucene/index/TestSortingCodecReader.java       | 10 ++++++++
 3 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
index f5831fd..fe2d5e1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SortingCodecReader.java
@@ -29,6 +29,7 @@ import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PointsReader;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.codecs.VectorReader;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.Bits;
@@ -304,6 +305,32 @@ public final class SortingCodecReader extends FilterCodecReader {
   }
 
   @Override
+  public VectorReader getVectorReader() {
+    VectorReader delegate = in.getVectorReader();
+    return new VectorReader() {
+      @Override
+      public void checkIntegrity() throws IOException {
+        delegate.checkIntegrity();
+      }
+
+      @Override
+      public VectorValues getVectorValues(String field) throws IOException {
+        return new VectorValuesWriter.SortingVectorValues(delegate.getVectorValues(field), docMap);
+      }
+
+      @Override
+      public void close() throws IOException {
+        delegate.close();
+      }
+
+      @Override
+      public long ramBytesUsed() {
+        return delegate.ramBytesUsed();
+      }
+    };
+  }
+
+  @Override
   public NormsProducer getNormsReader() {
     final NormsProducer delegate = in.getNormsReader();
     return new NormsProducer() {
diff --git a/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
index c4e9701..ae39b3a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
@@ -98,7 +98,7 @@ class VectorValuesWriter {
     }
   }
 
-  private static class SortingVectorValues extends VectorValues {
+  static class SortingVectorValues extends VectorValues {
 
     private final VectorValues delegate;
     private final VectorValues.RandomAccess randomAccess;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
index fb54048..d2393ea 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSortingCodecReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.VectorField;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Sort;
@@ -126,6 +127,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
           doc.add(new SortedDocValuesField("binary_sorted_dv", new BytesRef(Integer.toString(docId))));
           doc.add(new BinaryDocValuesField("binary_dv", new BytesRef(Integer.toString(docId))));
           doc.add(new SortedSetDocValuesField("sorted_set_dv", new BytesRef(Integer.toString(docId))));
+          doc.add(new VectorField("vector", new float[] { (float) docId }));
           doc.add(new NumericDocValuesField("foo", random().nextInt(20)));
 
           FieldType ft = new FieldType(StringField.TYPE_NOT_STORED);
@@ -180,6 +182,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
             SortedNumericDocValues sorted_numeric_dv = leaf.getSortedNumericDocValues("sorted_numeric_dv");
             SortedSetDocValues sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");
             SortedDocValues binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");
+            VectorValues vectorValues = leaf.getVectorValues("vector");
             NumericDocValues ids = leaf.getNumericDocValues("id");
             long prevValue = -1;
             boolean usingAltIds = false;
@@ -194,6 +197,7 @@ public class TestSortingCodecReader extends LuceneTestCase {
                 sorted_numeric_dv = leaf.getSortedNumericDocValues("sorted_numeric_dv");
                 sorted_set_dv = leaf.getSortedSetDocValues("sorted_set_dv");
                 binary_sorted_dv = leaf.getSortedDocValues("binary_sorted_dv");
+                vectorValues = leaf.getVectorValues("vector");
                 prevValue = -1;
               }
               assertTrue(prevValue + " < " + ids.longValue(), prevValue < ids.longValue());
@@ -202,11 +206,17 @@ public class TestSortingCodecReader extends LuceneTestCase {
               assertTrue(sorted_numeric_dv.advanceExact(idNext));
               assertTrue(sorted_set_dv.advanceExact(idNext));
               assertTrue(binary_sorted_dv.advanceExact(idNext));
+              assertEquals(idNext, vectorValues.advance(idNext));
               assertEquals(new BytesRef(ids.longValue() + ""), binary_dv.binaryValue());
               assertEquals(new BytesRef(ids.longValue() + ""), binary_sorted_dv.binaryValue());
               assertEquals(new BytesRef(ids.longValue() + ""), sorted_set_dv.lookupOrd(sorted_set_dv.nextOrd()));
               assertEquals(1, sorted_numeric_dv.docValueCount());
               assertEquals(ids.longValue(), sorted_numeric_dv.nextValue());
+
+              float[] vectorValue = vectorValues.vectorValue();
+              assertEquals(1, vectorValue.length);
+              assertEquals((float) ids.longValue(), vectorValue[0], 0.001f);
+
               Fields termVectors = leaf.getTermVectors(idNext);
               assertTrue(termVectors.terms("term_vectors").iterator().seekExact(new BytesRef("test" + ids.longValue())));
               assertEquals(Long.toString(ids.longValue()), leaf.document(idNext).get("id"));