You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Robert Muir <rc...@gmail.com> on 2022/08/31 14:24:00 UTC
Re: [lucene] branch main updated: SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
can we backport to 9.x if you get a chance? I'm still seeing this test
trip in 9.x jenkins builds.
On Mon, Aug 29, 2022 at 11:50 AM <so...@apache.org> wrote:
>
> This is an automated email from the ASF dual-hosted git repository.
>
> sokolov pushed a commit to branch main
> in repository https://gitbox.apache.org/repos/asf/lucene.git
>
>
> The following commit(s) were added to refs/heads/main by this push:
> new 61ef031f7fa SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> 61ef031f7fa is described below
>
> commit 61ef031f7fa3abdd7c8c2f36db71ad2289b66131
> Author: Michael Sokolov <so...@falutin.net>
> AuthorDate: Mon Aug 29 11:49:52 2022 -0400
>
> SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> ---
> .../apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java | 6 +++---
> lucene/core/src/test/org/apache/lucene/document/TestField.java | 4 ++++
> 2 files changed, 7 insertions(+), 3 deletions(-)
>
> diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> index e4b0ceb5916..10700f5de6f 100644
> --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> @@ -41,7 +41,6 @@ import org.apache.lucene.store.BufferedChecksumIndexInput;
> import org.apache.lucene.store.ChecksumIndexInput;
> import org.apache.lucene.store.IOContext;
> import org.apache.lucene.store.IndexInput;
> -import org.apache.lucene.util.BitSet;
> import org.apache.lucene.util.Bits;
> import org.apache.lucene.util.BytesRef;
> import org.apache.lucene.util.BytesRefBuilder;
> @@ -187,8 +186,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
> @Override
> public TopDocs searchExhaustively(
> String field, float[] target, int k, DocIdSetIterator acceptDocs) throws IOException {
> - int numDocs = (int) acceptDocs.cost();
> - return search(field, target, k, BitSet.of(acceptDocs, numDocs), Integer.MAX_VALUE);
> + FieldInfo info = readState.fieldInfos.fieldInfo(field);
> + VectorSimilarityFunction vectorSimilarity = info.getVectorSimilarityFunction();
> + return exhaustiveSearch(getVectorValues(field), acceptDocs, vectorSimilarity, target, k);
> }
>
> @Override
> diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> index 781f2b613c6..6aa5518f33b 100644
> --- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
> +++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> @@ -20,6 +20,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
>
> import java.io.StringReader;
> import java.nio.charset.StandardCharsets;
> +import org.apache.lucene.codecs.Codec;
> import org.apache.lucene.index.DirectoryReader;
> import org.apache.lucene.index.IndexReader;
> import org.apache.lucene.index.IndexWriter;
> @@ -513,6 +514,9 @@ public class TestField extends LuceneTestCase {
> }
>
> public void testKnnVectorField() throws Exception {
> + if (Codec.getDefault().getName().equals("SimpleText")) {
> + return;
> + }
> try (Directory dir = newDirectory();
> IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
> Document doc = new Document();
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: [lucene] branch main updated: SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
Posted by Robert Muir <rc...@gmail.com>.
thanks for fixing!
On Wed, Aug 31, 2022 at 2:43 PM Michael Sokolov <ms...@gmail.com> wrote:
>
> Oh -- sorry, I guess I forgot to backport. Thanks for tracking it down
> - I'll push to branch_9x shortly
>
> On Wed, Aug 31, 2022 at 10:25 AM Robert Muir <rc...@gmail.com> wrote:
> >
> > can we backport to 9.x if you get a chance? I'm still seeing this test
> > trip in 9.x jenkins builds.
> >
> >
> > On Mon, Aug 29, 2022 at 11:50 AM <so...@apache.org> wrote:
> > >
> > > This is an automated email from the ASF dual-hosted git repository.
> > >
> > > sokolov pushed a commit to branch main
> > > in repository https://gitbox.apache.org/repos/asf/lucene.git
> > >
> > >
> > > The following commit(s) were added to refs/heads/main by this push:
> > > new 61ef031f7fa SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> > > 61ef031f7fa is described below
> > >
> > > commit 61ef031f7fa3abdd7c8c2f36db71ad2289b66131
> > > Author: Michael Sokolov <so...@falutin.net>
> > > AuthorDate: Mon Aug 29 11:49:52 2022 -0400
> > >
> > > SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> > > ---
> > > .../apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java | 6 +++---
> > > lucene/core/src/test/org/apache/lucene/document/TestField.java | 4 ++++
> > > 2 files changed, 7 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > > index e4b0ceb5916..10700f5de6f 100644
> > > --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > > +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > > @@ -41,7 +41,6 @@ import org.apache.lucene.store.BufferedChecksumIndexInput;
> > > import org.apache.lucene.store.ChecksumIndexInput;
> > > import org.apache.lucene.store.IOContext;
> > > import org.apache.lucene.store.IndexInput;
> > > -import org.apache.lucene.util.BitSet;
> > > import org.apache.lucene.util.Bits;
> > > import org.apache.lucene.util.BytesRef;
> > > import org.apache.lucene.util.BytesRefBuilder;
> > > @@ -187,8 +186,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
> > > @Override
> > > public TopDocs searchExhaustively(
> > > String field, float[] target, int k, DocIdSetIterator acceptDocs) throws IOException {
> > > - int numDocs = (int) acceptDocs.cost();
> > > - return search(field, target, k, BitSet.of(acceptDocs, numDocs), Integer.MAX_VALUE);
> > > + FieldInfo info = readState.fieldInfos.fieldInfo(field);
> > > + VectorSimilarityFunction vectorSimilarity = info.getVectorSimilarityFunction();
> > > + return exhaustiveSearch(getVectorValues(field), acceptDocs, vectorSimilarity, target, k);
> > > }
> > >
> > > @Override
> > > diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > > index 781f2b613c6..6aa5518f33b 100644
> > > --- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > > +++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > > @@ -20,6 +20,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
> > >
> > > import java.io.StringReader;
> > > import java.nio.charset.StandardCharsets;
> > > +import org.apache.lucene.codecs.Codec;
> > > import org.apache.lucene.index.DirectoryReader;
> > > import org.apache.lucene.index.IndexReader;
> > > import org.apache.lucene.index.IndexWriter;
> > > @@ -513,6 +514,9 @@ public class TestField extends LuceneTestCase {
> > > }
> > >
> > > public void testKnnVectorField() throws Exception {
> > > + if (Codec.getDefault().getName().equals("SimpleText")) {
> > > + return;
> > > + }
> > > try (Directory dir = newDirectory();
> > > IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
> > > Document doc = new Document();
> > >
> >
> > ---------------------------------------------------------------------
> > To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> > For additional commands, e-mail: dev-help@lucene.apache.org
> >
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: [lucene] branch main updated: SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
Posted by Michael Sokolov <ms...@gmail.com>.
Oh -- sorry, I guess I forgot to backport. Thanks for tracking it down
- I'll push to branch_9x shortly
On Wed, Aug 31, 2022 at 10:25 AM Robert Muir <rc...@gmail.com> wrote:
>
> can we backport to 9.x if you get a chance? I'm still seeing this test
> trip in 9.x jenkins builds.
>
>
> On Mon, Aug 29, 2022 at 11:50 AM <so...@apache.org> wrote:
> >
> > This is an automated email from the ASF dual-hosted git repository.
> >
> > sokolov pushed a commit to branch main
> > in repository https://gitbox.apache.org/repos/asf/lucene.git
> >
> >
> > The following commit(s) were added to refs/heads/main by this push:
> > new 61ef031f7fa SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> > 61ef031f7fa is described below
> >
> > commit 61ef031f7fa3abdd7c8c2f36db71ad2289b66131
> > Author: Michael Sokolov <so...@falutin.net>
> > AuthorDate: Mon Aug 29 11:49:52 2022 -0400
> >
> > SimpleText knn vectors; fix searchExhaustively and suppress a byte format test case (#11725)
> > ---
> > .../apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java | 6 +++---
> > lucene/core/src/test/org/apache/lucene/document/TestField.java | 4 ++++
> > 2 files changed, 7 insertions(+), 3 deletions(-)
> >
> > diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > index e4b0ceb5916..10700f5de6f 100644
> > --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextKnnVectorsReader.java
> > @@ -41,7 +41,6 @@ import org.apache.lucene.store.BufferedChecksumIndexInput;
> > import org.apache.lucene.store.ChecksumIndexInput;
> > import org.apache.lucene.store.IOContext;
> > import org.apache.lucene.store.IndexInput;
> > -import org.apache.lucene.util.BitSet;
> > import org.apache.lucene.util.Bits;
> > import org.apache.lucene.util.BytesRef;
> > import org.apache.lucene.util.BytesRefBuilder;
> > @@ -187,8 +186,9 @@ public class SimpleTextKnnVectorsReader extends KnnVectorsReader {
> > @Override
> > public TopDocs searchExhaustively(
> > String field, float[] target, int k, DocIdSetIterator acceptDocs) throws IOException {
> > - int numDocs = (int) acceptDocs.cost();
> > - return search(field, target, k, BitSet.of(acceptDocs, numDocs), Integer.MAX_VALUE);
> > + FieldInfo info = readState.fieldInfos.fieldInfo(field);
> > + VectorSimilarityFunction vectorSimilarity = info.getVectorSimilarityFunction();
> > + return exhaustiveSearch(getVectorValues(field), acceptDocs, vectorSimilarity, target, k);
> > }
> >
> > @Override
> > diff --git a/lucene/core/src/test/org/apache/lucene/document/TestField.java b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > index 781f2b613c6..6aa5518f33b 100644
> > --- a/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > +++ b/lucene/core/src/test/org/apache/lucene/document/TestField.java
> > @@ -20,6 +20,7 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
> >
> > import java.io.StringReader;
> > import java.nio.charset.StandardCharsets;
> > +import org.apache.lucene.codecs.Codec;
> > import org.apache.lucene.index.DirectoryReader;
> > import org.apache.lucene.index.IndexReader;
> > import org.apache.lucene.index.IndexWriter;
> > @@ -513,6 +514,9 @@ public class TestField extends LuceneTestCase {
> > }
> >
> > public void testKnnVectorField() throws Exception {
> > + if (Codec.getDefault().getName().equals("SimpleText")) {
> > + return;
> > + }
> > try (Directory dir = newDirectory();
> > IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) {
> > Document doc = new Document();
> >
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org