You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2022/05/05 17:30:55 UTC

[lucene] branch vectors-disi-direct updated: LUCENE-10502: Use IndexedDISI to store docIds and DirectMonotonicWriter/Reader to handle ordToDoc (#792)

This is an automated email from the ASF dual-hosted git repository.

mayya pushed a commit to branch vectors-disi-direct
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/vectors-disi-direct by this push:
     new b3867da5443 LUCENE-10502: Use IndexedDISI to store docIds and DirectMonotonicWriter/Reader to handle ordToDoc (#792)
b3867da5443 is described below

commit b3867da5443f58c554a3fd8391d3c98e0b2b7790
Author: Lu Xugang <q1...@Gmail.com>
AuthorDate: Fri May 6 01:30:49 2022 +0800

    LUCENE-10502: Use IndexedDISI to store docIds and DirectMonotonicWriter/Reader to handle ordToDoc (#792)
    
    * LUCENE-10502: Use IndexedDISI to store docIds and DirectMonotonicWriter/Reader to handle ordToDoc
    
    * TestBackwardsCompatibility was temporarily removed for skipping test
    
    * dense case and empty case do not need to store ordToMap mapping
    
    * fix
    
    * invert if condition
    
    * add subclass of dense sparse and empty
    
    * spotless
    
    * remove the ord variable
    
    * move `getOffHeapVectorValues` to `OffHeapVectorValues` class as a static method and rename it as `load`
    
    * move the getAcceptOrds method to OffHeapVectorValues
    
    * move OffHeapVectorValues to its own class
    
    * keep OffHeapVectorValues and all its subclasses in one place
    
    * make `OffHeapVectorValues`'s subclasses private except `DenseOffHeapVectorValues`
    
    * add some short comments
---
 .../lucene90/Lucene90HnswGraphBuilder.java         |    3 +-
 .../backward_index/TestBackwardsCompatibility.java | 2218 --------------------
 .../apache/lucene/codecs/lucene90/IndexedDISI.java |    9 +-
 .../codecs/lucene91/Lucene91HnswVectorsFormat.java |    2 +
 .../codecs/lucene91/Lucene91HnswVectorsReader.java |  224 +-
 .../codecs/lucene91/Lucene91HnswVectorsWriter.java |   51 +-
 .../codecs/lucene91/OffHeapVectorValues.java       |  321 +++
 .../index/RandomAccessVectorValuesProducer.java    |    4 +-
 .../apache/lucene/index/VectorValuesWriter.java    |    2 +-
 .../apache/lucene/util/hnsw/HnswGraphBuilder.java  |    3 +-
 .../test/org/apache/lucene/index/TestKnnGraph.java |   10 +-
 11 files changed, 427 insertions(+), 2420 deletions(-)

diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswGraphBuilder.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswGraphBuilder.java
index b59ba3b4a7a..0c2ea0d4768 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswGraphBuilder.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswGraphBuilder.java
@@ -78,7 +78,8 @@ public final class Lucene90HnswGraphBuilder {
       VectorSimilarityFunction similarityFunction,
       int maxConn,
       int beamWidth,
-      long seed) {
+      long seed)
+      throws IOException {
     vectorValues = vectors.randomAccess();
     buildVectors = vectors.randomAccess();
     this.similarityFunction = Objects.requireNonNull(similarityFunction);
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
deleted file mode 100644
index b2528ce683b..00000000000
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
+++ /dev/null
@@ -1,2218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_index;
-
-import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
-import static org.apache.lucene.util.Version.LUCENE_9_0_0;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.lang.reflect.Modifier;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.text.ParsePosition;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.document.BinaryPoint;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
-import org.apache.lucene.document.DoublePoint;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.FloatPoint;
-import org.apache.lucene.document.IntPoint;
-import org.apache.lucene.document.KnnVectorField;
-import org.apache.lucene.document.LongPoint;
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.document.SortedDocValuesField;
-import org.apache.lucene.document.SortedNumericDocValuesField;
-import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.index.CheckIndex;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.Fields;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexFormatTooOldException;
-import org.apache.lucene.index.IndexOptions;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.IndexUpgrader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.LogByteSizeMergePolicy;
-import org.apache.lucene.index.MergePolicy;
-import org.apache.lucene.index.MultiBits;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.MultiTerms;
-import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.SegmentCommitInfo;
-import org.apache.lucene.index.SegmentInfos;
-import org.apache.lucene.index.SegmentReader;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.index.StandardDirectoryReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.VectorSimilarityFunction;
-import org.apache.lucene.index.VectorValues;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.FieldDoc;
-import org.apache.lucene.search.FieldExistsQuery;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.KnnVectorQuery;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.NIOFSDirectory;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.index.RandomIndexWriter;
-import org.apache.lucene.tests.store.BaseDirectoryWrapper;
-import org.apache.lucene.tests.util.LineFileDocs;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.Version;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-/*
-  Verify we can read previous versions' indexes, do searches
-  against them, and add documents to them.
-*/
-// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows
-// machines occasionally
-@SuppressWarnings("deprecation")
-public class TestBackwardsCompatibility extends LuceneTestCase {
-
-  // Backcompat index generation, described below, is mostly automated in:
-  //
-  //    dev-tools/scripts/addBackcompatIndexes.py
-  //
-  // For usage information, see:
-  //
-  //    http://wiki.apache.org/lucene-java/ReleaseTodo#Generate_Backcompat_Indexes
-  //
-  // -----
-  //
-  // To generate backcompat indexes with the current default codec, run the following ant command:
-  //  ant test -Dtestcase=TestBackwardsCompatibility -Dtests.bwcdir=/path/to/store/indexes
-  //           -Dtests.codec=default -Dtests.useSecurityManager=false
-  // Also add testmethod with one of the index creation methods below, for example:
-  //    -Dtestmethod=testCreateCFS
-  //
-  // Zip up the generated indexes:
-  //
-  //    cd /path/to/store/indexes/index.cfs   ; zip index.<VERSION>-cfs.zip *
-  //    cd /path/to/store/indexes/index.nocfs ; zip index.<VERSION>-nocfs.zip *
-  //
-  // Then move those 2 zip files to your trunk checkout and add them
-  // to the oldNames array.
-
-  private static final int DOCS_COUNT = 35;
-  private static final int DELETED_ID = 7;
-
-  private static final int KNN_VECTOR_MIN_SUPPORTED_VERSION = LUCENE_9_0_0.major;
-  private static final String KNN_VECTOR_FIELD = "knn_field";
-  private static final FieldType KNN_VECTOR_FIELD_TYPE =
-      KnnVectorField.createFieldType(3, VectorSimilarityFunction.COSINE);
-  private static final float[] KNN_VECTOR = {0.2f, -0.1f, 0.1f};
-
-  public void testCreateCFS() throws IOException {
-    createIndex("index.cfs", true, false);
-  }
-
-  public void testCreateNoCFS() throws IOException {
-    createIndex("index.nocfs", false, false);
-  }
-
-  // These are only needed for the special upgrade test to verify
-  // that also single-segment indexes are correctly upgraded by IndexUpgrader.
-  // You don't need them to be build for non-4.0 (the test is happy with just one
-  // "old" segment format, version is unimportant:
-
-  public void testCreateSingleSegmentCFS() throws IOException {
-    createIndex("index.singlesegment-cfs", true, true);
-  }
-
-  public void testCreateSingleSegmentNoCFS() throws IOException {
-    createIndex("index.singlesegment-nocfs", false, true);
-  }
-
-  private Path getIndexDir() {
-    String path = System.getProperty("tests.bwcdir");
-    assumeTrue(
-        "backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes",
-        path != null);
-    return Paths.get(path);
-  }
-
-  public void testCreateMoreTermsIndex() throws Exception {
-
-    Path indexDir = getIndexDir().resolve("moreterms");
-    Files.deleteIfExists(indexDir);
-    Directory dir = newFSDirectory(indexDir);
-
-    LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
-    mp.setNoCFSRatio(1.0);
-    mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
-    MockAnalyzer analyzer = new MockAnalyzer(random());
-    analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
-
-    IndexWriterConfig conf =
-        new IndexWriterConfig(analyzer).setMergePolicy(mp).setUseCompoundFile(false);
-    IndexWriter writer = new IndexWriter(dir, conf);
-    LineFileDocs docs = new LineFileDocs(new Random(0));
-    for (int i = 0; i < 50; i++) {
-      writer.addDocument(docs.nextDoc());
-    }
-    docs.close();
-    writer.close();
-    dir.close();
-
-    // Gives you time to copy the index out!: (there is also
-    // a test option to not remove temp dir...):
-    Thread.sleep(100000);
-  }
-
-  // ant test -Dtestcase=TestBackwardsCompatibility -Dtestmethod=testCreateSortedIndex
-  // -Dtests.codec=default -Dtests.useSecurityManager=false -Dtests.bwcdir=/tmp/sorted
-  public void testCreateSortedIndex() throws Exception {
-
-    Path indexDir = getIndexDir().resolve("sorted");
-    Files.deleteIfExists(indexDir);
-    Directory dir = newFSDirectory(indexDir);
-
-    LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
-    mp.setNoCFSRatio(1.0);
-    mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
-    MockAnalyzer analyzer = new MockAnalyzer(random());
-    analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
-
-    // TODO: remove randomness
-    IndexWriterConfig conf = new IndexWriterConfig(analyzer);
-    conf.setMergePolicy(mp);
-    conf.setUseCompoundFile(false);
-    conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));
-    IndexWriter writer = new IndexWriter(dir, conf);
-    LineFileDocs docs = new LineFileDocs(random());
-    SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
-    parser.setTimeZone(TimeZone.getTimeZone("UTC"));
-    ParsePosition position = new ParsePosition(0);
-    Field dateDVField = null;
-    for (int i = 0; i < 50; i++) {
-      Document doc = docs.nextDoc();
-      String dateString = doc.get("date");
-
-      position.setIndex(0);
-      Date date = parser.parse(dateString, position);
-      if (position.getErrorIndex() != -1) {
-        throw new AssertionError("failed to parse \"" + dateString + "\" as date");
-      }
-      if (position.getIndex() != dateString.length()) {
-        throw new AssertionError("failed to parse \"" + dateString + "\" as date");
-      }
-      if (dateDVField == null) {
-        dateDVField = new NumericDocValuesField("dateDV", 0l);
-        doc.add(dateDVField);
-      }
-      dateDVField.setLongValue(date.getTime());
-      if (i == 250) {
-        writer.commit();
-      }
-      writer.addDocument(doc);
-    }
-    writer.forceMerge(1);
-    writer.close();
-    dir.close();
-  }
-
-  private void updateNumeric(IndexWriter writer, String id, String f, String cf, long value)
-      throws IOException {
-    writer.updateNumericDocValue(new Term("id", id), f, value);
-    writer.updateNumericDocValue(new Term("id", id), cf, value * 2);
-  }
-
-  private void updateBinary(IndexWriter writer, String id, String f, String cf, long value)
-      throws IOException {
-    writer.updateBinaryDocValue(new Term("id", id), f, toBytes(value));
-    writer.updateBinaryDocValue(new Term("id", id), cf, toBytes(value * 2));
-  }
-
-  // Creates an index with DocValues updates
-  public void testCreateIndexWithDocValuesUpdates() throws Exception {
-    Path indexDir = getIndexDir().resolve("dvupdates");
-    Files.deleteIfExists(indexDir);
-    Directory dir = newFSDirectory(indexDir);
-
-    IndexWriterConfig conf =
-        new IndexWriterConfig(new MockAnalyzer(random()))
-            .setUseCompoundFile(false)
-            .setMergePolicy(NoMergePolicy.INSTANCE);
-    IndexWriter writer = new IndexWriter(dir, conf);
-    // create an index w/ few doc-values fields, some with updates and some without
-    for (int i = 0; i < 30; i++) {
-      Document doc = new Document();
-      doc.add(new StringField("id", "" + i, Field.Store.NO));
-      doc.add(new NumericDocValuesField("ndv1", i));
-      doc.add(new NumericDocValuesField("ndv1_c", i * 2));
-      doc.add(new NumericDocValuesField("ndv2", i * 3));
-      doc.add(new NumericDocValuesField("ndv2_c", i * 6));
-      doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));
-      doc.add(new BinaryDocValuesField("bdv1_c", toBytes(i * 2)));
-      doc.add(new BinaryDocValuesField("bdv2", toBytes(i * 3)));
-      doc.add(new BinaryDocValuesField("bdv2_c", toBytes(i * 6)));
-      writer.addDocument(doc);
-      if ((i + 1) % 10 == 0) {
-        writer.commit(); // flush every 10 docs
-      }
-    }
-
-    // first segment: no updates
-
-    // second segment: update two fields, same gen
-    updateNumeric(writer, "10", "ndv1", "ndv1_c", 100L);
-    updateBinary(writer, "11", "bdv1", "bdv1_c", 100L);
-    writer.commit();
-
-    // third segment: update few fields, different gens, few docs
-    updateNumeric(writer, "20", "ndv1", "ndv1_c", 100L);
-    updateBinary(writer, "21", "bdv1", "bdv1_c", 100L);
-    writer.commit();
-    updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again
-    writer.commit();
-
-    writer.close();
-    dir.close();
-  }
-
-  public void testCreateEmptyIndex() throws Exception {
-    Path indexDir = getIndexDir().resolve("emptyIndex");
-    Files.deleteIfExists(indexDir);
-    IndexWriterConfig conf =
-        new IndexWriterConfig(new MockAnalyzer(random()))
-            .setUseCompoundFile(false)
-            .setMergePolicy(NoMergePolicy.INSTANCE);
-    try (Directory dir = newFSDirectory(indexDir);
-        IndexWriter writer = new IndexWriter(dir, conf)) {
-      writer.flush();
-    }
-  }
-
-  static final String[] oldNames = {
-    "9.0.0-cfs", // Force on separate lines
-    "9.0.0-nocfs",
-    "9.1.0-cfs",
-    "9.1.0-nocfs"
-  };
-
-  public static String[] getOldNames() {
-    return oldNames;
-  }
-
-  static final String[] oldSortedNames = {
-    "sorted.9.0.0", // Force on separate lines
-    "sorted.9.1.0"
-  };
-
-  public static String[] getOldSortedNames() {
-    return oldSortedNames;
-  }
-
-  static final String[] unsupportedNames = {
-    "1.9.0-cfs",
-    "1.9.0-nocfs",
-    "2.0.0-cfs",
-    "2.0.0-nocfs",
-    "2.1.0-cfs",
-    "2.1.0-nocfs",
-    "2.2.0-cfs",
-    "2.2.0-nocfs",
-    "2.3.0-cfs",
-    "2.3.0-nocfs",
-    "2.4.0-cfs",
-    "2.4.0-nocfs",
-    "2.4.1-cfs",
-    "2.4.1-nocfs",
-    "2.9.0-cfs",
-    "2.9.0-nocfs",
-    "2.9.1-cfs",
-    "2.9.1-nocfs",
-    "2.9.2-cfs",
-    "2.9.2-nocfs",
-    "2.9.3-cfs",
-    "2.9.3-nocfs",
-    "2.9.4-cfs",
-    "2.9.4-nocfs",
-    "3.0.0-cfs",
-    "3.0.0-nocfs",
-    "3.0.1-cfs",
-    "3.0.1-nocfs",
-    "3.0.2-cfs",
-    "3.0.2-nocfs",
-    "3.0.3-cfs",
-    "3.0.3-nocfs",
-    "3.1.0-cfs",
-    "3.1.0-nocfs",
-    "3.2.0-cfs",
-    "3.2.0-nocfs",
-    "3.3.0-cfs",
-    "3.3.0-nocfs",
-    "3.4.0-cfs",
-    "3.4.0-nocfs",
-    "3.5.0-cfs",
-    "3.5.0-nocfs",
-    "3.6.0-cfs",
-    "3.6.0-nocfs",
-    "3.6.1-cfs",
-    "3.6.1-nocfs",
-    "3.6.2-cfs",
-    "3.6.2-nocfs",
-    "4.0.0-cfs",
-    "4.0.0-cfs",
-    "4.0.0-nocfs",
-    "4.0.0.1-cfs",
-    "4.0.0.1-nocfs",
-    "4.0.0.2-cfs",
-    "4.0.0.2-nocfs",
-    "4.1.0-cfs",
-    "4.1.0-nocfs",
-    "4.2.0-cfs",
-    "4.2.0-nocfs",
-    "4.2.1-cfs",
-    "4.2.1-nocfs",
-    "4.3.0-cfs",
-    "4.3.0-nocfs",
-    "4.3.1-cfs",
-    "4.3.1-nocfs",
-    "4.4.0-cfs",
-    "4.4.0-nocfs",
-    "4.5.0-cfs",
-    "4.5.0-nocfs",
-    "4.5.1-cfs",
-    "4.5.1-nocfs",
-    "4.6.0-cfs",
-    "4.6.0-nocfs",
-    "4.6.1-cfs",
-    "4.6.1-nocfs",
-    "4.7.0-cfs",
-    "4.7.0-nocfs",
-    "4.7.1-cfs",
-    "4.7.1-nocfs",
-    "4.7.2-cfs",
-    "4.7.2-nocfs",
-    "4.8.0-cfs",
-    "4.8.0-nocfs",
-    "4.8.1-cfs",
-    "4.8.1-nocfs",
-    "4.9.0-cfs",
-    "4.9.0-nocfs",
-    "4.9.1-cfs",
-    "4.9.1-nocfs",
-    "4.10.0-cfs",
-    "4.10.0-nocfs",
-    "4.10.1-cfs",
-    "4.10.1-nocfs",
-    "4.10.2-cfs",
-    "4.10.2-nocfs",
-    "4.10.3-cfs",
-    "4.10.3-nocfs",
-    "4.10.4-cfs",
-    "4.10.4-nocfs",
-    "5x-with-4x-segments-cfs",
-    "5x-with-4x-segments-nocfs",
-    "5.0.0.singlesegment-cfs",
-    "5.0.0.singlesegment-nocfs",
-    "5.0.0-cfs",
-    "5.0.0-nocfs",
-    "5.1.0-cfs",
-    "5.1.0-nocfs",
-    "5.2.0-cfs",
-    "5.2.0-nocfs",
-    "5.2.1-cfs",
-    "5.2.1-nocfs",
-    "5.3.0-cfs",
-    "5.3.0-nocfs",
-    "5.3.1-cfs",
-    "5.3.1-nocfs",
-    "5.3.2-cfs",
-    "5.3.2-nocfs",
-    "5.4.0-cfs",
-    "5.4.0-nocfs",
-    "5.4.1-cfs",
-    "5.4.1-nocfs",
-    "5.5.0-cfs",
-    "5.5.0-nocfs",
-    "5.5.1-cfs",
-    "5.5.1-nocfs",
-    "5.5.2-cfs",
-    "5.5.2-nocfs",
-    "5.5.3-cfs",
-    "5.5.3-nocfs",
-    "5.5.4-cfs",
-    "5.5.4-nocfs",
-    "5.5.5-cfs",
-    "5.5.5-nocfs",
-    "6.0.0-cfs",
-    "6.0.0-nocfs",
-    "6.0.1-cfs",
-    "6.0.1-nocfs",
-    "6.1.0-cfs",
-    "6.1.0-nocfs",
-    "6.2.0-cfs",
-    "6.2.0-nocfs",
-    "6.2.1-cfs",
-    "6.2.1-nocfs",
-    "6.3.0-cfs",
-    "6.3.0-nocfs",
-    "6.4.0-cfs",
-    "6.4.0-nocfs",
-    "6.4.1-cfs",
-    "6.4.1-nocfs",
-    "6.4.2-cfs",
-    "6.4.2-nocfs",
-    "6.5.0-cfs",
-    "6.5.0-nocfs",
-    "6.5.1-cfs",
-    "6.5.1-nocfs",
-    "6.6.0-cfs",
-    "6.6.0-nocfs",
-    "6.6.1-cfs",
-    "6.6.1-nocfs",
-    "6.6.2-cfs",
-    "6.6.2-nocfs",
-    "6.6.3-cfs",
-    "6.6.3-nocfs",
-    "6.6.4-cfs",
-    "6.6.4-nocfs",
-    "6.6.5-cfs",
-    "6.6.5-nocfs",
-    "6.6.6-cfs",
-    "6.6.6-nocfs",
-    "7.0.0-cfs",
-    "7.0.0-nocfs",
-    "7.0.1-cfs",
-    "7.0.1-nocfs",
-    "7.1.0-cfs",
-    "7.1.0-nocfs",
-    "7.2.0-cfs",
-    "7.2.0-nocfs",
-    "7.2.1-cfs",
-    "7.2.1-nocfs",
-    "7.3.0-cfs",
-    "7.3.0-nocfs",
-    "7.3.1-cfs",
-    "7.3.1-nocfs",
-    "7.4.0-cfs",
-    "7.4.0-nocfs",
-    "7.5.0-cfs",
-    "7.5.0-nocfs",
-    "7.6.0-cfs",
-    "7.6.0-nocfs",
-    "7.7.0-cfs",
-    "7.7.0-nocfs",
-    "7.7.1-cfs",
-    "7.7.1-nocfs",
-    "7.7.2-cfs",
-    "7.7.2-nocfs",
-    "7.7.3-cfs",
-    "7.7.3-nocfs",
-    "8.0.0-cfs",
-    "8.0.0-nocfs",
-    "8.1.0-cfs",
-    "8.1.0-nocfs",
-    "8.1.1-cfs",
-    "8.1.1-nocfs",
-    "8.2.0-cfs",
-    "8.2.0-nocfs",
-    "8.3.0-cfs",
-    "8.3.0-nocfs",
-    "8.3.1-cfs",
-    "8.3.1-nocfs",
-    "8.4.0-cfs",
-    "8.4.0-nocfs",
-    "8.4.1-cfs",
-    "8.4.1-nocfs",
-    "8.5.0-cfs",
-    "8.5.0-nocfs",
-    "8.5.1-cfs",
-    "8.5.1-nocfs",
-    "8.5.2-cfs",
-    "8.5.2-nocfs",
-    "8.6.0-cfs",
-    "8.6.0-nocfs",
-    "8.6.1-cfs",
-    "8.6.1-nocfs",
-    "8.6.2-cfs",
-    "8.6.2-nocfs",
-    "8.6.3-cfs",
-    "8.6.3-nocfs",
-    "8.7.0-cfs",
-    "8.7.0-nocfs",
-    "8.8.0-cfs",
-    "8.8.0-nocfs",
-    "8.8.1-cfs",
-    "8.8.1-nocfs",
-    "8.8.2-cfs",
-    "8.8.2-nocfs",
-    "8.9.0-cfs",
-    "8.9.0-nocfs",
-    "8.10.0-cfs",
-    "8.10.0-nocfs",
-    "8.10.1-cfs",
-    "8.10.1-nocfs",
-    "8.11.0-cfs",
-    "8.11.0-nocfs",
-    "8.11.1-cfs",
-    "8.11.1-nocfs"
-  };
-
-  static final int MIN_BINARY_SUPPORTED_MAJOR = Version.MIN_SUPPORTED_MAJOR - 1;
-
-  static final String[] binarySupportedNames;
-
-  static {
-    ArrayList<String> list = new ArrayList<>();
-    for (String name : unsupportedNames) {
-      if (name.startsWith(MIN_BINARY_SUPPORTED_MAJOR + ".")) {
-        list.add(name);
-      }
-    }
-    binarySupportedNames = list.toArray(new String[0]);
-  }
-
-  // TODO: on 6.0.0 release, gen the single segment indices and add here:
-  static final String[] oldSingleSegmentNames = {};
-
-  public static String[] getOldSingleSegmentNames() {
-    return oldSingleSegmentNames;
-  }
-
-  static Map<String, Directory> oldIndexDirs;
-
-  /** Randomizes the use of some of hte constructor variations */
-  private static IndexUpgrader newIndexUpgrader(Directory dir) {
-    final boolean streamType = random().nextBoolean();
-    final int choice = TestUtil.nextInt(random(), 0, 2);
-    switch (choice) {
-      case 0:
-        return new IndexUpgrader(dir);
-      case 1:
-        return new IndexUpgrader(dir, streamType ? null : InfoStream.NO_OUTPUT, false);
-      case 2:
-        return new IndexUpgrader(dir, newIndexWriterConfig(null), false);
-      default:
-        fail("case statement didn't get updated when random bounds changed");
-    }
-    return null; // never get here
-  }
-
-  @BeforeClass
-  public static void beforeClass() throws Exception {
-    List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
-    names.addAll(Arrays.asList(oldNames));
-    names.addAll(Arrays.asList(oldSingleSegmentNames));
-    oldIndexDirs = new HashMap<>();
-    for (String name : names) {
-      Path dir = createTempDir(name);
-      InputStream resource =
-          TestBackwardsCompatibility.class.getResourceAsStream("index." + name + ".zip");
-      assertNotNull("Index name " + name + " not found", resource);
-      TestUtil.unzip(resource, dir);
-      oldIndexDirs.put(name, newFSDirectory(dir));
-    }
-  }
-
-  @AfterClass
-  public static void afterClass() throws Exception {
-    for (Directory d : oldIndexDirs.values()) {
-      d.close();
-    }
-    oldIndexDirs = null;
-  }
-
-  public void testAllVersionHaveCfsAndNocfs() {
-    // ensure all tested versions with cfs also have nocfs
-    String[] files = new String[oldNames.length];
-    System.arraycopy(oldNames, 0, files, 0, oldNames.length);
-    Arrays.sort(files);
-    String prevFile = "";
-    for (String file : files) {
-      if (prevFile.endsWith("-cfs")) {
-        String prefix = prevFile.replace("-cfs", "");
-        assertEquals("Missing -nocfs for backcompat index " + prefix, prefix + "-nocfs", file);
-      }
-    }
-  }
-
-  public void testAllVersionsTested() throws Exception {
-    Pattern constantPattern = Pattern.compile("LUCENE_(\\d+)_(\\d+)_(\\d+)(_ALPHA|_BETA)?");
-    // find the unique versions according to Version.java
-    List<String> expectedVersions = new ArrayList<>();
-    for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) {
-      if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {
-        Version v = (Version) field.get(Version.class);
-        if (v.equals(Version.LATEST)) {
-          continue;
-        }
-
-        Matcher constant = constantPattern.matcher(field.getName());
-        if (constant.matches() == false) {
-          continue;
-        }
-
-        expectedVersions.add(v.toString() + "-cfs");
-      }
-    }
-
-    // BEGIN TRUNK ONLY BLOCK
-    // on trunk, the last release of the prev major release is also untested
-    Version lastPrevMajorVersion = null;
-    for (java.lang.reflect.Field field : Version.class.getDeclaredFields()) {
-      if (Modifier.isStatic(field.getModifiers()) && field.getType() == Version.class) {
-        Version v = (Version) field.get(Version.class);
-        Matcher constant = constantPattern.matcher(field.getName());
-        if (constant.matches() == false) continue;
-        if (v.major == Version.LATEST.major - 1
-            && (lastPrevMajorVersion == null || v.onOrAfter(lastPrevMajorVersion))) {
-          lastPrevMajorVersion = v;
-        }
-      }
-    }
-    assertNotNull(lastPrevMajorVersion);
-    expectedVersions.remove(lastPrevMajorVersion.toString() + "-cfs");
-    // END TRUNK ONLY BLOCK
-
-    Collections.sort(expectedVersions);
-
-    // find what versions we are testing
-    List<String> testedVersions = new ArrayList<>();
-    for (String testedVersion : oldNames) {
-      if (testedVersion.endsWith("-cfs") == false) {
-        continue;
-      }
-      testedVersions.add(testedVersion);
-    }
-    Collections.sort(testedVersions);
-
-    int i = 0;
-    int j = 0;
-    List<String> missingFiles = new ArrayList<>();
-    List<String> extraFiles = new ArrayList<>();
-    while (i < expectedVersions.size() && j < testedVersions.size()) {
-      String expectedVersion = expectedVersions.get(i);
-      String testedVersion = testedVersions.get(j);
-      int compare = expectedVersion.compareTo(testedVersion);
-      if (compare == 0) { // equal, we can move on
-        ++i;
-        ++j;
-      } else if (compare < 0) { // didn't find test for version constant
-        missingFiles.add(expectedVersion);
-        ++i;
-      } else { // extra test file
-        extraFiles.add(testedVersion);
-        ++j;
-      }
-    }
-    while (i < expectedVersions.size()) {
-      missingFiles.add(expectedVersions.get(i));
-      ++i;
-    }
-    while (j < testedVersions.size()) {
-      missingFiles.add(testedVersions.get(j));
-      ++j;
-    }
-
-    // we could be missing up to 1 file, which may be due to a release that is in progress
-    if (missingFiles.size() <= 1 && extraFiles.isEmpty()) {
-      // success
-      return;
-    }
-
-    StringBuffer msg = new StringBuffer();
-    if (missingFiles.size() > 1) {
-      msg.append("Missing backcompat test files:\n");
-      for (String missingFile : missingFiles) {
-        msg.append("  " + missingFile + "\n");
-      }
-    }
-    if (extraFiles.isEmpty() == false) {
-      msg.append("Extra backcompat test files:\n");
-      for (String extraFile : extraFiles) {
-        msg.append("  " + extraFile + "\n");
-      }
-    }
-    fail(msg.toString());
-  }
-
-  /**
-   * This test checks that *only* IndexFormatTooOldExceptions are thrown when you open and operate
-   * on too old indexes!
-   */
-  public void testUnsupportedOldIndexes() throws Exception {
-    for (int i = 0; i < unsupportedNames.length; i++) {
-      if (VERBOSE) {
-        System.out.println("TEST: index " + unsupportedNames[i]);
-      }
-      Path oldIndexDir = createTempDir(unsupportedNames[i]);
-      TestUtil.unzip(
-          getDataInputStream("unsupported." + unsupportedNames[i] + ".zip"), oldIndexDir);
-      BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir);
-      // don't checkindex, these are intentionally not supported
-      dir.setCheckIndexOnClose(false);
-
-      IndexReader reader = null;
-      IndexWriter writer = null;
-      try {
-        reader = DirectoryReader.open(dir);
-        fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);
-      } catch (IndexFormatTooOldException e) {
-        if (e.getReason() != null) {
-          assertNull(e.getVersion());
-          assertNull(e.getMinVersion());
-          assertNull(e.getMaxVersion());
-          assertEquals(
-              e.getMessage(),
-              new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())
-                  .getMessage());
-        } else {
-          assertNotNull(e.getVersion());
-          assertNotNull(e.getMinVersion());
-          assertNotNull(e.getMaxVersion());
-          assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());
-          assertTrue(
-              e.getMessage(),
-              e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());
-          assertEquals(
-              e.getMessage(),
-              new IndexFormatTooOldException(
-                      e.getResourceDescription(),
-                      e.getVersion(),
-                      e.getMinVersion(),
-                      e.getMaxVersion())
-                  .getMessage());
-        }
-        // pass
-        if (VERBOSE) {
-          System.out.println("TEST: got expected exc:");
-          e.printStackTrace(System.out);
-        }
-      } finally {
-        if (reader != null) reader.close();
-        reader = null;
-      }
-
-      try {
-        writer =
-            new IndexWriter(
-                dir, newIndexWriterConfig(new MockAnalyzer(random())).setCommitOnClose(false));
-        fail("IndexWriter creation should not pass for " + unsupportedNames[i]);
-      } catch (IndexFormatTooOldException e) {
-        if (e.getReason() != null) {
-          assertNull(e.getVersion());
-          assertNull(e.getMinVersion());
-          assertNull(e.getMaxVersion());
-          assertEquals(
-              e.getMessage(),
-              new IndexFormatTooOldException(e.getResourceDescription(), e.getReason())
-                  .getMessage());
-        } else {
-          assertNotNull(e.getVersion());
-          assertNotNull(e.getMinVersion());
-          assertNotNull(e.getMaxVersion());
-          assertTrue(e.getMessage(), e.getMaxVersion() >= e.getMinVersion());
-          assertTrue(
-              e.getMessage(),
-              e.getMaxVersion() < e.getVersion() || e.getVersion() < e.getMinVersion());
-          assertEquals(
-              e.getMessage(),
-              new IndexFormatTooOldException(
-                      e.getResourceDescription(),
-                      e.getVersion(),
-                      e.getMinVersion(),
-                      e.getMaxVersion())
-                  .getMessage());
-        }
-        // pass
-        if (VERBOSE) {
-          System.out.println("TEST: got expected exc:");
-          e.printStackTrace(System.out);
-        }
-        // Make sure exc message includes a path=
-        assertTrue("got exc message: " + e.getMessage(), e.getMessage().indexOf("path=\"") != -1);
-      } finally {
-        // we should fail to open IW, and so it should be null when we get here.
-        // However, if the test fails (i.e., IW did not fail on open), we need
-        // to close IW. However, if merges are run, IW may throw
-        // IndexFormatTooOldException, and we don't want to mask the fail()
-        // above, so close without waiting for merges.
-        if (writer != null) {
-          try {
-            writer.commit();
-          } finally {
-            writer.close();
-          }
-        }
-      }
-
-      ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-      CheckIndex checker = new CheckIndex(dir);
-      checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8));
-      CheckIndex.Status indexStatus = checker.checkIndex();
-      if (unsupportedNames[i].startsWith("8.")) {
-        assertTrue(indexStatus.clean);
-      } else {
-        assertFalse(indexStatus.clean);
-        // CheckIndex doesn't enforce a minimum version, so we either get an
-        // IndexFormatTooOldException
-        // or an IllegalArgumentException saying that the codec doesn't exist.
-        boolean formatTooOld =
-            bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName());
-        boolean missingCodec = bos.toString(IOUtils.UTF_8).contains("Could not load codec");
-        assertTrue(formatTooOld || missingCodec);
-      }
-      checker.close();
-
-      dir.close();
-    }
-  }
-
-  public void testFullyMergeOldIndex() throws Exception {
-    for (String name : oldNames) {
-      if (VERBOSE) {
-        System.out.println("\nTEST: index=" + name);
-      }
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-
-      final SegmentInfos oldSegInfos = SegmentInfos.readLatestCommit(dir);
-
-      IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())));
-      w.forceMerge(1);
-      w.close();
-
-      final SegmentInfos segInfos = SegmentInfos.readLatestCommit(dir);
-      assertEquals(
-          oldSegInfos.getIndexCreatedVersionMajor(), segInfos.getIndexCreatedVersionMajor());
-      assertEquals(Version.LATEST, segInfos.asList().get(0).info.getVersion());
-      assertEquals(
-          oldSegInfos.asList().get(0).info.getMinVersion(),
-          segInfos.asList().get(0).info.getMinVersion());
-
-      dir.close();
-    }
-  }
-
-  public void testAddOldIndexes() throws IOException {
-    for (String name : oldNames) {
-      if (VERBOSE) {
-        System.out.println("\nTEST: old index " + name);
-      }
-      Directory oldDir = oldIndexDirs.get(name);
-      SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
-
-      Directory targetDir = newDirectory();
-      if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
-        // both indexes are not compatible
-        Directory targetDir2 = newDirectory();
-        IndexWriter w =
-            new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
-        IllegalArgumentException e =
-            expectThrows(IllegalArgumentException.class, () -> w.addIndexes(oldDir));
-        assertTrue(
-            e.getMessage(),
-            e.getMessage()
-                .startsWith(
-                    "Cannot use addIndexes(Directory) with indexes that have been created by a different Lucene version."));
-        w.close();
-        targetDir2.close();
-
-        // for the next test, we simulate writing to an index that was created on the same major
-        // version
-        new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
-      }
-
-      IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
-      w.addIndexes(oldDir);
-      w.close();
-
-      SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);
-      assertNull(
-          "none of the segments should have been upgraded",
-          si.asList().stream()
-              .filter( // depending on the MergePolicy we might see these segments merged away
-                  sci ->
-                      sci.getId() != null
-                          && sci.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0)) == false)
-              .findAny()
-              .orElse(null));
-      if (VERBOSE) {
-        System.out.println("\nTEST: done adding indices; now close");
-      }
-
-      targetDir.close();
-    }
-  }
-
-  public void testAddOldIndexesReader() throws IOException {
-    for (String name : oldNames) {
-      Directory oldDir = oldIndexDirs.get(name);
-      SegmentInfos infos = SegmentInfos.readLatestCommit(oldDir);
-      DirectoryReader reader = DirectoryReader.open(oldDir);
-
-      Directory targetDir = newDirectory();
-      if (infos.getCommitLuceneVersion().major != Version.LATEST.major) {
-        Directory targetDir2 = newDirectory();
-        IndexWriter w =
-            new IndexWriter(targetDir2, newIndexWriterConfig(new MockAnalyzer(random())));
-        IllegalArgumentException e =
-            expectThrows(
-                IllegalArgumentException.class, () -> TestUtil.addIndexesSlowly(w, reader));
-        assertEquals(
-            e.getMessage(),
-            "Cannot merge a segment that has been created with major version 9 into this index which has been created by major version 10");
-        w.close();
-        targetDir2.close();
-
-        // for the next test, we simulate writing to an index that was created on the same major
-        // version
-        new SegmentInfos(infos.getIndexCreatedVersionMajor()).commit(targetDir);
-      }
-      IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(new MockAnalyzer(random())));
-      TestUtil.addIndexesSlowly(w, reader);
-      w.close();
-      reader.close();
-      SegmentInfos si = SegmentInfos.readLatestCommit(targetDir);
-      assertNull(
-          "all SCIs should have an id now",
-          si.asList().stream().filter(sci -> sci.getId() == null).findAny().orElse(null));
-      targetDir.close();
-    }
-  }
-
-  public void testSearchOldIndex() throws Exception {
-    for (String name : oldNames) {
-      Version version = Version.parse(name.substring(0, name.indexOf('-')));
-      searchIndex(oldIndexDirs.get(name), name, Version.MIN_SUPPORTED_MAJOR, version);
-    }
-
-    if (TEST_NIGHTLY) {
-      for (String name : binarySupportedNames) {
-        Path oldIndexDir = createTempDir(name);
-        TestUtil.unzip(getDataInputStream("unsupported." + name + ".zip"), oldIndexDir);
-        try (BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir)) {
-          Version version = Version.parse(name.substring(0, name.indexOf('-')));
-          searchIndex(dir, name, MIN_BINARY_SUPPORTED_MAJOR, version);
-        }
-      }
-    }
-  }
-
-  public void testIndexOldIndexNoAdds() throws Exception {
-    for (String name : oldNames) {
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-      Version version = Version.parse(name.substring(0, name.indexOf('-')));
-      changeIndexNoAdds(random(), dir, version);
-      dir.close();
-    }
-  }
-
-  public void testIndexOldIndex() throws Exception {
-    for (String name : oldNames) {
-      if (VERBOSE) {
-        System.out.println("TEST: oldName=" + name);
-      }
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-      Version v = Version.parse(name.substring(0, name.indexOf('-')));
-      changeIndexWithAdds(random(), dir, v);
-      dir.close();
-    }
-  }
-
-  private void doTestHits(ScoreDoc[] hits, int expectedCount, IndexReader reader)
-      throws IOException {
-    final int hitCount = hits.length;
-    assertEquals("wrong number of hits", expectedCount, hitCount);
-    for (int i = 0; i < hitCount; i++) {
-      reader.document(hits[i].doc);
-      reader.getTermVectors(hits[i].doc);
-    }
-  }
-
-  public void searchIndex(
-      Directory dir, String oldName, int minIndexMajorVersion, Version nameVersion)
-      throws IOException {
-    // QueryParser parser = new QueryParser("contents", new MockAnalyzer(random));
-    // Query query = parser.parse("handle:1");
-    IndexCommit indexCommit = DirectoryReader.listCommits(dir).get(0);
-    IndexReader reader = DirectoryReader.open(indexCommit, minIndexMajorVersion, null);
-    IndexSearcher searcher = newSearcher(reader);
-
-    TestUtil.checkIndex(dir);
-
-    final Bits liveDocs = MultiBits.getLiveDocs(reader);
-    assertNotNull(liveDocs);
-
-    for (int i = 0; i < DOCS_COUNT; i++) {
-      if (liveDocs.get(i)) {
-        Document d = reader.document(i);
-        List<IndexableField> fields = d.getFields();
-        boolean isProxDoc = d.getField("content3") == null;
-        if (isProxDoc) {
-          assertEquals(7, fields.size());
-          IndexableField f = d.getField("id");
-          assertEquals("" + i, f.stringValue());
-
-          f = d.getField("utf8");
-          assertEquals(
-              "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
-
-          f = d.getField("autf8");
-          assertEquals(
-              "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
-
-          f = d.getField("content2");
-          assertEquals("here is more content with aaa aaa aaa", f.stringValue());
-
-          f = d.getField("fie\u2C77ld");
-          assertEquals("field with non-ascii name", f.stringValue());
-        }
-
-        Fields tfvFields = reader.getTermVectors(i);
-        assertNotNull("i=" + i, tfvFields);
-        Terms tfv = tfvFields.terms("utf8");
-        assertNotNull("docID=" + i + " index=" + oldName, tfv);
-      } else {
-        assertEquals(DELETED_ID, i);
-      }
-    }
-
-    // check docvalues fields
-    NumericDocValues dvByte = MultiDocValues.getNumericValues(reader, "dvByte");
-    BinaryDocValues dvBytesDerefFixed = MultiDocValues.getBinaryValues(reader, "dvBytesDerefFixed");
-    BinaryDocValues dvBytesDerefVar = MultiDocValues.getBinaryValues(reader, "dvBytesDerefVar");
-    SortedDocValues dvBytesSortedFixed =
-        MultiDocValues.getSortedValues(reader, "dvBytesSortedFixed");
-    SortedDocValues dvBytesSortedVar = MultiDocValues.getSortedValues(reader, "dvBytesSortedVar");
-    BinaryDocValues dvBytesStraightFixed =
-        MultiDocValues.getBinaryValues(reader, "dvBytesStraightFixed");
-    BinaryDocValues dvBytesStraightVar =
-        MultiDocValues.getBinaryValues(reader, "dvBytesStraightVar");
-    NumericDocValues dvDouble = MultiDocValues.getNumericValues(reader, "dvDouble");
-    NumericDocValues dvFloat = MultiDocValues.getNumericValues(reader, "dvFloat");
-    NumericDocValues dvInt = MultiDocValues.getNumericValues(reader, "dvInt");
-    NumericDocValues dvLong = MultiDocValues.getNumericValues(reader, "dvLong");
-    NumericDocValues dvPacked = MultiDocValues.getNumericValues(reader, "dvPacked");
-    NumericDocValues dvShort = MultiDocValues.getNumericValues(reader, "dvShort");
-
-    SortedSetDocValues dvSortedSet = MultiDocValues.getSortedSetValues(reader, "dvSortedSet");
-    SortedNumericDocValues dvSortedNumeric =
-        MultiDocValues.getSortedNumericValues(reader, "dvSortedNumeric");
-
-    for (int i = 0; i < DOCS_COUNT; i++) {
-      int id = Integer.parseInt(reader.document(i).get("id"));
-      assertEquals(i, dvByte.nextDoc());
-      assertEquals(id, dvByte.longValue());
-
-      byte[] bytes =
-          new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
-      BytesRef expectedRef = new BytesRef(bytes);
-
-      assertEquals(i, dvBytesDerefFixed.nextDoc());
-      BytesRef term = dvBytesDerefFixed.binaryValue();
-      assertEquals(expectedRef, term);
-      assertEquals(i, dvBytesDerefVar.nextDoc());
-      term = dvBytesDerefVar.binaryValue();
-      assertEquals(expectedRef, term);
-      assertEquals(i, dvBytesSortedFixed.nextDoc());
-      term = dvBytesSortedFixed.lookupOrd(dvBytesSortedFixed.ordValue());
-      assertEquals(expectedRef, term);
-      assertEquals(i, dvBytesSortedVar.nextDoc());
-      term = dvBytesSortedVar.lookupOrd(dvBytesSortedVar.ordValue());
-      assertEquals(expectedRef, term);
-      assertEquals(i, dvBytesStraightFixed.nextDoc());
-      term = dvBytesStraightFixed.binaryValue();
-      assertEquals(expectedRef, term);
-      assertEquals(i, dvBytesStraightVar.nextDoc());
-      term = dvBytesStraightVar.binaryValue();
-      assertEquals(expectedRef, term);
-
-      assertEquals(i, dvDouble.nextDoc());
-      assertEquals((double) id, Double.longBitsToDouble(dvDouble.longValue()), 0D);
-      assertEquals(i, dvFloat.nextDoc());
-      assertEquals((float) id, Float.intBitsToFloat((int) dvFloat.longValue()), 0F);
-      assertEquals(i, dvInt.nextDoc());
-      assertEquals(id, dvInt.longValue());
-      assertEquals(i, dvLong.nextDoc());
-      assertEquals(id, dvLong.longValue());
-      assertEquals(i, dvPacked.nextDoc());
-      assertEquals(id, dvPacked.longValue());
-      assertEquals(i, dvShort.nextDoc());
-      assertEquals(id, dvShort.longValue());
-
-      assertEquals(i, dvSortedSet.nextDoc());
-      long ord = dvSortedSet.nextOrd();
-      assertEquals(SortedSetDocValues.NO_MORE_ORDS, dvSortedSet.nextOrd());
-      term = dvSortedSet.lookupOrd(ord);
-      assertEquals(expectedRef, term);
-
-      assertEquals(i, dvSortedNumeric.nextDoc());
-      assertEquals(1, dvSortedNumeric.docValueCount());
-      assertEquals(id, dvSortedNumeric.nextValue());
-    }
-
-    ScoreDoc[] hits =
-        searcher.search(new TermQuery(new Term(new String("content"), "aaa")), 1000).scoreDocs;
-
-    // First document should be #0
-    Document d = searcher.getIndexReader().document(hits[0].doc);
-    assertEquals("didn't get the right document first", "0", d.get("id"));
-
-    doTestHits(hits, 34, searcher.getIndexReader());
-
-    hits = searcher.search(new TermQuery(new Term(new String("content5"), "aaa")), 1000).scoreDocs;
-
-    doTestHits(hits, 34, searcher.getIndexReader());
-
-    hits = searcher.search(new TermQuery(new Term(new String("content6"), "aaa")), 1000).scoreDocs;
-
-    doTestHits(hits, 34, searcher.getIndexReader());
-
-    hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), 1000).scoreDocs;
-    assertEquals(34, hits.length);
-    hits =
-        searcher.search(
-                new TermQuery(new Term(new String("utf8"), "lu\uD834\uDD1Ece\uD834\uDD60ne")), 1000)
-            .scoreDocs;
-    assertEquals(34, hits.length);
-    hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), 1000).scoreDocs;
-    assertEquals(34, hits.length);
-
-    doTestHits(
-        searcher.search(IntPoint.newRangeQuery("intPoint1d", 0, 34), 1000).scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(
-                IntPoint.newRangeQuery("intPoint2d", new int[] {0, 0}, new int[] {34, 68}), 1000)
-            .scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(FloatPoint.newRangeQuery("floatPoint1d", 0f, 34f), 1000).scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(
-                FloatPoint.newRangeQuery(
-                    "floatPoint2d", new float[] {0f, 0f}, new float[] {34f, 68f}),
-                1000)
-            .scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(LongPoint.newRangeQuery("longPoint1d", 0, 34), 1000).scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(
-                LongPoint.newRangeQuery("longPoint2d", new long[] {0, 0}, new long[] {34, 68}),
-                1000)
-            .scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(DoublePoint.newRangeQuery("doublePoint1d", 0.0, 34.0), 1000).scoreDocs,
-        34,
-        searcher.getIndexReader());
-    doTestHits(
-        searcher.search(
-                DoublePoint.newRangeQuery(
-                    "doublePoint2d", new double[] {0.0, 0.0}, new double[] {34.0, 68.0}),
-                1000)
-            .scoreDocs,
-        34,
-        searcher.getIndexReader());
-
-    byte[] bytes1 = new byte[4];
-    byte[] bytes2 = new byte[] {0, 0, 0, (byte) 34};
-    doTestHits(
-        searcher.search(BinaryPoint.newRangeQuery("binaryPoint1d", bytes1, bytes2), 1000).scoreDocs,
-        34,
-        searcher.getIndexReader());
-    byte[] bytes3 = new byte[] {0, 0, 0, (byte) 68};
-    doTestHits(
-        searcher.search(
-                BinaryPoint.newRangeQuery(
-                    "binaryPoint2d", new byte[][] {bytes1, bytes1}, new byte[][] {bytes2, bytes3}),
-                1000)
-            .scoreDocs,
-        34,
-        searcher.getIndexReader());
-
-    // test vector values and KNN search
-    if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {
-      // test vector values
-      int cnt = 0;
-      for (LeafReaderContext ctx : reader.leaves()) {
-        VectorValues values = ctx.reader().getVectorValues(KNN_VECTOR_FIELD);
-        if (values != null) {
-          assertEquals(KNN_VECTOR_FIELD_TYPE.vectorDimension(), values.dimension());
-          for (int doc = values.nextDoc(); doc != NO_MORE_DOCS; doc = values.nextDoc()) {
-            float[] expectedVector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * cnt};
-            assertArrayEquals(
-                "vectors do not match for doc=" + cnt, expectedVector, values.vectorValue(), 0);
-            cnt++;
-          }
-        }
-      }
-      assertEquals(DOCS_COUNT, cnt);
-
-      // test KNN search
-      ScoreDoc[] scoreDocs = assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");
-      for (int i = 0; i < scoreDocs.length; i++) {
-        int id = Integer.parseInt(reader.document(scoreDocs[i].doc).get("id"));
-        int expectedId = i < DELETED_ID ? i : i + 1;
-        assertEquals(expectedId, id);
-      }
-    }
-
-    reader.close();
-  }
-
-  private static ScoreDoc[] assertKNNSearch(
-      IndexSearcher searcher,
-      float[] queryVector,
-      int k,
-      int expectedHitsCount,
-      String expectedFirstDocId)
-      throws IOException {
-    ScoreDoc[] hits =
-        searcher.search(new KnnVectorQuery(KNN_VECTOR_FIELD, queryVector, k), k).scoreDocs;
-    assertEquals("wrong number of hits", expectedHitsCount, hits.length);
-    Document d = searcher.doc(hits[0].doc);
-    assertEquals("wrong first document", expectedFirstDocId, d.get("id"));
-    return hits;
-  }
-
-  public void changeIndexWithAdds(Random random, Directory dir, Version nameVersion)
-      throws IOException {
-    SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
-    assertEquals(nameVersion, infos.getCommitLuceneVersion());
-    assertEquals(nameVersion, infos.getMinSegmentLuceneVersion());
-
-    // open writer
-    IndexWriter writer =
-        new IndexWriter(
-            dir,
-            newIndexWriterConfig(new MockAnalyzer(random))
-                .setOpenMode(OpenMode.APPEND)
-                .setMergePolicy(newLogMergePolicy()));
-    // add 10 docs
-    for (int i = 0; i < 10; i++) {
-      addDoc(writer, DOCS_COUNT + i);
-    }
-
-    // make sure writer sees right total -- writer seems not to know about deletes in .del?
-    final int expected = 45;
-    assertEquals("wrong doc count", expected, writer.getDocStats().numDocs);
-    writer.close();
-
-    // make sure searching sees right # hits for term search
-    IndexReader reader = DirectoryReader.open(dir);
-    IndexSearcher searcher = newSearcher(reader);
-    ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;
-    Document d = searcher.getIndexReader().document(hits[0].doc);
-    assertEquals("wrong first document", "0", d.get("id"));
-    doTestHits(hits, 44, searcher.getIndexReader());
-
-    if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {
-      // make sure KNN search sees all hits (graph may not be used if k is big)
-      assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");
-      // make sure KNN search using HNSW graph sees newly added docs
-      assertKNNSearch(
-          searcher,
-          new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},
-          10,
-          10,
-          "44");
-    }
-    reader.close();
-
-    // fully merge
-    writer =
-        new IndexWriter(
-            dir,
-            newIndexWriterConfig(new MockAnalyzer(random))
-                .setOpenMode(OpenMode.APPEND)
-                .setMergePolicy(newLogMergePolicy()));
-    writer.forceMerge(1);
-    writer.close();
-
-    reader = DirectoryReader.open(dir);
-    searcher = newSearcher(reader);
-    // make sure searching sees right # hits fot term search
-    hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;
-    assertEquals("wrong number of hits", 44, hits.length);
-    d = searcher.doc(hits[0].doc);
-    doTestHits(hits, 44, searcher.getIndexReader());
-    assertEquals("wrong first document", "0", d.get("id"));
-
-    if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {
-      // make sure KNN search sees all hits
-      assertKNNSearch(searcher, KNN_VECTOR, 1000, 44, "0");
-      // make sure KNN search using HNSW graph sees newly added docs
-      assertKNNSearch(
-          searcher,
-          new float[] {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * 44},
-          10,
-          10,
-          "44");
-    }
-    reader.close();
-  }
-
-  public void changeIndexNoAdds(Random random, Directory dir, Version nameVersion)
-      throws IOException {
-    // make sure searching sees right # hits for term search
-    DirectoryReader reader = DirectoryReader.open(dir);
-    IndexSearcher searcher = newSearcher(reader);
-    ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;
-    assertEquals("wrong number of hits", 34, hits.length);
-    Document d = searcher.doc(hits[0].doc);
-    assertEquals("wrong first document", "0", d.get("id"));
-
-    if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {
-      // make sure KNN search sees all hits
-      assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");
-      // make sure KNN search using HNSW graph retrieves correct results
-      assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");
-    }
-    reader.close();
-
-    // fully merge
-    IndexWriter writer =
-        new IndexWriter(
-            dir, newIndexWriterConfig(new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
-    writer.forceMerge(1);
-    writer.close();
-
-    reader = DirectoryReader.open(dir);
-    searcher = newSearcher(reader);
-    // make sure searching sees right # hits fot term search
-    hits = searcher.search(new TermQuery(new Term("content", "aaa")), 1000).scoreDocs;
-    assertEquals("wrong number of hits", 34, hits.length);
-    doTestHits(hits, 34, searcher.getIndexReader());
-    // make sure searching sees right # hits for KNN search
-    if (nameVersion.major >= KNN_VECTOR_MIN_SUPPORTED_VERSION) {
-      // make sure KNN search sees all hits
-      assertKNNSearch(searcher, KNN_VECTOR, 1000, 34, "0");
-      // make sure KNN search using HNSW graph retrieves correct results
-      assertKNNSearch(searcher, KNN_VECTOR, 10, 10, "0");
-    }
-    reader.close();
-  }
-
-  public void createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException {
-    Path indexDir = getIndexDir().resolve(dirName);
-    Files.deleteIfExists(indexDir);
-    Directory dir = newFSDirectory(indexDir);
-    LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
-    mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);
-    mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
-    // TODO: remove randomness
-    IndexWriterConfig conf =
-        new IndexWriterConfig(new MockAnalyzer(random()))
-            .setMaxBufferedDocs(10)
-            .setMergePolicy(NoMergePolicy.INSTANCE);
-    IndexWriter writer = new IndexWriter(dir, conf);
-
-    for (int i = 0; i < DOCS_COUNT; i++) {
-      addDoc(writer, i);
-    }
-    assertEquals("wrong doc count", DOCS_COUNT, writer.getDocStats().maxDoc);
-    if (fullyMerged) {
-      writer.forceMerge(1);
-    }
-    writer.close();
-
-    if (!fullyMerged) {
-      // open fresh writer so we get no prx file in the added segment
-      mp = new LogByteSizeMergePolicy();
-      mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);
-      // TODO: remove randomness
-      conf =
-          new IndexWriterConfig(new MockAnalyzer(random()))
-              .setMaxBufferedDocs(10)
-              .setMergePolicy(NoMergePolicy.INSTANCE);
-      writer = new IndexWriter(dir, conf);
-      addNoProxDoc(writer);
-      writer.close();
-
-      conf =
-          new IndexWriterConfig(new MockAnalyzer(random()))
-              .setMaxBufferedDocs(10)
-              .setMergePolicy(NoMergePolicy.INSTANCE);
-      writer = new IndexWriter(dir, conf);
-      Term searchTerm = new Term("id", String.valueOf(DELETED_ID));
-      writer.deleteDocuments(searchTerm);
-      writer.close();
-    }
-
-    dir.close();
-  }
-
-  private void addDoc(IndexWriter writer, int id) throws IOException {
-    Document doc = new Document();
-    doc.add(new TextField("content", "aaa", Field.Store.NO));
-    doc.add(new StringField("id", Integer.toString(id), Field.Store.YES));
-    FieldType customType2 = new FieldType(TextField.TYPE_STORED);
-    customType2.setStoreTermVectors(true);
-    customType2.setStoreTermVectorPositions(true);
-    customType2.setStoreTermVectorOffsets(true);
-    doc.add(
-        new Field(
-            "autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
-    doc.add(
-        new Field(
-            "utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", customType2));
-    doc.add(new Field("content2", "here is more content with aaa aaa aaa", customType2));
-    doc.add(new Field("fie\u2C77ld", "field with non-ascii name", customType2));
-
-    // add docvalues fields
-    doc.add(new NumericDocValuesField("dvByte", (byte) id));
-    byte[] bytes =
-        new byte[] {(byte) (id >>> 24), (byte) (id >>> 16), (byte) (id >>> 8), (byte) id};
-    BytesRef ref = new BytesRef(bytes);
-    doc.add(new BinaryDocValuesField("dvBytesDerefFixed", ref));
-    doc.add(new BinaryDocValuesField("dvBytesDerefVar", ref));
-    doc.add(new SortedDocValuesField("dvBytesSortedFixed", ref));
-    doc.add(new SortedDocValuesField("dvBytesSortedVar", ref));
-    doc.add(new BinaryDocValuesField("dvBytesStraightFixed", ref));
-    doc.add(new BinaryDocValuesField("dvBytesStraightVar", ref));
-    doc.add(new DoubleDocValuesField("dvDouble", (double) id));
-    doc.add(new FloatDocValuesField("dvFloat", (float) id));
-    doc.add(new NumericDocValuesField("dvInt", id));
-    doc.add(new NumericDocValuesField("dvLong", id));
-    doc.add(new NumericDocValuesField("dvPacked", id));
-    doc.add(new NumericDocValuesField("dvShort", (short) id));
-    doc.add(new SortedSetDocValuesField("dvSortedSet", ref));
-    doc.add(new SortedNumericDocValuesField("dvSortedNumeric", id));
-
-    doc.add(new IntPoint("intPoint1d", id));
-    doc.add(new IntPoint("intPoint2d", id, 2 * id));
-    doc.add(new FloatPoint("floatPoint1d", (float) id));
-    doc.add(new FloatPoint("floatPoint2d", (float) id, (float) 2 * id));
-    doc.add(new LongPoint("longPoint1d", id));
-    doc.add(new LongPoint("longPoint2d", id, 2 * id));
-    doc.add(new DoublePoint("doublePoint1d", (double) id));
-    doc.add(new DoublePoint("doublePoint2d", (double) id, (double) 2 * id));
-    doc.add(new BinaryPoint("binaryPoint1d", bytes));
-    doc.add(new BinaryPoint("binaryPoint2d", bytes, bytes));
-
-    // a field with both offsets and term vectors for a cross-check
-    FieldType customType3 = new FieldType(TextField.TYPE_STORED);
-    customType3.setStoreTermVectors(true);
-    customType3.setStoreTermVectorPositions(true);
-    customType3.setStoreTermVectorOffsets(true);
-    customType3.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
-    doc.add(new Field("content5", "here is more content with aaa aaa aaa", customType3));
-    // a field that omits only positions
-    FieldType customType4 = new FieldType(TextField.TYPE_STORED);
-    customType4.setStoreTermVectors(true);
-    customType4.setStoreTermVectorPositions(false);
-    customType4.setStoreTermVectorOffsets(true);
-    customType4.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
-    doc.add(new Field("content6", "here is more content with aaa aaa aaa", customType4));
-
-    float[] vector = {KNN_VECTOR[0], KNN_VECTOR[1], KNN_VECTOR[2] + 0.1f * id};
-    doc.add(new KnnVectorField(KNN_VECTOR_FIELD, vector, KNN_VECTOR_FIELD_TYPE));
-
-    // TODO:
-    //   index different norms types via similarity (we use a random one currently?!)
-    //   remove any analyzer randomness, explicitly add payloads for certain fields.
-    writer.addDocument(doc);
-  }
-
-  private void addNoProxDoc(IndexWriter writer) throws IOException {
-    Document doc = new Document();
-    FieldType customType = new FieldType(TextField.TYPE_STORED);
-    customType.setIndexOptions(IndexOptions.DOCS);
-    Field f = new Field("content3", "aaa", customType);
-    doc.add(f);
-    FieldType customType2 = new FieldType();
-    customType2.setStored(true);
-    customType2.setIndexOptions(IndexOptions.DOCS);
-    f = new Field("content4", "aaa", customType2);
-    doc.add(f);
-    writer.addDocument(doc);
-  }
-
-  private int countDocs(PostingsEnum docs) throws IOException {
-    int count = 0;
-    while ((docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-      count++;
-    }
-    return count;
-  }
-
-  // flex: test basics of TermsEnum api on non-flex index
-  public void testNextIntoWrongField() throws Exception {
-    for (String name : oldNames) {
-      Directory dir = oldIndexDirs.get(name);
-      IndexReader r = DirectoryReader.open(dir);
-      TermsEnum terms = MultiTerms.getTerms(r, "content").iterator();
-      BytesRef t = terms.next();
-      assertNotNull(t);
-
-      // content field only has term aaa:
-      assertEquals("aaa", t.utf8ToString());
-      assertNull(terms.next());
-
-      BytesRef aaaTerm = new BytesRef("aaa");
-
-      // should be found exactly
-      assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));
-      assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));
-      assertNull(terms.next());
-
-      // should hit end of field
-      assertEquals(TermsEnum.SeekStatus.END, terms.seekCeil(new BytesRef("bbb")));
-      assertNull(terms.next());
-
-      // should seek to aaa
-      assertEquals(TermsEnum.SeekStatus.NOT_FOUND, terms.seekCeil(new BytesRef("a")));
-      assertTrue(terms.term().bytesEquals(aaaTerm));
-      assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));
-      assertNull(terms.next());
-
-      assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(aaaTerm));
-      assertEquals(DOCS_COUNT, countDocs(TestUtil.docs(random(), terms, null, PostingsEnum.NONE)));
-      assertNull(terms.next());
-
-      r.close();
-    }
-  }
-
-  /**
-   * Test that we didn't forget to bump the current Constants.LUCENE_MAIN_VERSION. This is important
-   * so that we can determine which version of lucene wrote the segment.
-   */
-  public void testOldVersions() throws Exception {
-    // first create a little index with the current code and get the version
-    Directory currentDir = newDirectory();
-    RandomIndexWriter riw = new RandomIndexWriter(random(), currentDir);
-    riw.addDocument(new Document());
-    riw.close();
-    DirectoryReader ir = DirectoryReader.open(currentDir);
-    SegmentReader air = (SegmentReader) ir.leaves().get(0).reader();
-    Version currentVersion = air.getSegmentInfo().info.getVersion();
-    assertNotNull(currentVersion); // only 3.0 segments can have a null version
-    ir.close();
-    currentDir.close();
-
-    // now check all the old indexes, their version should be < the current version
-    for (String name : oldNames) {
-      Directory dir = oldIndexDirs.get(name);
-      DirectoryReader r = DirectoryReader.open(dir);
-      for (LeafReaderContext context : r.leaves()) {
-        air = (SegmentReader) context.reader();
-        Version oldVersion = air.getSegmentInfo().info.getVersion();
-        assertNotNull(oldVersion); // only 3.0 segments can have a null version
-        assertTrue(
-            "current Version.LATEST is <= an old index: did you forget to bump it?!",
-            currentVersion.onOrAfter(oldVersion));
-      }
-      r.close();
-    }
-  }
-
-  public void testIndexCreatedVersion() throws IOException {
-    for (String name : oldNames) {
-      Directory dir = oldIndexDirs.get(name);
-      SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
-      // those indexes are created by a single version so we can
-      // compare the commit version with the created version
-      assertEquals(infos.getCommitLuceneVersion().major, infos.getIndexCreatedVersionMajor());
-    }
-  }
-
-  public void testSegmentCommitInfoId() throws IOException {
-    for (String name : oldNames) {
-      Directory dir = oldIndexDirs.get(name);
-      SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
-      for (SegmentCommitInfo info : infos) {
-        if (info.info.getVersion().onOrAfter(Version.fromBits(8, 6, 0))) {
-          assertNotNull(info.toString(), info.getId());
-        } else {
-          assertNull(info.toString(), info.getId());
-        }
-      }
-    }
-  }
-
-  public void verifyUsesDefaultCodec(Directory dir, String name) throws Exception {
-    DirectoryReader r = DirectoryReader.open(dir);
-    for (LeafReaderContext context : r.leaves()) {
-      SegmentReader air = (SegmentReader) context.reader();
-      Codec codec = air.getSegmentInfo().info.getCodec();
-      assertTrue(
-          "codec used in "
-              + name
-              + " ("
-              + codec.getName()
-              + ") is not a default codec (does not begin with Lucene)",
-          codec.getName().startsWith("Lucene"));
-    }
-    r.close();
-  }
-
-  public void testAllIndexesUseDefaultCodec() throws Exception {
-    for (String name : oldNames) {
-      Directory dir = oldIndexDirs.get(name);
-      verifyUsesDefaultCodec(dir, name);
-    }
-  }
-
-  private int checkAllSegmentsUpgraded(Directory dir, int indexCreatedVersion) throws IOException {
-    final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
-    if (VERBOSE) {
-      System.out.println("checkAllSegmentsUpgraded: " + infos);
-    }
-    for (SegmentCommitInfo si : infos) {
-      assertEquals(Version.LATEST, si.info.getVersion());
-      assertNotNull(si.getId());
-    }
-    assertEquals(Version.LATEST, infos.getCommitLuceneVersion());
-    assertEquals(indexCreatedVersion, infos.getIndexCreatedVersionMajor());
-    return infos.size();
-  }
-
-  private int getNumberOfSegments(Directory dir) throws IOException {
-    final SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
-    return infos.size();
-  }
-
-  public void testUpgradeOldIndex() throws Exception {
-    List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
-    names.addAll(Arrays.asList(oldNames));
-    names.addAll(Arrays.asList(oldSingleSegmentNames));
-    for (String name : names) {
-      if (VERBOSE) {
-        System.out.println("testUpgradeOldIndex: index=" + name);
-      }
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-      int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
-
-      newIndexUpgrader(dir).upgrade();
-
-      checkAllSegmentsUpgraded(dir, indexCreatedVersion);
-
-      dir.close();
-    }
-  }
-
-  public void testIndexUpgraderCommandLineArgs() throws Exception {
-
-    PrintStream savedSystemOut = System.out;
-    System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
-    try {
-      for (Map.Entry<String, Directory> entry : oldIndexDirs.entrySet()) {
-        String name = entry.getKey();
-        Directory origDir = entry.getValue();
-        int indexCreatedVersion =
-            SegmentInfos.readLatestCommit(origDir).getIndexCreatedVersionMajor();
-        Path dir = createTempDir(name);
-        try (FSDirectory fsDir = FSDirectory.open(dir)) {
-          // beware that ExtraFS might add extraXXX files
-          Set<String> extraFiles = Set.of(fsDir.listAll());
-          for (String file : origDir.listAll()) {
-            if (extraFiles.contains(file) == false) {
-              fsDir.copyFrom(origDir, file, file, IOContext.DEFAULT);
-            }
-          }
-        }
-
-        String path = dir.toAbsolutePath().toString();
-
-        List<String> args = new ArrayList<>();
-        if (random().nextBoolean()) {
-          args.add("-verbose");
-        }
-        if (random().nextBoolean()) {
-          args.add("-delete-prior-commits");
-        }
-        if (random().nextBoolean()) {
-          // TODO: need to better randomize this, but ...
-          //  - LuceneTestCase.FS_DIRECTORIES is private
-          //  - newFSDirectory returns BaseDirectoryWrapper
-          //  - BaseDirectoryWrapper doesn't expose delegate
-          Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;
-
-          args.add("-dir-impl");
-          args.add(dirImpl.getName());
-        }
-        args.add(path);
-
-        IndexUpgrader.main(args.toArray(new String[0]));
-
-        Directory upgradedDir = newFSDirectory(dir);
-        try {
-          checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);
-        } finally {
-          upgradedDir.close();
-        }
-      }
-    } finally {
-      System.setOut(savedSystemOut);
-    }
-  }
-
-  public void testUpgradeOldSingleSegmentIndexWithAdditions() throws Exception {
-    for (String name : oldSingleSegmentNames) {
-      if (VERBOSE) {
-        System.out.println("testUpgradeOldSingleSegmentIndexWithAdditions: index=" + name);
-      }
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-      assertEquals("Original index must be single segment", 1, getNumberOfSegments(dir));
-      int indexCreatedVersion = SegmentInfos.readLatestCommit(dir).getIndexCreatedVersionMajor();
-
-      // create a bunch of dummy segments
-      int id = 40;
-      Directory ramDir = new ByteBuffersDirectory();
-      for (int i = 0; i < 3; i++) {
-        // only use Log- or TieredMergePolicy, to make document addition predictable and not
-        // suddenly merge:
-        MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
-        IndexWriterConfig iwc =
-            new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(mp);
-        IndexWriter w = new IndexWriter(ramDir, iwc);
-        // add few more docs:
-        for (int j = 0; j < RANDOM_MULTIPLIER * random().nextInt(30); j++) {
-          addDoc(w, id++);
-        }
-        try {
-          w.commit();
-        } finally {
-          w.close();
-        }
-      }
-
-      // add dummy segments (which are all in current
-      // version) to single segment index
-      MergePolicy mp = random().nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
-      IndexWriterConfig iwc = new IndexWriterConfig(null).setMergePolicy(mp);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      w.addIndexes(ramDir);
-      try {
-        w.commit();
-      } finally {
-        w.close();
-      }
-
-      // determine count of segments in modified index
-      final int origSegCount = getNumberOfSegments(dir);
-
-      // ensure there is only one commit
-      assertEquals(1, DirectoryReader.listCommits(dir).size());
-      newIndexUpgrader(dir).upgrade();
-
-      final int segCount = checkAllSegmentsUpgraded(dir, indexCreatedVersion);
-      assertEquals(
-          "Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged",
-          origSegCount,
-          segCount);
-
-      dir.close();
-    }
-  }
-
-  public static final String emptyIndex = "empty.9.0.0.zip";
-
-  public void testUpgradeEmptyOldIndex() throws Exception {
-    Path oldIndexDir = createTempDir("emptyIndex");
-    TestUtil.unzip(getDataInputStream(emptyIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-
-    newIndexUpgrader(dir).upgrade();
-
-    checkAllSegmentsUpgraded(dir, 9);
-
-    dir.close();
-  }
-
-  public static final String moreTermsIndex = "moreterms.9.0.0.zip";
-
-  public void testMoreTerms() throws Exception {
-    Path oldIndexDir = createTempDir("moreterms");
-    TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-    DirectoryReader reader = DirectoryReader.open(dir);
-
-    verifyUsesDefaultCodec(dir, moreTermsIndex);
-    TestUtil.checkIndex(dir);
-    searchExampleIndex(reader);
-
-    reader.close();
-    dir.close();
-  }
-
-  public static final String dvUpdatesIndex = "dvupdates.9.0.0.zip";
-
-  private void assertNumericDocValues(LeafReader r, String f, String cf) throws IOException {
-    NumericDocValues ndvf = r.getNumericDocValues(f);
-    NumericDocValues ndvcf = r.getNumericDocValues(cf);
-    for (int i = 0; i < r.maxDoc(); i++) {
-      assertEquals(i, ndvcf.nextDoc());
-      assertEquals(i, ndvf.nextDoc());
-      assertEquals(ndvcf.longValue(), ndvf.longValue() * 2);
-    }
-  }
-
-  private void assertBinaryDocValues(LeafReader r, String f, String cf) throws IOException {
-    BinaryDocValues bdvf = r.getBinaryDocValues(f);
-    BinaryDocValues bdvcf = r.getBinaryDocValues(cf);
-    for (int i = 0; i < r.maxDoc(); i++) {
-      assertEquals(i, bdvf.nextDoc());
-      assertEquals(i, bdvcf.nextDoc());
-      assertEquals(getValue(bdvcf), getValue(bdvf) * 2);
-    }
-  }
-
-  private void verifyDocValues(Directory dir) throws IOException {
-    DirectoryReader reader = DirectoryReader.open(dir);
-    for (LeafReaderContext context : reader.leaves()) {
-      LeafReader r = context.reader();
-      assertNumericDocValues(r, "ndv1", "ndv1_c");
-      assertNumericDocValues(r, "ndv2", "ndv2_c");
-      assertBinaryDocValues(r, "bdv1", "bdv1_c");
-      assertBinaryDocValues(r, "bdv2", "bdv2_c");
-    }
-    reader.close();
-  }
-
-  public void testDocValuesUpdates() throws Exception {
-    Path oldIndexDir = createTempDir("dvupdates");
-    TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-    verifyUsesDefaultCodec(dir, dvUpdatesIndex);
-
-    verifyDocValues(dir);
-
-    // update fields and verify index
-    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
-    IndexWriter writer = new IndexWriter(dir, conf);
-    updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);
-    updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);
-    updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);
-    updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);
-
-    writer.commit();
-    verifyDocValues(dir);
-
-    // merge all segments
-    writer.forceMerge(1);
-    writer.commit();
-    verifyDocValues(dir);
-
-    writer.close();
-    dir.close();
-  }
-
-  public void testDeletes() throws Exception {
-    Path oldIndexDir = createTempDir("dvupdates");
-    TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-    verifyUsesDefaultCodec(dir, dvUpdatesIndex);
-
-    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
-    IndexWriter writer = new IndexWriter(dir, conf);
-
-    int maxDoc = writer.getDocStats().maxDoc;
-    writer.deleteDocuments(new Term("id", "1"));
-    if (random().nextBoolean()) {
-      writer.commit();
-    }
-
-    writer.forceMerge(1);
-    writer.commit();
-    assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);
-
-    writer.close();
-    dir.close();
-  }
-
-  public void testSoftDeletes() throws Exception {
-    Path oldIndexDir = createTempDir("dvupdates");
-    TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-    verifyUsesDefaultCodec(dir, dvUpdatesIndex);
-    IndexWriterConfig conf =
-        new IndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("__soft_delete");
-    IndexWriter writer = new IndexWriter(dir, conf);
-    int maxDoc = writer.getDocStats().maxDoc;
-    writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("__soft_delete", 1));
-
-    if (random().nextBoolean()) {
-      writer.commit();
-    }
-    writer.forceMerge(1);
-    writer.commit();
-    assertEquals(maxDoc - 1, writer.getDocStats().maxDoc);
-    writer.close();
-    dir.close();
-  }
-
-  public void testDocValuesUpdatesWithNewField() throws Exception {
-    Path oldIndexDir = createTempDir("dvupdates");
-    TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
-    Directory dir = newFSDirectory(oldIndexDir);
-    verifyUsesDefaultCodec(dir, dvUpdatesIndex);
-
-    // update fields and verify index
-    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
-    IndexWriter writer = new IndexWriter(dir, conf);
-    // introduce a new field that we later update
-    writer.addDocument(
-        Arrays.asList(
-            new StringField("id", "" + Integer.MAX_VALUE, Field.Store.NO),
-            new NumericDocValuesField("new_numeric", 1),
-            new BinaryDocValuesField("new_binary", toBytes(1))));
-    writer.updateNumericDocValue(new Term("id", "1"), "new_numeric", 1);
-    writer.updateBinaryDocValue(new Term("id", "1"), "new_binary", toBytes(1));
-
-    writer.commit();
-    Runnable assertDV =
-        () -> {
-          boolean found = false;
-          try (DirectoryReader reader = DirectoryReader.open(dir)) {
-            for (LeafReaderContext ctx : reader.leaves()) {
-              LeafReader leafReader = ctx.reader();
-              TermsEnum id = leafReader.terms("id").iterator();
-              if (id.seekExact(new BytesRef("1"))) {
-                PostingsEnum postings = id.postings(null, PostingsEnum.NONE);
-                NumericDocValues numericDocValues = leafReader.getNumericDocValues("new_numeric");
-                BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("new_binary");
-                int doc;
-                while ((doc = postings.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-                  found = true;
-                  assertTrue(binaryDocValues.advanceExact(doc));
-                  assertTrue(numericDocValues.advanceExact(doc));
-                  assertEquals(1, numericDocValues.longValue());
-                  assertEquals(toBytes(1), binaryDocValues.binaryValue());
-                }
-              }
-            }
-          } catch (IOException e) {
-            throw new AssertionError(e);
-          }
-          assertTrue(found);
-        };
-    assertDV.run();
-    // merge all segments
-    writer.forceMerge(1);
-    writer.commit();
-    assertDV.run();
-    writer.close();
-    dir.close();
-  }
-
-  // LUCENE-5907
-  public void testUpgradeWithNRTReader() throws Exception {
-    for (String name : oldNames) {
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-
-      IndexWriter writer =
-          new IndexWriter(
-              dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
-      writer.addDocument(new Document());
-      DirectoryReader r = DirectoryReader.open(writer);
-      writer.commit();
-      r.close();
-      writer.forceMerge(1);
-      writer.commit();
-      writer.rollback();
-      SegmentInfos.readLatestCommit(dir);
-      dir.close();
-    }
-  }
-
-  // LUCENE-5907
-  public void testUpgradeThenMultipleCommits() throws Exception {
-    for (String name : oldNames) {
-      Directory dir = newDirectory(oldIndexDirs.get(name));
-
-      IndexWriter writer =
-          new IndexWriter(
-              dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
-      writer.addDocument(new Document());
-      writer.commit();
-      writer.addDocument(new Document());
-      writer.commit();
-      writer.close();
-      dir.close();
-    }
-  }
-
-  public void testSortedIndex() throws Exception {
-    for (String name : oldSortedNames) {
-      Path path = createTempDir("sorted");
-      InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");
-      assertNotNull("Sorted index index " + name + " not found", resource);
-      TestUtil.unzip(resource, path);
-
-      Directory dir = newFSDirectory(path);
-      DirectoryReader reader = DirectoryReader.open(dir);
-
-      assertEquals(1, reader.leaves().size());
-      Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
-      assertNotNull(sort);
-      assertEquals("<long: \"dateDV\">!", sort.toString());
-
-      // This will confirm the docs are really sorted
-      TestUtil.checkIndex(dir);
-
-      searchExampleIndex(reader);
-
-      reader.close();
-      dir.close();
-    }
-  }
-
-  private void searchExampleIndex(DirectoryReader reader) throws IOException {
-    IndexSearcher searcher = newSearcher(reader);
-
-    TopDocs topDocs = searcher.search(new FieldExistsQuery("titleTokenized"), 10);
-    assertEquals(50, topDocs.totalHits.value);
-
-    topDocs = searcher.search(new FieldExistsQuery("titleDV"), 10);
-    assertEquals(50, topDocs.totalHits.value);
-
-    topDocs = searcher.search(new TermQuery(new Term("body", "ja")), 10);
-    assertTrue(topDocs.totalHits.value > 0);
-
-    topDocs =
-        searcher.search(
-            IntPoint.newRangeQuery("docid_int", 42, 44),
-            10,
-            new Sort(new SortField("docid_intDV", SortField.Type.INT)));
-    assertEquals(3, topDocs.totalHits.value);
-    assertEquals(3, topDocs.scoreDocs.length);
-    assertEquals(42, ((FieldDoc) topDocs.scoreDocs[0]).fields[0]);
-    assertEquals(43, ((FieldDoc) topDocs.scoreDocs[1]).fields[0]);
-    assertEquals(44, ((FieldDoc) topDocs.scoreDocs[2]).fields[0]);
-
-    topDocs = searcher.search(new TermQuery(new Term("body", "the")), 5);
-    assertTrue(topDocs.totalHits.value > 0);
-
-    topDocs =
-        searcher.search(
-            new MatchAllDocsQuery(), 5, new Sort(new SortField("dateDV", SortField.Type.LONG)));
-    assertEquals(50, topDocs.totalHits.value);
-    assertEquals(5, topDocs.scoreDocs.length);
-    long firstDate = (Long) ((FieldDoc) topDocs.scoreDocs[0]).fields[0];
-    long lastDate = (Long) ((FieldDoc) topDocs.scoreDocs[4]).fields[0];
-    assertTrue(firstDate <= lastDate);
-  }
-
-  static long getValue(BinaryDocValues bdv) throws IOException {
-    BytesRef term = bdv.binaryValue();
-    int idx = term.offset;
-    byte b = term.bytes[idx++];
-    long value = b & 0x7FL;
-    for (int shift = 7; (b & 0x80L) != 0; shift += 7) {
-      b = term.bytes[idx++];
-      value |= (b & 0x7FL) << shift;
-    }
-    return value;
-  }
-
-  // encodes a long into a BytesRef as VLong so that we get varying number of bytes when we update
-  static BytesRef toBytes(long value) {
-    BytesRef bytes = new BytesRef(10); // negative longs may take 10 bytes
-    while ((value & ~0x7FL) != 0L) {
-      bytes.bytes[bytes.length++] = (byte) ((value & 0x7FL) | 0x80L);
-      value >>>= 7;
-    }
-    bytes.bytes[bytes.length++] = (byte) value;
-    return bytes;
-  }
-
-  public void testFailOpenOldIndex() throws IOException {
-    for (String name : oldNames) {
-      Directory directory = oldIndexDirs.get(name);
-      IndexCommit commit = DirectoryReader.listCommits(directory).get(0);
-      IndexFormatTooOldException ex =
-          expectThrows(
-              IndexFormatTooOldException.class,
-              () -> StandardDirectoryReader.open(commit, Version.LATEST.major, null));
-      assertTrue(
-          ex.getMessage()
-              .contains(
-                  "only supports reading from version " + Version.LATEST.major + " upwards."));
-      // now open with allowed min version
-      StandardDirectoryReader.open(commit, Version.MIN_SUPPORTED_MAJOR, null).close();
-    }
-  }
-
-  @Nightly
-  public void testReadNMinusTwoCommit() throws IOException {
-    for (String name : binarySupportedNames) {
-      Path oldIndexDir = createTempDir(name);
-      TestUtil.unzip(getDataInputStream("unsupported." + name + ".zip"), oldIndexDir);
-      try (BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir)) {
-        IndexCommit commit = DirectoryReader.listCommits(dir).get(0);
-        StandardDirectoryReader.open(commit, MIN_BINARY_SUPPORTED_MAJOR, null).close();
-      }
-    }
-  }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
index 22da100fb47..295c844c9ae 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/IndexedDISI.java
@@ -61,7 +61,8 @@ import org.apache.lucene.util.RoaringDocIdSet;
  * <p>Each int-pair entry consists of 2 logical parts:
  *
  * <p>The first 32 bit int holds the index (number of set bits in the blocks) up to just before the
- * wanted block. The maximum number of set bits is the maximum number of documents, which is < 2^31.
+ * wanted block. The maximum number of set bits is the maximum number of documents, which is less
+ * than 2^31.
  *
  * <p>The next int holds the offset in bytes into the underlying slice. As there is a maximum of
  * 2^16 blocks, it follows that the maximum size of any block must not exceed 2^15 bytes to avoid
@@ -89,7 +90,7 @@ import org.apache.lucene.util.RoaringDocIdSet;
  *
  * @lucene.internal
  */
-final class IndexedDISI extends DocIdSetIterator {
+public final class IndexedDISI extends DocIdSetIterator {
 
   // jump-table time/space trade-offs to consider:
   // The block offsets and the block indexes could be stored in more compressed form with
@@ -185,7 +186,7 @@ final class IndexedDISI extends DocIdSetIterator {
    * @return the number of jump-table entries following the blocks, -1 for no entries. This should
    *     be stored in meta and used when creating an instance of IndexedDISI.
    */
-  static short writeBitSet(DocIdSetIterator it, IndexOutput out, byte denseRankPower)
+  public static short writeBitSet(DocIdSetIterator it, IndexOutput out, byte denseRankPower)
       throws IOException {
     final long origo = out.getFilePointer(); // All jumps are relative to the origo
     if ((denseRankPower < 7 || denseRankPower > 15) && denseRankPower != -1) {
@@ -309,7 +310,7 @@ final class IndexedDISI extends DocIdSetIterator {
    *     #writeBitSet(DocIdSetIterator, IndexOutput, byte)}
    * @param cost normally the number of logical docIDs.
    */
-  IndexedDISI(
+  public IndexedDISI(
       IndexInput in,
       long offset,
       long length,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsFormat.java
index 596353cfb3a..49eaf74c240 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsFormat.java
@@ -101,6 +101,8 @@ public final class Lucene91HnswVectorsFormat extends KnnVectorsFormat {
    */
   public static final int DEFAULT_BEAM_WIDTH = 100;
 
+  static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
+
   /**
    * Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
    * {@link Lucene91HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsReader.java
index 84f5006a856..1df95f980b9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsReader.java
@@ -20,19 +20,15 @@ package org.apache.lucene.codecs.lucene91;
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.function.IntUnaryOperator;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.RandomAccessVectorValues;
-import org.apache.lucene.index.RandomAccessVectorValuesProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.index.VectorValues;
@@ -43,12 +39,12 @@ import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.hnsw.HnswGraph;
 import org.apache.lucene.util.hnsw.HnswGraphSearcher;
 import org.apache.lucene.util.hnsw.NeighborQueue;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
 
 /**
  * Reads vectors from the index segments along with index data structures supporting KNN search.
@@ -189,7 +185,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     return VectorSimilarityFunction.values()[similarityFunctionId];
   }
 
-  private FieldEntry readField(DataInput input) throws IOException {
+  private FieldEntry readField(IndexInput input) throws IOException {
     VectorSimilarityFunction similarityFunction = readSimilarityFunction(input);
     return new FieldEntry(input, similarityFunction);
   }
@@ -200,9 +196,6 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     totalBytes +=
         RamUsageEstimator.sizeOfMap(
             fields, RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class));
-    for (FieldEntry entry : fields.values()) {
-      totalBytes += RamUsageEstimator.sizeOf(entry.ordToDoc);
-    }
     return totalBytes;
   }
 
@@ -215,7 +208,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
   @Override
   public VectorValues getVectorValues(String field) throws IOException {
     FieldEntry fieldEntry = fields.get(field);
-    return getOffHeapVectorValues(fieldEntry);
+    return OffHeapVectorValues.load(fieldEntry, vectorData);
   }
 
   @Override
@@ -229,7 +222,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
 
     // bound k by total number of vectors to prevent oversizing data structures
     k = Math.min(k, fieldEntry.size());
-    OffHeapVectorValues vectorValues = getOffHeapVectorValues(fieldEntry);
+    OffHeapVectorValues vectorValues = OffHeapVectorValues.load(fieldEntry, vectorData);
 
     NeighborQueue results =
         HnswGraphSearcher.search(
@@ -238,7 +231,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
             vectorValues,
             fieldEntry.similarityFunction,
             getGraph(fieldEntry),
-            getAcceptOrds(acceptDocs, fieldEntry),
+            vectorValues.getAcceptOrds(acceptDocs),
             visitedLimit);
 
     int i = 0;
@@ -247,7 +240,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
       int node = results.topNode();
       float score = fieldEntry.similarityFunction.convertToScore(results.topScore());
       results.pop();
-      scoreDocs[scoreDocs.length - ++i] = new ScoreDoc(fieldEntry.ordToDoc(node), score);
+      scoreDocs[scoreDocs.length - ++i] = new ScoreDoc(vectorValues.ordToDoc(node), score);
     }
 
     TotalHits.Relation relation =
@@ -257,33 +250,6 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     return new TopDocs(new TotalHits(results.visitedCount(), relation), scoreDocs);
   }
 
-  private OffHeapVectorValues getOffHeapVectorValues(FieldEntry fieldEntry) throws IOException {
-    IndexInput bytesSlice =
-        vectorData.slice("vector-data", fieldEntry.vectorDataOffset, fieldEntry.vectorDataLength);
-    return new OffHeapVectorValues(
-        fieldEntry.dimension, fieldEntry.size(), fieldEntry.ordToDoc, bytesSlice);
-  }
-
-  private Bits getAcceptOrds(Bits acceptDocs, FieldEntry fieldEntry) {
-    if (fieldEntry.ordToDoc == null) {
-      return acceptDocs;
-    }
-    if (acceptDocs == null) {
-      return null;
-    }
-    return new Bits() {
-      @Override
-      public boolean get(int index) {
-        return acceptDocs.get(fieldEntry.ordToDoc(index));
-      }
-
-      @Override
-      public int length() {
-        return fieldEntry.size;
-      }
-    };
-  }
-
   /** Get knn graph values; used for testing */
   public HnswGraph getGraph(String field) throws IOException {
     FieldInfo info = fieldInfos.fieldInfo(field);
@@ -309,7 +275,7 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     IOUtils.close(vectorData, vectorIndex);
   }
 
-  private static class FieldEntry {
+  static class FieldEntry {
 
     final VectorSimilarityFunction similarityFunction;
     final long vectorDataOffset;
@@ -319,14 +285,29 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     final int maxConn;
     final int numLevels;
     final int dimension;
-    private final int size;
-    final int[] ordToDoc;
-    private final IntUnaryOperator ordToDocOperator;
+    final int size;
     final int[][] nodesByLevel;
     // for each level the start offsets in vectorIndex file from where to read neighbours
     final long[] graphOffsetsByLevel;
 
-    FieldEntry(DataInput input, VectorSimilarityFunction similarityFunction) throws IOException {
+    // the following four variables used to read docIds encoded by IndexDISI
+    // special values of docsWithFieldOffset are -1 and -2
+    // -1 : dense
+    // -2 : empty
+    // other: sparse
+    final long docsWithFieldOffset;
+    final long docsWithFieldLength;
+    final short jumpTableEntryCount;
+    final byte denseRankPower;
+
+    // the following four variables used to read ordToDoc encoded by DirectMonotonicWriter
+    // note that only spare case needs to store ordToDoc
+    final long addressesOffset;
+    final int blockShift;
+    final DirectMonotonicReader.Meta meta;
+    final long addressesLength;
+
+    FieldEntry(IndexInput input, VectorSimilarityFunction similarityFunction) throws IOException {
       this.similarityFunction = similarityFunction;
       vectorDataOffset = input.readVLong();
       vectorDataLength = input.readVLong();
@@ -335,23 +316,24 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
       dimension = input.readInt();
       size = input.readInt();
 
-      int denseSparseMarker = input.readByte();
-      if (denseSparseMarker == -1) {
-        ordToDoc = null; // each document has a vector value
+      docsWithFieldOffset = input.readLong();
+      docsWithFieldLength = input.readLong();
+      jumpTableEntryCount = input.readShort();
+      denseRankPower = input.readByte();
+
+      // dense or empty
+      if (docsWithFieldOffset == -1 || docsWithFieldOffset == -2) {
+        addressesOffset = 0;
+        blockShift = 0;
+        meta = null;
+        addressesLength = 0;
       } else {
-        assert denseSparseMarker == 0;
-        // TODO: Can we read docIDs from disk directly instead of loading giant arrays in memory?
-        //  Or possibly switch to something like DirectMonotonicReader if it doesn't slow down
-        // searches.
-
-        // as not all docs have vector values, fill a mapping from dense vector ordinals to docIds
-        ordToDoc = new int[size];
-        for (int i = 0; i < size; i++) {
-          int doc = input.readInt();
-          ordToDoc[i] = doc;
-        }
+        // sparse
+        addressesOffset = input.readLong();
+        blockShift = input.readVInt();
+        meta = DirectMonotonicReader.loadMeta(input, size, blockShift);
+        addressesLength = input.readLong();
       }
-      ordToDocOperator = ordToDoc == null ? IntUnaryOperator.identity() : (ord) -> ordToDoc[ord];
 
       // read nodes by level
       maxConn = input.readInt();
@@ -388,128 +370,6 @@ public final class Lucene91HnswVectorsReader extends KnnVectorsReader {
     int size() {
       return size;
     }
-
-    int ordToDoc(int ord) {
-      return ordToDocOperator.applyAsInt(ord);
-    }
-  }
-
-  /** Read the vector values from the index input. This supports both iterated and random access. */
-  static class OffHeapVectorValues extends VectorValues
-      implements RandomAccessVectorValues, RandomAccessVectorValuesProducer {
-
-    private final int dimension;
-    private final int size;
-    private final int[] ordToDoc;
-    private final IntUnaryOperator ordToDocOperator;
-    private final IndexInput dataIn;
-    private final BytesRef binaryValue;
-    private final ByteBuffer byteBuffer;
-    private final int byteSize;
-    private final float[] value;
-
-    private int ord = -1;
-    private int doc = -1;
-
-    OffHeapVectorValues(int dimension, int size, int[] ordToDoc, IndexInput dataIn) {
-      this.dimension = dimension;
-      this.size = size;
-      this.ordToDoc = ordToDoc;
-      ordToDocOperator = ordToDoc == null ? IntUnaryOperator.identity() : (ord) -> ordToDoc[ord];
-      this.dataIn = dataIn;
-      byteSize = Float.BYTES * dimension;
-      byteBuffer = ByteBuffer.allocate(byteSize);
-      value = new float[dimension];
-      binaryValue = new BytesRef(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize);
-    }
-
-    @Override
-    public int dimension() {
-      return dimension;
-    }
-
-    @Override
-    public int size() {
-      return size;
-    }
-
-    @Override
-    public float[] vectorValue() throws IOException {
-      dataIn.seek((long) ord * byteSize);
-      dataIn.readFloats(value, 0, value.length);
-      return value;
-    }
-
-    @Override
-    public BytesRef binaryValue() throws IOException {
-      dataIn.seek((long) ord * byteSize);
-      dataIn.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize, false);
-      return binaryValue;
-    }
-
-    @Override
-    public int docID() {
-      return doc;
-    }
-
-    @Override
-    public int nextDoc() {
-      if (++ord >= size) {
-        doc = NO_MORE_DOCS;
-      } else {
-        doc = ordToDocOperator.applyAsInt(ord);
-      }
-      return doc;
-    }
-
-    @Override
-    public int advance(int target) {
-      assert docID() < target;
-
-      if (ordToDoc == null) {
-        ord = target;
-      } else {
-        ord = Arrays.binarySearch(ordToDoc, ord + 1, ordToDoc.length, target);
-        if (ord < 0) {
-          ord = -(ord + 1);
-        }
-      }
-
-      if (ord < size) {
-        doc = ordToDocOperator.applyAsInt(ord);
-      } else {
-        doc = NO_MORE_DOCS;
-      }
-      return doc;
-    }
-
-    @Override
-    public long cost() {
-      return size;
-    }
-
-    @Override
-    public RandomAccessVectorValues randomAccess() {
-      return new OffHeapVectorValues(dimension, size, ordToDoc, dataIn.clone());
-    }
-
-    @Override
-    public float[] vectorValue(int targetOrd) throws IOException {
-      dataIn.seek((long) targetOrd * byteSize);
-      dataIn.readFloats(value, 0, value.length);
-      return value;
-    }
-
-    @Override
-    public BytesRef binaryValue(int targetOrd) throws IOException {
-      readValue(targetOrd);
-      return binaryValue;
-    }
-
-    private void readValue(int targetOrd) throws IOException {
-      dataIn.seek((long) targetOrd * byteSize);
-      dataIn.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize);
-    }
   }
 
   /** Read the nearest-neighbors graph from the index input */
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsWriter.java
index 6d42a5af321..0653a8938b5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/Lucene91HnswVectorsWriter.java
@@ -17,6 +17,7 @@
 
 package org.apache.lucene.codecs.lucene91;
 
+import static org.apache.lucene.codecs.lucene91.Lucene91HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 
 import java.io.IOException;
@@ -24,6 +25,7 @@ import java.util.Arrays;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.KnnVectorsReader;
 import org.apache.lucene.codecs.KnnVectorsWriter;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
 import org.apache.lucene.index.DocsWithFieldSet;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.IndexFileNames;
@@ -40,6 +42,7 @@ import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator;
 import org.apache.lucene.util.hnsw.HnswGraphBuilder;
 import org.apache.lucene.util.hnsw.NeighborArray;
 import org.apache.lucene.util.hnsw.OnHeapHnswGraph;
+import org.apache.lucene.util.packed.DirectMonotonicWriter;
 
 /**
  * Writes vector values and knn graphs to index segments.
@@ -140,11 +143,12 @@ public final class Lucene91HnswVectorsWriter extends KnnVectorsWriter {
 
       long vectorIndexOffset = vectorIndex.getFilePointer();
       // build the graph using the temporary vector data
-      // we pass null for ordToDoc mapping, for the graph construction doesn't need to know docIds
+      // we use Lucene91HnswVectorsReader.DenseOffHeapVectorValues for the graph construction
+      // doesn't need to know docIds
       // TODO: separate random access vector values from DocIdSetIterator?
-      Lucene91HnswVectorsReader.OffHeapVectorValues offHeapVectors =
-          new Lucene91HnswVectorsReader.OffHeapVectorValues(
-              vectors.dimension(), docsWithField.cardinality(), null, vectorDataInput);
+      OffHeapVectorValues offHeapVectors =
+          new OffHeapVectorValues.DenseOffHeapVectorValues(
+              vectors.dimension(), docsWithField.cardinality(), vectorDataInput);
       OnHeapHnswGraph graph =
           offHeapVectors.size() == 0
               ? null
@@ -207,14 +211,41 @@ public final class Lucene91HnswVectorsWriter extends KnnVectorsWriter {
     // write docIDs
     int count = docsWithField.cardinality();
     meta.writeInt(count);
-    if (count == maxDoc) {
-      meta.writeByte((byte) -1); // dense marker, each document has a vector value
+    if (count == 0) {
+      meta.writeLong(-2); // docsWithFieldOffset
+      meta.writeLong(0L); // docsWithFieldLength
+      meta.writeShort((short) -1); // jumpTableEntryCount
+      meta.writeByte((byte) -1); // denseRankPower
+    } else if (count == maxDoc) {
+      meta.writeLong(-1); // docsWithFieldOffset
+      meta.writeLong(0L); // docsWithFieldLength
+      meta.writeShort((short) -1); // jumpTableEntryCount
+      meta.writeByte((byte) -1); // denseRankPower
     } else {
-      meta.writeByte((byte) 0); // sparse marker, some documents don't have vector values
-      DocIdSetIterator iter = docsWithField.iterator();
-      for (int doc = iter.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) {
-        meta.writeInt(doc);
+      long offset = vectorData.getFilePointer();
+      meta.writeLong(offset); // docsWithFieldOffset
+      final short jumpTableEntryCount =
+          IndexedDISI.writeBitSet(
+              docsWithField.iterator(), vectorData, IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+      meta.writeLong(vectorData.getFilePointer() - offset); // docsWithFieldLength
+      meta.writeShort(jumpTableEntryCount);
+      meta.writeByte(IndexedDISI.DEFAULT_DENSE_RANK_POWER);
+
+      // write ordToDoc mapping
+      long start = vectorData.getFilePointer();
+      meta.writeLong(start);
+      meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+      // dense case and empty case do not need to store ordToMap mapping
+      final DirectMonotonicWriter ordToDocWriter =
+          DirectMonotonicWriter.getInstance(meta, vectorData, count, DIRECT_MONOTONIC_BLOCK_SHIFT);
+      DocIdSetIterator iterator = docsWithField.iterator();
+      for (int doc = iterator.nextDoc();
+          doc != DocIdSetIterator.NO_MORE_DOCS;
+          doc = iterator.nextDoc()) {
+        ordToDocWriter.add(doc);
       }
+      ordToDocWriter.finish();
+      meta.writeLong(vectorData.getFilePointer() - start);
     }
 
     meta.writeInt(maxConn);
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene91/OffHeapVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/OffHeapVectorValues.java
new file mode 100644
index 00000000000..2b846db7f5b
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene91/OffHeapVectorValues.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.codecs.lucene91;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.lucene.codecs.lucene90.IndexedDISI;
+import org.apache.lucene.index.RandomAccessVectorValues;
+import org.apache.lucene.index.RandomAccessVectorValuesProducer;
+import org.apache.lucene.index.VectorValues;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.packed.DirectMonotonicReader;
+
+/** Read the vector values from the index input. This supports both iterated and random access. */
+abstract class OffHeapVectorValues extends VectorValues
+    implements RandomAccessVectorValues, RandomAccessVectorValuesProducer {
+
+  protected final int dimension;
+  protected final int size;
+  protected final IndexInput slice;
+  protected final BytesRef binaryValue;
+  protected final ByteBuffer byteBuffer;
+  protected final int byteSize;
+  protected final float[] value;
+
+  OffHeapVectorValues(int dimension, int size, IndexInput slice) {
+    this.dimension = dimension;
+    this.size = size;
+    this.slice = slice;
+    byteSize = Float.BYTES * dimension;
+    byteBuffer = ByteBuffer.allocate(byteSize);
+    value = new float[dimension];
+    binaryValue = new BytesRef(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize);
+  }
+
+  @Override
+  public int dimension() {
+    return dimension;
+  }
+
+  @Override
+  public int size() {
+    return size;
+  }
+
+  @Override
+  public long cost() {
+    return size;
+  }
+
+  @Override
+  public float[] vectorValue(int targetOrd) throws IOException {
+    slice.seek((long) targetOrd * byteSize);
+    slice.readFloats(value, 0, value.length);
+    return value;
+  }
+
+  @Override
+  public BytesRef binaryValue(int targetOrd) throws IOException {
+    readValue(targetOrd);
+    return binaryValue;
+  }
+
+  private void readValue(int targetOrd) throws IOException {
+    slice.seek((long) targetOrd * byteSize);
+    slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize);
+  }
+
+  public abstract int ordToDoc(int ord);
+
+  static OffHeapVectorValues load(
+      Lucene91HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
+    if (fieldEntry.docsWithFieldOffset == -2) {
+      return new EmptyOffHeapVectorValues(fieldEntry.dimension);
+    }
+    IndexInput bytesSlice =
+        vectorData.slice("vector-data", fieldEntry.vectorDataOffset, fieldEntry.vectorDataLength);
+    if (fieldEntry.docsWithFieldOffset == -1) {
+      return new DenseOffHeapVectorValues(fieldEntry.dimension, fieldEntry.size, bytesSlice);
+    } else {
+      return new SparseOffHeapVectorValues(fieldEntry, vectorData, bytesSlice);
+    }
+  }
+
+  abstract Bits getAcceptOrds(Bits acceptDocs);
+
+  static class DenseOffHeapVectorValues extends OffHeapVectorValues {
+
+    private int doc = -1;
+
+    public DenseOffHeapVectorValues(int dimension, int size, IndexInput slice) {
+      super(dimension, size, slice);
+    }
+
+    @Override
+    public float[] vectorValue() throws IOException {
+      slice.seek((long) doc * byteSize);
+      slice.readFloats(value, 0, value.length);
+      return value;
+    }
+
+    @Override
+    public BytesRef binaryValue() throws IOException {
+      slice.seek((long) doc * byteSize);
+      slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize, false);
+      return binaryValue;
+    }
+
+    @Override
+    public int docID() {
+      return doc;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      return advance(doc + 1);
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      assert docID() < target;
+      if (target >= size) {
+        return doc = NO_MORE_DOCS;
+      }
+      return doc = target;
+    }
+
+    @Override
+    public RandomAccessVectorValues randomAccess() throws IOException {
+      return new DenseOffHeapVectorValues(dimension, size, slice.clone());
+    }
+
+    @Override
+    public int ordToDoc(int ord) {
+      return ord;
+    }
+
+    @Override
+    Bits getAcceptOrds(Bits acceptDocs) {
+      return acceptDocs;
+    }
+  }
+
+  private static class SparseOffHeapVectorValues extends OffHeapVectorValues {
+    private final DirectMonotonicReader ordToDoc;
+    private final IndexedDISI disi;
+    // dataIn was used to init a new IndexedDIS for #randomAccess()
+    private final IndexInput dataIn;
+    private final Lucene91HnswVectorsReader.FieldEntry fieldEntry;
+
+    public SparseOffHeapVectorValues(
+        Lucene91HnswVectorsReader.FieldEntry fieldEntry, IndexInput dataIn, IndexInput slice)
+        throws IOException {
+
+      super(fieldEntry.dimension, fieldEntry.size, slice);
+      this.fieldEntry = fieldEntry;
+      final RandomAccessInput addressesData =
+          dataIn.randomAccessSlice(fieldEntry.addressesOffset, fieldEntry.addressesLength);
+      this.dataIn = dataIn;
+      this.ordToDoc = DirectMonotonicReader.getInstance(fieldEntry.meta, addressesData);
+      this.disi =
+          new IndexedDISI(
+              dataIn,
+              fieldEntry.docsWithFieldOffset,
+              fieldEntry.docsWithFieldLength,
+              fieldEntry.jumpTableEntryCount,
+              fieldEntry.denseRankPower,
+              fieldEntry.size);
+    }
+
+    @Override
+    public float[] vectorValue() throws IOException {
+      slice.seek((long) (disi.index()) * byteSize);
+      slice.readFloats(value, 0, value.length);
+      return value;
+    }
+
+    @Override
+    public BytesRef binaryValue() throws IOException {
+      slice.seek((long) (disi.index()) * byteSize);
+      slice.readBytes(byteBuffer.array(), byteBuffer.arrayOffset(), byteSize, false);
+      return binaryValue;
+    }
+
+    @Override
+    public int docID() {
+      return disi.docID();
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      return disi.nextDoc();
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      assert docID() < target;
+      return disi.advance(target);
+    }
+
+    @Override
+    public RandomAccessVectorValues randomAccess() throws IOException {
+      return new SparseOffHeapVectorValues(fieldEntry, dataIn, slice.clone());
+    }
+
+    @Override
+    public int ordToDoc(int ord) {
+      return (int) ordToDoc.get(ord);
+    }
+
+    @Override
+    Bits getAcceptOrds(Bits acceptDocs) {
+      if (acceptDocs == null) {
+        return null;
+      }
+      return new Bits() {
+        @Override
+        public boolean get(int index) {
+          return acceptDocs.get(ordToDoc(index));
+        }
+
+        @Override
+        public int length() {
+          return size;
+        }
+      };
+    }
+  }
+
+  private static class EmptyOffHeapVectorValues extends OffHeapVectorValues {
+
+    public EmptyOffHeapVectorValues(int dimension) {
+      super(dimension, 0, null);
+    }
+
+    private int doc = -1;
+
+    @Override
+    public int dimension() {
+      return super.dimension();
+    }
+
+    @Override
+    public int size() {
+      return 0;
+    }
+
+    @Override
+    public float[] vectorValue() throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public BytesRef binaryValue() throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int docID() {
+      return doc;
+    }
+
+    @Override
+    public int nextDoc() throws IOException {
+      return advance(doc + 1);
+    }
+
+    @Override
+    public int advance(int target) throws IOException {
+      return doc = NO_MORE_DOCS;
+    }
+
+    @Override
+    public long cost() {
+      return 0;
+    }
+
+    @Override
+    public RandomAccessVectorValues randomAccess() throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public float[] vectorValue(int targetOrd) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public BytesRef binaryValue(int targetOrd) throws IOException {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public int ordToDoc(int ord) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    Bits getAcceptOrds(Bits acceptDocs) {
+      return null;
+    }
+  }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/index/RandomAccessVectorValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/RandomAccessVectorValuesProducer.java
index 54e2dc15580..cce960022a6 100644
--- a/lucene/core/src/java/org/apache/lucene/index/RandomAccessVectorValuesProducer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/RandomAccessVectorValuesProducer.java
@@ -17,6 +17,8 @@
 
 package org.apache.lucene.index;
 
+import java.io.IOException;
+
 /**
  * Something (generally a {@link VectorValues}) that provides a {@link RandomAccessVectorValues}.
  *
@@ -29,5 +31,5 @@ public interface RandomAccessVectorValuesProducer {
    * Successive calls will retrieve independent copies that do not overwrite each others' returned
    * values.
    */
-  RandomAccessVectorValues randomAccess();
+  RandomAccessVectorValues randomAccess() throws IOException;
 }
diff --git a/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java b/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
index c5b2c9781c9..5c12cda2b8c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/VectorValuesWriter.java
@@ -225,7 +225,7 @@ class VectorValuesWriter {
     }
 
     @Override
-    public RandomAccessVectorValues randomAccess() {
+    public RandomAccessVectorValues randomAccess() throws IOException {
 
       // Must make a new delegate randomAccess so that we have our own distinct float[]
       final RandomAccessVectorValues delegateRA =
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
index dcd1c25a77f..9ccde77b38d 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraphBuilder.java
@@ -79,7 +79,8 @@ public final class HnswGraphBuilder {
       VectorSimilarityFunction similarityFunction,
       int maxConn,
       int beamWidth,
-      long seed) {
+      long seed)
+      throws IOException {
     vectorValues = vectors.randomAccess();
     buildVectors = vectors.randomAccess();
     this.similarityFunction = Objects.requireNonNull(similarityFunction);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
index 471b83ed0a1..f9c544b2e86 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
@@ -451,8 +451,9 @@ public class TestKnnGraph extends LuceneTestCase {
 
         // assert vector values:
         // stored vector values are the same as original
+        int nextDocWithVectors = 0;
         for (int i = 0; i < reader.maxDoc(); i++) {
-          int nextDocWithVectors = vectorValues.advance(i);
+          nextDocWithVectors = vectorValues.advance(i);
           while (i < nextDocWithVectors && i < reader.maxDoc()) {
             int id = Integer.parseInt(reader.document(i).get("id"));
             assertNull("document " + id + " has no vector, but was expected to", values[id]);
@@ -471,7 +472,12 @@ public class TestKnnGraph extends LuceneTestCase {
               0f);
           numDocsWithVectors++;
         }
-        assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
+        // if IndexDisi.doc == NO_MORE_DOCS, we should not call IndexDisi.nextDoc()
+        if (nextDocWithVectors != NO_MORE_DOCS) {
+          assertEquals(NO_MORE_DOCS, vectorValues.nextDoc());
+        } else {
+          assertEquals(NO_MORE_DOCS, vectorValues.docID());
+        }
 
         // assert graph values:
         // For each level of the graph assert that: