You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2022/12/07 07:52:00 UTC
[lucene] branch main updated: GITHUB-11830 Better optimize storage for vector connections (#11860)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new d0be9ab57c9 GITHUB-11830 Better optimize storage for vector connections (#11860)
d0be9ab57c9 is described below
commit d0be9ab57c9c1e79ce7363c3ca85f3bcdb7c94ad
Author: Benjamin Trent <be...@gmail.com>
AuthorDate: Wed Dec 7 02:51:54 2022 -0500
GITHUB-11830 Better optimize storage for vector connections (#11860)
---
lucene/CHANGES.txt | 3 +
lucene/backward-codecs/src/java/module-info.java | 7 +-
.../lucene94/ExpandingVectorValues.java | 6 +-
.../backward_codecs}/lucene94/Lucene94Codec.java | 5 +-
.../lucene94/Lucene94HnswVectorsFormat.java | 6 +-
.../lucene94/Lucene94HnswVectorsReader.java | 2 +-
.../lucene94/Lucene94HnswVectorsWriter.java | 4 +-
.../lucene94/OffHeapVectorValues.java | 2 +-
.../backward_codecs}/lucene94/package-info.java | 10 +-
.../services/org.apache.lucene.codecs.Codec | 1 +
.../org.apache.lucene.codecs.KnnVectorsFormat | 1 +
.../lucene94/TestLucene94HnswVectorsFormat.java | 2 +-
.../benchmark/byTask/tasks/CreateIndexTask.java | 4 +-
lucene/core/src/java/module-info.java | 9 +-
.../src/java/org/apache/lucene/codecs/Codec.java | 2 +-
.../lucene/codecs/lucene94/package-info.java | 405 +--------------------
.../ExpandingVectorValues.java | 4 +-
.../Lucene95Codec.java} | 25 +-
.../Lucene95HnswVectorsFormat.java} | 80 ++--
.../Lucene95HnswVectorsReader.java} | 156 ++++----
.../Lucene95HnswVectorsWriter.java} | 187 ++++++----
.../OffHeapVectorValues.java | 8 +-
.../{lucene94 => lucene95}/package-info.java | 11 +-
.../org/apache/lucene/util/hnsw/HnswGraph.java | 22 ++
.../services/org.apache.lucene.codecs.Codec | 2 +-
.../org.apache.lucene.codecs.KnnVectorsFormat | 2 +-
...tLucene90StoredFieldsFormatHighCompression.java | 11 +-
.../TestLucene95HnswVectorsFormat.java} | 21 +-
.../test/org/apache/lucene/index/TestKnnGraph.java | 30 +-
.../apache/lucene/util/hnsw/KnnGraphTester.java | 12 +-
.../org/apache/lucene/util/hnsw/TestHnswGraph.java | 20 +-
.../search/suggest/document/TestSuggestField.java | 4 +-
.../tests/codecs/vector/ConfigurableMCodec.java | 12 +-
.../util/TestRuleSetupAndRestoreClassEnv.java | 8 +-
.../org/apache/lucene/tests/util/TestUtil.java | 8 +-
35 files changed, 417 insertions(+), 675 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6dd69fa32ae..8171b0cc632 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -152,6 +152,9 @@ Improvements
* GITHUB#11985: ExitableTerms to override Terms#getMin and Terms#getMax in order to avoid
iterating through the terms when the wrapped implementation caches such values. (Luca Cavanna)
+* GITHUB#11860: Improve storage efficiency of connections in the HNSW graph that Lucene uses for
+ vector search. (Ben Trent)
+
Bug Fixes
---------------------
* GITHUB#11726: Indexing term vectors on large documents could fail due to
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 1f51ead7d8f..205b1facaa8 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -33,6 +33,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene90;
exports org.apache.lucene.backward_codecs.lucene91;
exports org.apache.lucene.backward_codecs.lucene92;
+ exports org.apache.lucene.backward_codecs.lucene94;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;
@@ -44,7 +45,8 @@ module org.apache.lucene.backward_codecs {
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
- org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat;
+ org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat,
+ org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat;
provides org.apache.lucene.codecs.Codec with
org.apache.lucene.backward_codecs.lucene80.Lucene80Codec,
org.apache.lucene.backward_codecs.lucene84.Lucene84Codec,
@@ -52,5 +54,6 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene87.Lucene87Codec,
org.apache.lucene.backward_codecs.lucene90.Lucene90Codec,
org.apache.lucene.backward_codecs.lucene91.Lucene91Codec,
- org.apache.lucene.backward_codecs.lucene92.Lucene92Codec;
+ org.apache.lucene.backward_codecs.lucene92.Lucene92Codec,
+ org.apache.lucene.backward_codecs.lucene94.Lucene94Codec;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/ExpandingVectorValues.java
similarity index 88%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/ExpandingVectorValues.java
index 292ef6e09e0..21eb7316ed0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/ExpandingVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import java.io.IOException;
import org.apache.lucene.index.FilterVectorValues;
@@ -23,11 +23,13 @@ import org.apache.lucene.index.VectorValues;
import org.apache.lucene.util.BytesRef;
/** reads from byte-encoded data */
-public class ExpandingVectorValues extends FilterVectorValues {
+class ExpandingVectorValues extends FilterVectorValues {
private final float[] value;
/**
+ * Constructs ExpandingVectorValues with passed byte encoded VectorValues iterator
+ *
* @param in the wrapped values
*/
protected ExpandingVectorValues(VectorValues in) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
index db8be826d78..c43a4351e73 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94Codec.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
@@ -39,6 +39,7 @@ import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
+import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -48,7 +49,7 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
- * @see org.apache.lucene.codecs.lucene94 package documentation for file format details.
+ * @see org.apache.lucene.backward_codecs.lucene94 package documentation for file format details.
* @lucene.experimental
*/
public class Lucene94Codec extends Codec {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsFormat.java
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsFormat.java
index d263e8122a9..200bb17bad9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsFormat.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import java.io.IOException;
import org.apache.lucene.codecs.KnnVectorsFormat;
@@ -105,8 +105,8 @@ public final class Lucene94HnswVectorsFormat extends KnnVectorsFormat {
static final String VECTOR_DATA_EXTENSION = "vec";
static final String VECTOR_INDEX_EXTENSION = "vex";
- public static final int VERSION_START = 0;
- public static final int VERSION_CURRENT = 1;
+ static final int VERSION_START = 0;
+ static final int VERSION_CURRENT = 1;
/** Default number of maximum connections per node */
public static final int DEFAULT_MAX_CONN = 16;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
index 8d95614f9fe..361d2c9de15 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsReader.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
index 9ab3627b4ac..6da2df8ca81 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/Lucene94HnswVectorsWriter.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/OffHeapVectorValues.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/OffHeapVectorValues.java
index 23df75fa022..042ab6e4904 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/OffHeapVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import java.io.IOException;
import java.nio.ByteBuffer;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
index 84956d8ec0e..db32e6bce67 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene94/package-info.java
@@ -16,7 +16,7 @@
*/
/**
- * Lucene 9.3 file format.
+ * Lucene 9.4 file format.
*
* <h2>Apache Lucene - Index File Formats</h2>
*
@@ -180,8 +180,8 @@
* of files, recording dimensionally indexed fields, to enable fast numeric range filtering
* and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
* intersection (2D, 3D).
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}. The
- * vector format stores numeric vectors in a format optimized for random access and
+ * <li>{@link org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat Vector values}.
+ * The vector format stores numeric vectors in a format optimized for random access and
* computation, supporting high-dimensional nearest-neighbor search.
* </ul>
*
@@ -310,7 +310,7 @@
* <td>Holds indexed points</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat Vector values}</td>
* <td>.vec, .vem</td>
* <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data, and
* <code>.vem</code> the vector metadata</td>
@@ -419,4 +419,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index d9a3ef60880..92d161c885d 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -20,3 +20,4 @@ org.apache.lucene.backward_codecs.lucene87.Lucene87Codec
org.apache.lucene.backward_codecs.lucene90.Lucene90Codec
org.apache.lucene.backward_codecs.lucene91.Lucene91Codec
org.apache.lucene.backward_codecs.lucene92.Lucene92Codec
+org.apache.lucene.backward_codecs.lucene94.Lucene94Codec
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index 037584b0318..a42ccd0f040 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -16,3 +16,4 @@
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat
org.apache.lucene.backward_codecs.lucene92.Lucene92HnswVectorsFormat
+org.apache.lucene.backward_codecs.lucene94.Lucene94HnswVectorsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/TestLucene94HnswVectorsFormat.java
similarity index 96%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/TestLucene94HnswVectorsFormat.java
index 3590d3c168d..63462be4262 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene94/TestLucene94HnswVectorsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.backward_codecs.lucene94;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
index 243ade6fe04..9a14710fbd2 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
@@ -27,7 +27,7 @@ import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
@@ -152,7 +152,7 @@ public class CreateIndexTask extends PerfTask {
try {
final PostingsFormat postingsFormatChosen = PostingsFormat.forName(postingsFormat);
iwConf.setCodec(
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
return postingsFormatChosen;
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index fd1a4935250..f5d8cd275b7 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -15,8 +15,8 @@
* limitations under the License.
*/
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
/** Lucene Core. */
@SuppressWarnings("module") // the test framework is compiled after the core...
@@ -32,6 +32,7 @@ module org.apache.lucene.core {
exports org.apache.lucene.codecs.compressing;
exports org.apache.lucene.codecs.lucene90;
exports org.apache.lucene.codecs.lucene94;
+ exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.perfield;
@@ -63,11 +64,11 @@ module org.apache.lucene.core {
provides org.apache.lucene.analysis.TokenizerFactory with
org.apache.lucene.analysis.standard.StandardTokenizerFactory;
provides org.apache.lucene.codecs.Codec with
- Lucene94Codec;
+ Lucene95Codec;
provides org.apache.lucene.codecs.DocValuesFormat with
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
- Lucene94HnswVectorsFormat;
+ Lucene95HnswVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
provides org.apache.lucene.index.SortFieldProvider with
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
index 1192a45c388..0f512fccaa2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
@@ -55,7 +55,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
return LOADER;
}
- static Codec defaultCodec = LOADER.lookup("Lucene94");
+ static Codec defaultCodec = LOADER.lookup("Lucene95");
}
private final String name;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
index 84956d8ec0e..1a5d057a21d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
@@ -15,408 +15,5 @@
* limitations under the License.
*/
-/**
- * Lucene 9.3 file format.
- *
- * <h2>Apache Lucene - Index File Formats</h2>
- *
- * <div>
- *
- * <ul>
- * <li><a href="#Introduction">Introduction</a>
- * <li><a href="#Definitions">Definitions</a>
- * <ul>
- * <li><a href="#Inverted_Indexing">Inverted Indexing</a>
- * <li><a href="#Types_of_Fields">Types of Fields</a>
- * <li><a href="#Segments">Segments</a>
- * <li><a href="#Document_Numbers">Document Numbers</a>
- * </ul>
- * <li><a href="#Overview">Index Structure Overview</a>
- * <li><a href="#File_Naming">File Naming</a>
- * <li><a href="#file-names">Summary of File Extensions</a>
- * <ul>
- * <li><a href="#Lock_File">Lock File</a>
- * <li><a href="#History">History</a>
- * <li><a href="#Limitations">Limitations</a>
- * </ul>
- * </ul>
- *
- * </div> <a id="Introduction"></a>
- *
- * <h3>Introduction</h3>
- *
- * <div>
- *
- * <p>This document defines the index file formats used in this version of Lucene. If you are using
- * a different version of Lucene, please consult the copy of <code>docs/</code> that was distributed
- * with the version you are using.
- *
- * <p>This document attempts to provide a high-level definition of the Apache Lucene file formats.
- * </div> <a id="Definitions"></a>
- *
- * <h3>Definitions</h3>
- *
- * <div>
- *
- * <p>The fundamental concepts in Lucene are index, document, field and term.
- *
- * <p>An index contains a sequence of documents.
- *
- * <ul>
- * <li>A document is a sequence of fields.
- * <li>A field is a named sequence of terms.
- * <li>A term is a sequence of bytes.
- * </ul>
- *
- * <p>The same sequence of bytes in two different fields is considered a different term. Thus terms
- * are represented as a pair: the string naming the field, and the bytes within the field. <a
- * id="Inverted_Indexing"></a>
- *
- * <h4>Inverted Indexing</h4>
- *
- * <p>Lucene's index stores terms and statistics about those terms in order to make term-based
- * search more efficient. Lucene's terms index falls into the family of indexes known as an
- * <i>inverted index.</i> This is because it can list, for a term, the documents that contain it.
- * This is the inverse of the natural relationship, in which documents list terms. <a
- * id="Types_of_Fields"></a>
- *
- * <h4>Types of Fields</h4>
- *
- * <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored in the index
- * literally, in a non-inverted manner. Fields that are inverted are called <i>indexed</i>. A field
- * may be both stored and indexed.
- *
- * <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the text of a field
- * may be used literally as a term to be indexed. Most fields are tokenized, but sometimes it is
- * useful for certain identifier fields to be indexed literally.
- *
- * <p>See the {@link org.apache.lucene.document.Field Field} java docs for more information on
- * Fields. <a id="Segments"></a>
- *
- * <h4>Segments</h4>
- *
- * <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>. Each segment is a
- * fully independent index, which could be searched separately. Indexes evolve by:
- *
- * <ol>
- * <li>Creating new segments for newly added documents.
- * <li>Merging existing segments.
- * </ol>
- *
- * <p>Searches may involve multiple segments and/or multiple indexes, each index potentially
- * composed of a set of segments. <a id="Document_Numbers"></a>
- *
- * <h4>Document Numbers</h4>
- *
- * <p>Internally, Lucene refers to documents by an integer <i>document number</i>. The first
- * document added to an index is numbered zero, and each subsequent document added gets a number one
- * greater than the previous.
- *
- * <p>Note that a document's number may change, so caution should be taken when storing these
- * numbers outside of Lucene. In particular, numbers may change in the following situations:
- *
- * <ul>
- * <li>
- * <p>The numbers stored in each segment are unique only within the segment, and must be
- * converted before they can be used in a larger context. The standard technique is to
- * allocate each segment a range of values, based on the range of numbers used in that
- * segment. To convert a document number from a segment to an external value, the segment's
- * <i>base</i> document number is added. To convert an external value back to a
- * segment-specific value, the segment is identified by the range that the external value is
- * in, and the segment's base value is subtracted. For example two five document segments
- * might be combined, so that the first segment has a base value of zero, and the second of
- * five. Document three from the second segment would have an external value of eight.
- * <li>
- * <p>When documents are deleted, gaps are created in the numbering. These are eventually
- * removed as the index evolves through merging. Deleted documents are dropped when segments
- * are merged. A freshly-merged segment thus has no gaps in its numbering.
- * </ul>
- *
- * </div> <a id="Overview"></a>
- *
- * <h3>Index Structure Overview</h3>
- *
- * <div>
- *
- * <p>Each segment index maintains the following:
- *
- * <ul>
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90SegmentInfoFormat Segment info}. This
- * contains metadata about a segment, such as the number of documents, what files it uses, and
- * information about how the segment is sorted
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Field names}. This
- * contains metadata about the set of named fields used in the index.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Stored Field values}.
- * This contains, for each document, a list of attribute-value pairs, where the attributes are
- * field names. These are used to store auxiliary information about the document, such as its
- * title, url, or an identifier to access a database. The set of stored fields are what is
- * returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
- * documents. The dictionary also contains the number of documents which contain the term, and
- * pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
- * each field in each document, a value is stored that is multiplied into the score for hits
- * on that field.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each
- * field in each document, the term vector (sometimes called document vector) may be stored. A
- * term vector consists of term text and term frequency. To add Term Vectors to your index see
- * the {@link org.apache.lucene.document.Field Field} constructors
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-document values}. Like
- * stored values, these are also keyed by document number, but are generally intended to be
- * loaded into main memory for fast access. Whereas stored values are generally intended for
- * summary results from searches, per-document values are useful for things like scoring
- * factors.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live documents}. An
- * optional file indicating which documents are live.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}. Optional pair
- * of files, recording dimensionally indexed fields, to enable fast numeric range filtering
- * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
- * intersection (2D, 3D).
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}. The
- * vector format stores numeric vectors in a format optimized for random access and
- * computation, supporting high-dimensional nearest-neighbor search.
- * </ul>
- *
- * <p>Details on each of these are provided in their linked pages. </div> <a id="File_Naming"></a>
- *
- * <h3>File Naming</h3>
- *
- * <div>
- *
- * <p>All files belonging to a segment have the same name with varying extensions. The extensions
- * correspond to the different file formats described below. When using the Compound File format
- * (default for small segments) these files (except for the Segment info file, the Lock file, and
- * Deleted documents file) are collapsed into a single .cfs file (see below for details)
- *
- * <p>Typically, all segments in an index are stored in a single directory, although this is not
- * required.
- *
- * <p>File names are never re-used. That is, when any file is saved to the Directory it is given a
- * never before used filename. This is achieved using a simple generations approach. For example,
- * the first segments file is segments_1, then segments_2, etc. The generation is a sequential long
- * integer represented in alpha-numeric (base 36) form. </div> <a id="file-names"></a>
- *
- * <h3>Summary of File Extensions</h3>
- *
- * <div>
- *
- * <p>The following table summarizes the names and extensions of the files in Lucene:
- *
- * <table class="padding4" style="border-spacing: 1px; border-collapse: separate">
- * <caption>lucene filenames by extension</caption>
- * <tr>
- * <th>Name</th>
- * <th>Extension</th>
- * <th>Brief Description</th>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
- * <td>segments_N</td>
- * <td>Stores information about a commit point</td>
- * </tr>
- * <tr>
- * <td><a href="#Lock_File">Lock File</a></td>
- * <td>write.lock</td>
- * <td>The Write lock prevents multiple IndexWriters from writing to the same
- * file.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90SegmentInfoFormat Segment Info}</td>
- * <td>.si</td>
- * <td>Stores metadata about a segment</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat Compound File}</td>
- * <td>.cfs, .cfe</td>
- * <td>An optional "virtual" file consisting of all the other index files for
- * systems that frequently run out of file handles.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Fields}</td>
- * <td>.fnm</td>
- * <td>Stores information about the fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Index}</td>
- * <td>.fdx</td>
- * <td>Contains pointers to field data</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Data}</td>
- * <td>.fdt</td>
- * <td>The stored fields for documents</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Dictionary}</td>
- * <td>.tim</td>
- * <td>The term dictionary, stores term info</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Term Index}</td>
- * <td>.tip</td>
- * <td>The index into the Term Dictionary</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Frequencies}</td>
- * <td>.doc</td>
- * <td>Contains the list of docs which contain each term along with frequency</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Positions}</td>
- * <td>.pos</td>
- * <td>Stores position information about where a term occurs in the index</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat Payloads}</td>
- * <td>.pay</td>
- * <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Norms}</td>
- * <td>.nvd, .nvm</td>
- * <td>Encodes length and boost factors for docs and fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-Document Values}</td>
- * <td>.dvd, .dvm</td>
- * <td>Encodes additional scoring factors or other per-document information.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>
- * <td>.tvx</td>
- * <td>Stores offset into the document data file</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>
- * <td>.tvd</td>
- * <td>Contains term vector data.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live Documents}</td>
- * <td>.liv</td>
- * <td>Info about what documents are live</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}</td>
- * <td>.dii, .dim</td>
- * <td>Holds indexed points</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}</td>
- * <td>.vec, .vem</td>
- * <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data, and
- * <code>.vem</code> the vector metadata</td>
- * </tr>
- * </table>
- *
- * </div> <a id="Lock_File"></a>
- *
- * <h3>Lock File</h3>
- *
- * The write lock, which is stored in the index directory by default, is named "write.lock". If the
- * lock directory is different from the index directory then the write lock will be named
- * "XXXX-write.lock" where XXXX is a unique prefix derived from the full path to the index
- * directory. When this file is present, a writer is currently modifying the index (adding or
- * removing documents). This lock file ensures that only one writer is modifying the index at a
- * time. <a id="History"></a>
- *
- * <h3>History</h3>
- *
- * <p>Compatibility notes are provided in this document, describing how file formats have changed
- * from prior versions:
- *
- * <ul>
- * <li>In version 2.1, the file format was changed to allow lock-less commits (ie, no more commit
- * lock). The change is fully backwards compatible: you can open a pre-2.1 index for searching
- * or adding/deleting of docs. When the new segments file is saved (committed), it will be
- * written in the new file format (meaning no specific "upgrade" process is needed). But note
- * that once a commit has occurred, pre-2.1 Lucene will not be able to read the index.
- * <li>In version 2.3, the file format was changed to allow segments to share a single set of doc
- * store (vectors & stored fields) files. This allows for faster indexing in certain
- * cases. The change is fully backwards compatible (in the same way as the lock-less commits
- * change in 2.1).
- * <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not Java's modified
- * UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">LUCENE-510</a> for
- * details.
- * <li>In version 2.9, an optional opaque Map<String,String> CommitUserData may be passed to
- * IndexWriter's commit methods (and later retrieved), which is recorded in the segments_N
- * file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">LUCENE-1382</a> for
- * details. Also, diagnostics were added to each segment written recording details about why
- * it was written (due to flush, merge; which OS/JRE was used; etc.). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.
- * <li>In version 3.0, compressed fields are no longer written to the index (they can still be
- * read, but on merge the new segment will write them, uncompressed). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a> for details.
- * <li>In version 3.1, segments records the code version that created them. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
- * Additionally segments track explicitly whether or not they have term vectors. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a> for details.
- * <li>In version 3.2, numeric fields are written as natively to stored fields file, previously
- * they were stored in text format only.
- * <li>In version 3.4, fields can omit position data while still indexing term frequencies.
- * <li>In version 4.0, the format of the inverted index became extensible via the {@link
- * org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage ({@code DocValues})
- * was introduced. Normalization factors need no longer be a single byte, they can be any
- * {@link org.apache.lucene.index.NumericDocValues NumericDocValues}. Terms need not be
- * unicode strings, they can be any byte sequence. Term offsets can optionally be indexed into
- * the postings lists. Payloads can be stored in the term vectors.
- * <li>In version 4.1, the format of the postings list changed to use either of FOR compression or
- * variable-byte encoding, depending upon the frequency of the term. Terms appearing only once
- * were changed to inline directly into the term dictionary. Stored fields are compressed by
- * default.
- * <li>In version 4.2, term vectors are compressed by default. DocValues has a new multi-valued
- * type (SortedSet), that can be used for faceting/grouping/joining on multi-valued fields.
- * <li>In version 4.5, DocValues were extended to explicitly represent missing values.
- * <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
- * allow updating NumericDocValues fields.
- * <li>In version 4.8, checksum footers were added to the end of each index file for improved data
- * integrity. Specifically, the last 8 bytes of every index file contain the zlib-crc32
- * checksum of the file.
- * <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) that is
- * suitable for faceting/sorting/analytics.
- * <li>In version 5.4, DocValues have been improved to store more information on disk: addresses
- * for binary fields and ord indexes for multi-valued fields.
- * <li>In version 6.0, Points were added, for multi-dimensional range/distance search.
- * <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index
- * sorting.
- * <li>In version 7.0, DocValues have been improved to better support sparse doc values thanks to
- * an iterator API.
- * <li>In version 8.0, postings have been enhanced to record, for each block of doc ids, the (term
- * freq, normalization factor) pairs that may trigger the maximum score of the block. This
- * information is recorded alongside skip data in order to be able to skip blocks of doc ids
- * if they may not produce high enough scores. Additionally doc values and norms has been
- * extended with jump-tables to make access O(1) instead of O(n), where n is the number of
- * elements to skip when advancing in the data.
- * <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more
- * performant encoding that is vectorized.
- * <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to allow
- * user-defined sorts to be used
- * <li>In version 8.7, stored fields compression became adaptive to better handle documents with
- * smaller stored fields.
- * <li>In version 9.0, vector-valued fields were added.
- * <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.
- * <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by
- * IndexDISI. ordToDoc mappings was added to .vem.
- * </ul>
- *
- * <a id="Limitations"></a>
- *
- * <h3>Limitations</h3>
- *
- * <div>
- *
- * <p>Lucene uses a Java <code>int</code> to refer to document numbers, and the index file format
- * uses an <code>Int32</code> on-disk to store document numbers. This is a limitation of both the
- * index file format and the current implementation. Eventually these should be replaced with either
- * <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
- * VInt} values which have no limit. </div>
- */
+/** Lucene 9.4 file format. */
package org.apache.lucene.codecs.lucene94;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/ExpandingVectorValues.java
similarity index 93%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/ExpandingVectorValues.java
index 292ef6e09e0..3f62cdb7cb6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/ExpandingVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/ExpandingVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import java.io.IOException;
import org.apache.lucene.index.FilterVectorValues;
@@ -23,7 +23,7 @@ import org.apache.lucene.index.VectorValues;
import org.apache.lucene.util.BytesRef;
/** reads from byte-encoded data */
-public class ExpandingVectorValues extends FilterVectorValues {
+class ExpandingVectorValues extends FilterVectorValues {
private final float[] value;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95Codec.java
similarity index 91%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95Codec.java
index db8be826d78..fd495b6026f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95Codec.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
@@ -39,19 +39,20 @@ import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90SegmentInfoFormat;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
+import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
- * Implements the Lucene 9.4 index format
+ * Implements the Lucene 9.5 index format
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
- * @see org.apache.lucene.codecs.lucene94 package documentation for file format details.
+ * @see org.apache.lucene.codecs.lucene95 package documentation for file format details.
* @lucene.experimental
*/
-public class Lucene94Codec extends Codec {
+public class Lucene95Codec extends Codec {
/** Configuration option for the codec. */
public enum Mode {
@@ -79,7 +80,7 @@ public class Lucene94Codec extends Codec {
new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene94Codec.this.getPostingsFormatForField(field);
+ return Lucene95Codec.this.getPostingsFormatForField(field);
}
};
@@ -88,7 +89,7 @@ public class Lucene94Codec extends Codec {
new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene94Codec.this.getDocValuesFormatForField(field);
+ return Lucene95Codec.this.getDocValuesFormatForField(field);
}
};
@@ -97,14 +98,14 @@ public class Lucene94Codec extends Codec {
new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return Lucene94Codec.this.getKnnVectorsFormatForField(field);
+ return Lucene95Codec.this.getKnnVectorsFormatForField(field);
}
};
private final StoredFieldsFormat storedFieldsFormat;
/** Instantiates a new codec. */
- public Lucene94Codec() {
+ public Lucene95Codec() {
this(Mode.BEST_SPEED);
}
@@ -113,13 +114,13 @@ public class Lucene94Codec extends Codec {
*
* @param mode stored fields compression mode to use for newly flushed/merged segments.
*/
- public Lucene94Codec(Mode mode) {
- super("Lucene94");
+ public Lucene95Codec(Mode mode) {
+ super("Lucene95");
this.storedFieldsFormat =
new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
this.defaultPostingsFormat = new Lucene90PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
- this.defaultKnnVectorsFormat = new Lucene94HnswVectorsFormat();
+ this.defaultKnnVectorsFormat = new Lucene95HnswVectorsFormat();
}
@Override
@@ -195,7 +196,7 @@ public class Lucene94Codec extends Codec {
/**
* Returns the vectors format that should be used for writing new segments of <code>field</code>
*
- * <p>The default implementation always returns "lucene94".
+ * <p>The default implementation always returns "Lucene95".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
similarity index 65%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
index d263e8122a9..cb3e5ef8b10 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsFormat.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import java.io.IOException;
import org.apache.lucene.codecs.KnnVectorsFormat;
@@ -29,7 +29,7 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.hnsw.HnswGraph;
/**
- * Lucene 9.4 vector format, which encodes numeric vector values and an optional associated graph
+ * Lucene 9.5 vector format, which encodes numeric vector values and an optional associated graph
* connecting the documents having values. The graph is used to power HNSW search. The format
* consists of three files:
*
@@ -57,13 +57,13 @@ import org.apache.lucene.util.hnsw.HnswGraph;
* <ul>
* <li>For each node:
* <ul>
- * <li><b>[int32]</b> the number of neighbor nodes
- * <li><b>array[int32]</b> the neighbor ordinals
- * <li><b>array[int32]</b> padding if the number of the node's neighbors is less than
- * the maximum number of connections allowed on this level. Padding is equal to
- * ((maxConnOnLevel – the number of neighbours) * 4) bytes.
+ * <li><b>[vint]</b> the number of neighbor nodes
+ * <li><b>array[vint]</b> the delta encoded neighbor ordinals
* </ul>
* </ul>
+ * <li>After all levels are encoded memory offsets for each node's neighbor nodes encoded by
+ * {@link org.apache.lucene.util.packed.DirectMonotonicWriter} are appened to the end of the
+ * file.
* </ul>
*
* <h2>.vem (vector metadata) file</h2>
@@ -77,39 +77,53 @@ import org.apache.lucene.util.hnsw.HnswGraph;
* <li><b>[vlong]</b> length of this field's vectors, in bytes
* <li><b>[vlong]</b> offset to this field's index in the .vex file
* <li><b>[vlong]</b> length of this field's index data, in bytes
- * <li><b>[int]</b> dimension of this field's vectors
+ * <li><b>[vint]</b> dimension of this field's vectors
* <li><b>[int]</b> the number of documents having values for this field
* <li><b>[int8]</b> if equals to -1, dense – all documents have values for a field. If equals to
* 0, sparse – some documents missing values.
* <li>DocIds were encoded by {@link IndexedDISI#writeBitSet(DocIdSetIterator, IndexOutput, byte)}
* <li>OrdToDoc was encoded by {@link org.apache.lucene.util.packed.DirectMonotonicWriter}, note
* that only in sparse case
- * <li><b>[int]</b> the maximum number of connections (neigbours) that each node can have
- * <li><b>[int]</b> number of levels in the graph
+ * <li><b>[vint]</b> the maximum number of connections (neigbours) that each node can have
+ * <li><b>[vint]</b> number of levels in the graph
* <li>Graph nodes by level. For each level
* <ul>
- * <li><b>[int]</b> the number of nodes on this level
- * <li><b>array[int]</b> for levels greater than 0 list of nodes on this level, stored as
- * the level 0th nodes' ordinals.
+ * <li><b>[vint]</b> the number of nodes on this level
+ * <li><b>array[vint]</b> for levels greater than 0 list of nodes on this level, stored as
+ * the level 0th delta encoded nodes' ordinals.
* </ul>
* </ul>
*
* @lucene.experimental
*/
-public final class Lucene94HnswVectorsFormat extends KnnVectorsFormat {
+public final class Lucene95HnswVectorsFormat extends KnnVectorsFormat {
- static final String META_CODEC_NAME = "lucene94HnswVectorsFormatMeta";
- static final String VECTOR_DATA_CODEC_NAME = "lucene94HnswVectorsFormatData";
- static final String VECTOR_INDEX_CODEC_NAME = "lucene94HnswVectorsFormatIndex";
+ static final String META_CODEC_NAME = "Lucene95HnswVectorsFormatMeta";
+ static final String VECTOR_DATA_CODEC_NAME = "Lucene95HnswVectorsFormatData";
+ static final String VECTOR_INDEX_CODEC_NAME = "Lucene95HnswVectorsFormatIndex";
static final String META_EXTENSION = "vem";
static final String VECTOR_DATA_EXTENSION = "vec";
static final String VECTOR_INDEX_EXTENSION = "vex";
public static final int VERSION_START = 0;
- public static final int VERSION_CURRENT = 1;
+ public static final int VERSION_CURRENT = VERSION_START;
+ /**
+ * A maximum configurable maximum max conn.
+ *
+ * <p>NOTE: We eagerly populate `float[MAX_CONN*2]` and `int[MAX_CONN*2]`, so exceptionally large
+ * numbers here will use an inordinate amount of heap
+ */
+ private static final int MAXIMUM_MAX_CONN = 512;
/** Default number of maximum connections per node */
public static final int DEFAULT_MAX_CONN = 16;
+
+ /**
+ * The maximum size of the queue to maintain while searching during graph construction This
+ * maximum value preserves the ratio of the DEFAULT_BEAM_WIDTH/DEFAULT_MAX_CONN i.e. `6.25 * 16 =
+ * 3200`
+ */
+ private static final int MAXIMUM_BEAM_WIDTH = 3200;
/**
* Default number of the size of the queue maintained while searching during a graph construction.
*/
@@ -119,19 +133,19 @@ public final class Lucene94HnswVectorsFormat extends KnnVectorsFormat {
/**
* Controls how many of the nearest neighbor candidates are connected to the new node. Defaults to
- * {@link Lucene94HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details.
+ * {@link Lucene95HnswVectorsFormat#DEFAULT_MAX_CONN}. See {@link HnswGraph} for more details.
*/
private final int maxConn;
/**
* The number of candidate neighbors to track while searching the graph for each newly inserted
- * node. Defaults to to {@link Lucene94HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link
+ * node. Defaults to to {@link Lucene95HnswVectorsFormat#DEFAULT_BEAM_WIDTH}. See {@link
* HnswGraph} for details.
*/
private final int beamWidth;
/** Constructs a format using default graph construction parameters */
- public Lucene94HnswVectorsFormat() {
+ public Lucene95HnswVectorsFormat() {
this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH);
}
@@ -141,25 +155,39 @@ public final class Lucene94HnswVectorsFormat extends KnnVectorsFormat {
* @param maxConn the maximum number of connections to a node in the HNSW graph
* @param beamWidth the size of the queue maintained during graph construction.
*/
- public Lucene94HnswVectorsFormat(int maxConn, int beamWidth) {
- super("Lucene94HnswVectorsFormat");
+ public Lucene95HnswVectorsFormat(int maxConn, int beamWidth) {
+ super("Lucene95HnswVectorsFormat");
+ if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) {
+ throw new IllegalArgumentException(
+ "maxConn must be postive and less than or equal to"
+ + MAXIMUM_MAX_CONN
+ + "; maxConn="
+ + maxConn);
+ }
+ if (beamWidth <= 0 || beamWidth > MAXIMUM_BEAM_WIDTH) {
+ throw new IllegalArgumentException(
+ "beamWidth must be postive and less than or equal to"
+ + MAXIMUM_BEAM_WIDTH
+ + "; beamWidth="
+ + beamWidth);
+ }
this.maxConn = maxConn;
this.beamWidth = beamWidth;
}
@Override
public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException {
- return new Lucene94HnswVectorsWriter(state, maxConn, beamWidth);
+ return new Lucene95HnswVectorsWriter(state, maxConn, beamWidth);
}
@Override
public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException {
- return new Lucene94HnswVectorsReader(state);
+ return new Lucene95HnswVectorsReader(state);
}
@Override
public String toString() {
- return "Lucene94HnswVectorsFormat(name=Lucene94HnswVectorsFormat, maxConn="
+ return "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn="
+ maxConn
+ ", beamWidth="
+ beamWidth
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsReader.java
similarity index 77%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsReader.java
index 8d95614f9fe..b89e9c634c7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsReader.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -40,6 +40,8 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
@@ -53,14 +55,17 @@ import org.apache.lucene.util.packed.DirectMonotonicReader;
*
* @lucene.experimental
*/
-public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
+public final class Lucene95HnswVectorsReader extends KnnVectorsReader {
+
+ private static final long SHALLOW_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(Lucene95HnswVectorsFormat.class);
private final FieldInfos fieldInfos;
private final Map<String, FieldEntry> fields = new HashMap<>();
private final IndexInput vectorData;
private final IndexInput vectorIndex;
- Lucene94HnswVectorsReader(SegmentReadState state) throws IOException {
+ Lucene95HnswVectorsReader(SegmentReadState state) throws IOException {
this.fieldInfos = state.fieldInfos;
int versionMeta = readMetadata(state);
boolean success = false;
@@ -69,14 +74,14 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
openDataInput(
state,
versionMeta,
- Lucene94HnswVectorsFormat.VECTOR_DATA_EXTENSION,
- Lucene94HnswVectorsFormat.VECTOR_DATA_CODEC_NAME);
+ Lucene95HnswVectorsFormat.VECTOR_DATA_EXTENSION,
+ Lucene95HnswVectorsFormat.VECTOR_DATA_CODEC_NAME);
vectorIndex =
openDataInput(
state,
versionMeta,
- Lucene94HnswVectorsFormat.VECTOR_INDEX_EXTENSION,
- Lucene94HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME);
+ Lucene95HnswVectorsFormat.VECTOR_INDEX_EXTENSION,
+ Lucene95HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME);
success = true;
} finally {
if (success == false) {
@@ -88,7 +93,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
private int readMetadata(SegmentReadState state) throws IOException {
String metaFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene94HnswVectorsFormat.META_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene95HnswVectorsFormat.META_EXTENSION);
int versionMeta = -1;
try (ChecksumIndexInput meta =
state.directory.openChecksumInput(metaFileName, IOContext.READONCE)) {
@@ -97,9 +102,9 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
versionMeta =
CodecUtil.checkIndexHeader(
meta,
- Lucene94HnswVectorsFormat.META_CODEC_NAME,
- Lucene94HnswVectorsFormat.VERSION_START,
- Lucene94HnswVectorsFormat.VERSION_CURRENT,
+ Lucene95HnswVectorsFormat.META_CODEC_NAME,
+ Lucene95HnswVectorsFormat.VERSION_START,
+ Lucene95HnswVectorsFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
readFields(meta, state.fieldInfos);
@@ -124,8 +129,8 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
CodecUtil.checkIndexHeader(
in,
codecName,
- Lucene94HnswVectorsFormat.VERSION_START,
- Lucene94HnswVectorsFormat.VERSION_CURRENT,
+ Lucene95HnswVectorsFormat.VERSION_START,
+ Lucene95HnswVectorsFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
if (versionMeta != versionVectorData) {
@@ -220,11 +225,9 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
@Override
public long ramBytesUsed() {
- long totalBytes = RamUsageEstimator.shallowSizeOfInstance(Lucene94HnswVectorsFormat.class);
- totalBytes +=
- RamUsageEstimator.sizeOfMap(
+ return Lucene95HnswVectorsReader.SHALLOW_SIZE
+ + RamUsageEstimator.sizeOfMap(
fields, RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class));
- return totalBytes;
}
@Override
@@ -299,9 +302,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
}
private HnswGraph getGraph(FieldEntry entry) throws IOException {
- IndexInput bytesSlice =
- vectorIndex.slice("graph-data", entry.vectorIndexOffset, entry.vectorIndexLength);
- return new OffHeapHnswGraph(entry, bytesSlice);
+ return new OffHeapHnswGraph(entry, vectorIndex);
}
@Override
@@ -309,8 +310,9 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
IOUtils.close(vectorData, vectorIndex);
}
- static class FieldEntry {
-
+ static class FieldEntry implements Accountable {
+ private static final long SHALLOW_SIZE =
+ RamUsageEstimator.shallowSizeOfInstance(FieldEntry.class);
final VectorSimilarityFunction similarityFunction;
final VectorEncoding vectorEncoding;
final long vectorDataOffset;
@@ -323,7 +325,10 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
final int size;
final int[][] nodesByLevel;
// for each level the start offsets in vectorIndex file from where to read neighbours
- final long[] graphOffsetsByLevel;
+ final DirectMonotonicReader.Meta offsetsMeta;
+ final long offsetsOffset;
+ final int offsetsBlockShift;
+ final long offsetsLength;
// the following four variables used to read docIds encoded by IndexDISI
// special values of docsWithFieldOffset are -1 and -2
@@ -353,7 +358,7 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
vectorDataLength = input.readVLong();
vectorIndexOffset = input.readVLong();
vectorIndexLength = input.readVLong();
- dimension = input.readInt();
+ dimension = input.readVInt();
size = input.readInt();
docsWithFieldOffset = input.readLong();
@@ -376,47 +381,47 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
}
// read nodes by level
- M = input.readInt();
- numLevels = input.readInt();
+ M = input.readVInt();
+ numLevels = input.readVInt();
nodesByLevel = new int[numLevels][];
+ long numberOfOffsets = 0;
for (int level = 0; level < numLevels; level++) {
- int numNodesOnLevel = input.readInt();
- if (level == 0) {
- // we don't store nodes for level 0th, as this level contains all nodes
- assert numNodesOnLevel == size;
- nodesByLevel[0] = null;
- } else {
+ if (level > 0) {
+ int numNodesOnLevel = input.readVInt();
+ numberOfOffsets += numNodesOnLevel;
nodesByLevel[level] = new int[numNodesOnLevel];
- for (int i = 0; i < numNodesOnLevel; i++) {
- nodesByLevel[level][i] = input.readInt();
+ nodesByLevel[level][0] = input.readVInt();
+ for (int i = 1; i < numNodesOnLevel; i++) {
+ nodesByLevel[level][i] = nodesByLevel[level][i - 1] + input.readVInt();
}
- }
- }
-
- // calculate for each level the start offsets in vectorIndex file from where to read
- // neighbours
- graphOffsetsByLevel = new long[numLevels];
- final long connectionsAndSizeLevel0Bytes =
- Math.multiplyExact(Math.addExact(1, Math.multiplyExact(M, 2L)), Integer.BYTES);
- final long connectionsAndSizeBytes = Math.multiplyExact(Math.addExact(1L, M), Integer.BYTES);
- for (int level = 0; level < numLevels; level++) {
- if (level == 0) {
- graphOffsetsByLevel[level] = 0;
- } else if (level == 1) {
- graphOffsetsByLevel[level] = connectionsAndSizeLevel0Bytes * size;
} else {
- int numNodesOnPrevLevel = nodesByLevel[level - 1].length;
- graphOffsetsByLevel[level] =
- Math.addExact(
- graphOffsetsByLevel[level - 1],
- Math.multiplyExact(connectionsAndSizeBytes, numNodesOnPrevLevel));
+ numberOfOffsets += size;
}
}
+ if (numberOfOffsets > 0) {
+ offsetsOffset = input.readLong();
+ offsetsBlockShift = input.readVInt();
+ offsetsMeta = DirectMonotonicReader.loadMeta(input, numberOfOffsets, offsetsBlockShift);
+ offsetsLength = input.readLong();
+ } else {
+ offsetsOffset = 0;
+ offsetsBlockShift = 0;
+ offsetsMeta = null;
+ offsetsLength = 0;
+ }
}
int size() {
return size;
}
+
+ @Override
+ public long ramBytesUsed() {
+ return SHALLOW_SIZE
+ + Arrays.stream(nodesByLevel).mapToLong(nodes -> RamUsageEstimator.sizeOf(nodes)).sum()
+ + RamUsageEstimator.sizeOf(meta)
+ + RamUsageEstimator.sizeOf(offsetsMeta);
+ }
}
/** Read the nearest-neighbors graph from the index input */
@@ -424,27 +429,36 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
final IndexInput dataIn;
final int[][] nodesByLevel;
- final long[] graphOffsetsByLevel;
final int numLevels;
final int entryNode;
final int size;
- final long bytesForConns;
- final long bytesForConns0;
-
int arcCount;
int arcUpTo;
int arc;
-
- OffHeapHnswGraph(FieldEntry entry, IndexInput dataIn) {
- this.dataIn = dataIn;
+ private final DirectMonotonicReader graphLevelNodeOffsets;
+ private final long[] graphLevelNodeIndexOffsets;
+ // Allocated to be M*2 to track the current neighbors being explored
+ private final int[] currentNeighborsBuffer;
+
+ OffHeapHnswGraph(FieldEntry entry, IndexInput vectorIndex) throws IOException {
+ this.dataIn =
+ vectorIndex.slice("graph-data", entry.vectorIndexOffset, entry.vectorIndexLength);
this.nodesByLevel = entry.nodesByLevel;
this.numLevels = entry.numLevels;
this.entryNode = numLevels > 1 ? nodesByLevel[numLevels - 1][0] : 0;
this.size = entry.size();
- this.graphOffsetsByLevel = entry.graphOffsetsByLevel;
- this.bytesForConns = Math.multiplyExact(Math.addExact(entry.M, 1L), Integer.BYTES);
- this.bytesForConns0 =
- Math.multiplyExact(Math.addExact(Math.multiplyExact(entry.M, 2L), 1), Integer.BYTES);
+ final RandomAccessInput addressesData =
+ vectorIndex.randomAccessSlice(entry.offsetsOffset, entry.offsetsLength);
+ this.graphLevelNodeOffsets =
+ DirectMonotonicReader.getInstance(entry.offsetsMeta, addressesData);
+ this.currentNeighborsBuffer = new int[entry.M * 2];
+ graphLevelNodeIndexOffsets = new long[numLevels];
+ graphLevelNodeIndexOffsets[0] = 0;
+ for (int i = 1; i < numLevels; i++) {
+ // nodesByLevel is `null` for the zeroth level as we know its size
+ int nodeCount = nodesByLevel[i - 1] == null ? size : nodesByLevel[i - 1].length;
+ graphLevelNodeIndexOffsets[i] = graphLevelNodeIndexOffsets[i - 1] + nodeCount;
+ }
}
@Override
@@ -454,11 +468,15 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
? targetOrd
: Arrays.binarySearch(nodesByLevel[level], 0, nodesByLevel[level].length, targetOrd);
assert targetIndex >= 0;
- long graphDataOffset =
- graphOffsetsByLevel[level] + targetIndex * (level == 0 ? bytesForConns0 : bytesForConns);
// unsafe; no bounds checking
- dataIn.seek(graphDataOffset);
- arcCount = dataIn.readInt();
+ dataIn.seek(graphLevelNodeOffsets.get(targetIndex + graphLevelNodeIndexOffsets[level]));
+ arcCount = dataIn.readVInt();
+ if (arcCount > 0) {
+ currentNeighborsBuffer[0] = dataIn.readVInt();
+ for (int i = 1; i < arcCount; i++) {
+ currentNeighborsBuffer[i] = currentNeighborsBuffer[i - 1] + dataIn.readVInt();
+ }
+ }
arc = -1;
arcUpTo = 0;
}
@@ -473,8 +491,8 @@ public final class Lucene94HnswVectorsReader extends KnnVectorsReader {
if (arcUpTo >= arcCount) {
return NO_MORE_DOCS;
}
+ arc = currentNeighborsBuffer[arcUpTo];
++arcUpTo;
- arc = dataIn.readInt();
return arc;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsWriter.java
similarity index 80%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsWriter.java
index 9ab3627b4ac..60ecf6f6472 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/Lucene94HnswVectorsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/Lucene95HnswVectorsWriter.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
-import static org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
+import static org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
import java.io.IOException;
@@ -30,22 +30,12 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.KnnFieldVectorsWriter;
import org.apache.lucene.codecs.KnnVectorsWriter;
import org.apache.lucene.codecs.lucene90.IndexedDISI;
-import org.apache.lucene.index.DocsWithFieldSet;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.MergeState;
-import org.apache.lucene.index.RandomAccessVectorValues;
-import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.*;
import org.apache.lucene.index.Sorter;
-import org.apache.lucene.index.VectorValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.InfoStream;
-import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.*;
import org.apache.lucene.util.hnsw.HnswGraph;
import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator;
import org.apache.lucene.util.hnsw.HnswGraphBuilder;
@@ -58,7 +48,7 @@ import org.apache.lucene.util.packed.DirectMonotonicWriter;
*
* @lucene.experimental
*/
-public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
+public final class Lucene95HnswVectorsWriter extends KnnVectorsWriter {
private final SegmentWriteState segmentWriteState;
private final IndexOutput meta, vectorData, vectorIndex;
@@ -68,26 +58,25 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
private final List<FieldWriter<?>> fields = new ArrayList<>();
private boolean finished;
- Lucene94HnswVectorsWriter(SegmentWriteState state, int M, int beamWidth) throws IOException {
+ Lucene95HnswVectorsWriter(SegmentWriteState state, int M, int beamWidth) throws IOException {
this.M = M;
this.beamWidth = beamWidth;
segmentWriteState = state;
String metaFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene94HnswVectorsFormat.META_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene95HnswVectorsFormat.META_EXTENSION);
String vectorDataFileName =
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene94HnswVectorsFormat.VECTOR_DATA_EXTENSION);
+ Lucene95HnswVectorsFormat.VECTOR_DATA_EXTENSION);
String indexDataFileName =
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene94HnswVectorsFormat.VECTOR_INDEX_EXTENSION);
-
+ Lucene95HnswVectorsFormat.VECTOR_INDEX_EXTENSION);
boolean success = false;
try {
meta = state.directory.createOutput(metaFileName, state.context);
@@ -96,20 +85,20 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
CodecUtil.writeIndexHeader(
meta,
- Lucene94HnswVectorsFormat.META_CODEC_NAME,
- Lucene94HnswVectorsFormat.VERSION_CURRENT,
+ Lucene95HnswVectorsFormat.META_CODEC_NAME,
+ Lucene95HnswVectorsFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
CodecUtil.writeIndexHeader(
vectorData,
- Lucene94HnswVectorsFormat.VECTOR_DATA_CODEC_NAME,
- Lucene94HnswVectorsFormat.VERSION_CURRENT,
+ Lucene95HnswVectorsFormat.VECTOR_DATA_CODEC_NAME,
+ Lucene95HnswVectorsFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
CodecUtil.writeIndexHeader(
vectorIndex,
- Lucene94HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME,
- Lucene94HnswVectorsFormat.VERSION_CURRENT,
+ Lucene95HnswVectorsFormat.VECTOR_INDEX_CODEC_NAME,
+ Lucene95HnswVectorsFormat.VERSION_CURRENT,
state.segmentInfo.getId(),
state.segmentSuffix);
success = true;
@@ -178,7 +167,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
// write graph
long vectorIndexOffset = vectorIndex.getFilePointer();
OnHeapHnswGraph graph = fieldData.getGraph();
- writeGraph(graph);
+ int[][] graphLevelNodeOffsets = writeGraph(graph);
long vectorIndexLength = vectorIndex.getFilePointer() - vectorIndexOffset;
writeMeta(
@@ -189,7 +178,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
vectorIndexOffset,
vectorIndexLength,
fieldData.docsWithField,
- graph);
+ graph,
+ graphLevelNodeOffsets);
}
private void writeFloat32Vectors(FieldWriter<?> fieldData) throws IOException {
@@ -246,7 +236,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
// write graph
long vectorIndexOffset = vectorIndex.getFilePointer();
OnHeapHnswGraph graph = fieldData.getGraph();
- HnswGraph mockGraph = reconstructAndWriteGraph(graph, ordMap, oldOrdMap);
+ int[][] graphLevelNodeOffsets = graph == null ? new int[0][] : new int[graph.numLevels()][];
+ HnswGraph mockGraph = reconstructAndWriteGraph(graph, ordMap, oldOrdMap, graphLevelNodeOffsets);
long vectorIndexLength = vectorIndex.getFilePointer() - vectorIndexOffset;
writeMeta(
@@ -257,7 +248,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
vectorIndexOffset,
vectorIndexLength,
newDocsWithField,
- mockGraph);
+ mockGraph,
+ graphLevelNodeOffsets);
}
private long writeSortedFloat32Vectors(FieldWriter<?> fieldData, int[] ordMap)
@@ -283,9 +275,21 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
return vectorDataOffset;
}
- // reconstruct graph substituting old ordinals with new ordinals
+ /**
+ * Reconstructs the graph given the old and new node ids.
+ *
+ * <p>Additionally, the graph node connections are written to the vectorIndex.
+ *
+ * @param graph The current on heap graph
+ * @param newToOldMap the new node ids indexed to the old node ids
+ * @param oldToNewMap the old node ids indexed to the new node ids
+ * @param levelNodeOffsets where to place the new offsets for the nodes in the vector index.
+ * @return The graph
+ * @throws IOException if writing to vectorIndex fails
+ */
private HnswGraph reconstructAndWriteGraph(
- OnHeapHnswGraph graph, int[] newToOldMap, int[] oldToNewMap) throws IOException {
+ OnHeapHnswGraph graph, int[] newToOldMap, int[] oldToNewMap, int[][] levelNodeOffsets)
+ throws IOException {
if (graph == null) return null;
List<int[]> nodesByLevel = new ArrayList<>(graph.numLevels());
@@ -294,10 +298,13 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
int maxOrd = graph.size();
int maxConnOnLevel = M * 2;
NodesIterator nodesOnLevel0 = graph.getNodesOnLevel(0);
+ levelNodeOffsets[0] = new int[nodesOnLevel0.size()];
while (nodesOnLevel0.hasNext()) {
int node = nodesOnLevel0.nextInt();
NeighborArray neighbors = graph.getNeighbors(0, newToOldMap[node]);
+ long offset = vectorIndex.getFilePointer();
reconstructAndWriteNeigbours(neighbors, oldToNewMap, maxConnOnLevel, maxOrd);
+ levelNodeOffsets[0][node] = Math.toIntExact(vectorIndex.getFilePointer() - offset);
}
maxConnOnLevel = M;
@@ -310,9 +317,14 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
}
Arrays.sort(newNodes);
nodesByLevel.add(newNodes);
+ levelNodeOffsets[level] = new int[newNodes.length];
+ int nodeOffsetIndex = 0;
for (int node : newNodes) {
NeighborArray neighbors = graph.getNeighbors(level, newToOldMap[node]);
+ long offset = vectorIndex.getFilePointer();
reconstructAndWriteNeigbours(neighbors, oldToNewMap, maxConnOnLevel, maxOrd);
+ levelNodeOffsets[level][nodeOffsetIndex++] =
+ Math.toIntExact(vectorIndex.getFilePointer() - offset);
}
}
return new HnswGraph() {
@@ -356,7 +368,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
NeighborArray neighbors, int[] oldToNewMap, int maxConnOnLevel, int maxOrd)
throws IOException {
int size = neighbors.size();
- vectorIndex.writeInt(size);
+ vectorIndex.writeVInt(size);
// Destructively modify; it's ok we are discarding it after this
int[] nnodes = neighbors.node();
@@ -364,15 +376,14 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
nnodes[i] = oldToNewMap[nnodes[i]];
}
Arrays.sort(nnodes, 0, size);
- for (int i = 0; i < size; i++) {
- int nnode = nnodes[i];
- assert nnode < maxOrd : "node too large: " + nnode + ">=" + maxOrd;
- vectorIndex.writeInt(nnode);
+ // Now that we have sorted, do delta encoding to minimize the required bits to store the
+ // information
+ for (int i = size - 1; i > 0; --i) {
+ assert nnodes[i] < maxOrd : "node too large: " + nnodes[i] + ">=" + maxOrd;
+ nnodes[i] -= nnodes[i - 1];
}
- // if number of connections < maxConn,
- // add bogus values up to maxConn to have predictable offsets
- for (int i = size; i < maxConnOnLevel; i++) {
- vectorIndex.writeInt(0);
+ for (int i = 0; i < size; i++) {
+ vectorIndex.writeVInt(nnodes[i]);
}
}
@@ -402,7 +413,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
long vectorDataLength = vectorData.getFilePointer() - vectorDataOffset;
long vectorIndexOffset = vectorIndex.getFilePointer();
// build the graph using the temporary vector data
- // we use Lucene94HnswVectorsReader.DenseOffHeapVectorValues for the graph construction
+ // we use Lucene95HnswVectorsReader.DenseOffHeapVectorValues for the graph construction
// doesn't need to know docIds
// TODO: separate random access vector values from DocIdSetIterator?
int byteSize = vectors.dimension() * fieldInfo.getVectorEncoding().byteSize;
@@ -410,6 +421,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
new OffHeapVectorValues.DenseOffHeapVectorValues(
vectors.dimension(), docsWithField.cardinality(), vectorDataInput, byteSize);
OnHeapHnswGraph graph = null;
+ int[][] vectorIndexNodeOffsets = null;
if (offHeapVectors.size() != 0) {
// build graph
HnswGraphBuilder<?> hnswGraphBuilder =
@@ -422,7 +434,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
HnswGraphBuilder.randSeed);
hnswGraphBuilder.setInfoStream(segmentWriteState.infoStream);
graph = hnswGraphBuilder.build(offHeapVectors.copy());
- writeGraph(graph);
+ vectorIndexNodeOffsets = writeGraph(graph);
}
long vectorIndexLength = vectorIndex.getFilePointer() - vectorIndexOffset;
writeMeta(
@@ -433,7 +445,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
vectorIndexOffset,
vectorIndexLength,
docsWithField,
- graph);
+ graph,
+ vectorIndexNodeOffsets);
success = true;
} finally {
IOUtils.close(vectorDataInput);
@@ -447,33 +460,44 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
}
}
- private void writeGraph(OnHeapHnswGraph graph) throws IOException {
- if (graph == null) return;
+ /**
+ * @param graph Write the graph in a compressed format
+ * @return The non-cumulative offsets for the nodes. Should be used to create cumulative offsets.
+ * @throws IOException if writing to vectorIndex fails
+ */
+ private int[][] writeGraph(OnHeapHnswGraph graph) throws IOException {
+ if (graph == null) return new int[0][0];
// write vectors' neighbours on each level into the vectorIndex file
int countOnLevel0 = graph.size();
+ int[][] offsets = new int[graph.numLevels()][];
for (int level = 0; level < graph.numLevels(); level++) {
- int maxConnOnLevel = level == 0 ? (M * 2) : M;
NodesIterator nodesOnLevel = graph.getNodesOnLevel(level);
+ offsets[level] = new int[nodesOnLevel.size()];
+ int nodeOffsetId = 0;
while (nodesOnLevel.hasNext()) {
int node = nodesOnLevel.nextInt();
NeighborArray neighbors = graph.getNeighbors(level, node);
int size = neighbors.size();
- vectorIndex.writeInt(size);
+ // Write size in VInt as the neighbors list is typically small
+ long offsetStart = vectorIndex.getFilePointer();
+ vectorIndex.writeVInt(size);
// Destructively modify; it's ok we are discarding it after this
int[] nnodes = neighbors.node();
Arrays.sort(nnodes, 0, size);
- for (int i = 0; i < size; i++) {
- int nnode = nnodes[i];
- assert nnode < countOnLevel0 : "node too large: " + nnode + ">=" + countOnLevel0;
- vectorIndex.writeInt(nnode);
+ // Now that we have sorted, do delta encoding to minimize the required bits to store the
+ // information
+ for (int i = size - 1; i > 0; --i) {
+ assert nnodes[i] < countOnLevel0 : "node too large: " + nnodes[i] + ">=" + countOnLevel0;
+ nnodes[i] -= nnodes[i - 1];
}
- // if number of connections < maxConn, add bogus values up to maxConn to have predictable
- // offsets
- for (int i = size; i < maxConnOnLevel; i++) {
- vectorIndex.writeInt(0);
+ for (int i = 0; i < size; i++) {
+ vectorIndex.writeVInt(nnodes[i]);
}
+ offsets[level][nodeOffsetId++] =
+ Math.toIntExact(vectorIndex.getFilePointer() - offsetStart);
}
}
+ return offsets;
}
private void writeMeta(
@@ -484,7 +508,8 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
long vectorIndexOffset,
long vectorIndexLength,
DocsWithFieldSet docsWithField,
- HnswGraph graph)
+ HnswGraph graph,
+ int[][] graphLevelNodeOffsets)
throws IOException {
meta.writeInt(field.number);
meta.writeInt(field.getVectorEncoding().ordinal());
@@ -493,7 +518,7 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
meta.writeVLong(vectorDataLength);
meta.writeVLong(vectorIndexOffset);
meta.writeVLong(vectorIndexLength);
- meta.writeInt(field.getVectorDimension());
+ meta.writeVInt(field.getVectorDimension());
// write docIDs
int count = docsWithField.cardinality();
@@ -535,22 +560,47 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
meta.writeLong(vectorData.getFilePointer() - start);
}
- meta.writeInt(M);
+ meta.writeVInt(M);
// write graph nodes on each level
if (graph == null) {
- meta.writeInt(0);
+ meta.writeVInt(0);
} else {
- meta.writeInt(graph.numLevels());
+ meta.writeVInt(graph.numLevels());
+ long valueCount = 0;
for (int level = 0; level < graph.numLevels(); level++) {
NodesIterator nodesOnLevel = graph.getNodesOnLevel(level);
- meta.writeInt(nodesOnLevel.size()); // number of nodes on a level
+ valueCount += nodesOnLevel.size();
if (level > 0) {
- while (nodesOnLevel.hasNext()) {
- int node = nodesOnLevel.nextInt();
- meta.writeInt(node); // list of nodes on a level
+ int[] nol = new int[nodesOnLevel.size()];
+ int numberConsumed = nodesOnLevel.consume(nol);
+ assert numberConsumed == nodesOnLevel.size();
+ meta.writeVInt(nol.length); // number of nodes on a level
+ for (int i = nodesOnLevel.size() - 1; i > 0; --i) {
+ nol[i] -= nol[i - 1];
}
+ for (int n : nol) {
+ assert n >= 0 : "delta encoding for nodes failed; expected nodes to be sorted";
+ meta.writeVInt(n);
+ }
+ } else {
+ assert nodesOnLevel.size() == count : "Level 0 expects to have all nodes";
+ }
+ }
+ long start = vectorIndex.getFilePointer();
+ meta.writeLong(start);
+ meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
+ final DirectMonotonicWriter memoryOffsetsWriter =
+ DirectMonotonicWriter.getInstance(
+ meta, vectorIndex, valueCount, DIRECT_MONOTONIC_BLOCK_SHIFT);
+ long cumulativeOffsetSum = 0;
+ for (int[] levelOffsets : graphLevelNodeOffsets) {
+ for (int v : levelOffsets) {
+ memoryOffsetsWriter.add(cumulativeOffsetSum);
+ cumulativeOffsetSum += v;
}
}
+ memoryOffsetsWriter.finish();
+ meta.writeLong(vectorIndex.getFilePointer() - start);
}
}
@@ -658,11 +708,12 @@ public final class Lucene94HnswVectorsWriter extends KnnVectorsWriter {
@Override
public long ramBytesUsed() {
if (vectors.size() == 0) return 0;
- long vectorSize = vectors.size();
return docsWithField.ramBytesUsed()
- + vectorSize
+ + (long) vectors.size()
* (RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER)
- + vectorSize * fieldInfo.getVectorDimension() * fieldInfo.getVectorEncoding().byteSize
+ + (long) vectors.size()
+ * fieldInfo.getVectorDimension()
+ * fieldInfo.getVectorEncoding().byteSize
+ hnswGraphBuilder.getGraph().ramBytesUsed();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapVectorValues.java
similarity index 97%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapVectorValues.java
index 23df75fa022..92e052329c9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/OffHeapVectorValues.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/OffHeapVectorValues.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -80,7 +80,7 @@ abstract class OffHeapVectorValues extends VectorValues implements RandomAccessV
public abstract int ordToDoc(int ord);
static OffHeapVectorValues load(
- Lucene94HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
+ Lucene95HnswVectorsReader.FieldEntry fieldEntry, IndexInput vectorData) throws IOException {
if (fieldEntry.docsWithFieldOffset == -2) {
return new EmptyOffHeapVectorValues(fieldEntry.dimension);
}
@@ -163,10 +163,10 @@ abstract class OffHeapVectorValues extends VectorValues implements RandomAccessV
private final IndexedDISI disi;
// dataIn was used to init a new IndexedDIS for #randomAccess()
private final IndexInput dataIn;
- private final Lucene94HnswVectorsReader.FieldEntry fieldEntry;
+ private final Lucene95HnswVectorsReader.FieldEntry fieldEntry;
public SparseOffHeapVectorValues(
- Lucene94HnswVectorsReader.FieldEntry fieldEntry,
+ Lucene95HnswVectorsReader.FieldEntry fieldEntry,
IndexInput dataIn,
IndexInput slice,
int byteSize)
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/package-info.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
copy to lucene/core/src/java/org/apache/lucene/codecs/lucene95/package-info.java
index 84956d8ec0e..370a6edc103 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene94/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene95/package-info.java
@@ -16,7 +16,7 @@
*/
/**
- * Lucene 9.3 file format.
+ * Lucene 9.5 file format.
*
* <h2>Apache Lucene - Index File Formats</h2>
*
@@ -180,7 +180,7 @@
* of files, recording dimensionally indexed fields, to enable fast numeric range filtering
* and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
* intersection (2D, 3D).
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}. The
+ * <li>{@link org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat Vector values}. The
* vector format stores numeric vectors in a format optimized for random access and
* computation, supporting high-dimensional nearest-neighbor search.
* </ul>
@@ -310,7 +310,7 @@
* <td>Holds indexed points</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat Vector values}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat Vector values}</td>
* <td>.vec, .vem</td>
* <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data, and
* <code>.vem</code> the vector metadata</td>
@@ -405,6 +405,9 @@
* <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.
* <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by
* IndexDISI. ordToDoc mappings was added to .vem.
+ * <li>In version 9.5, HNSW graph connections were changed to be delta-encoded with vints.
+ * Additionally, metadata file size improvements were made by delta-encoding nodes by graph
+ * layer and not writing the node ids for the zeroth layer.
* </ul>
*
* <a id="Limitations"></a>
@@ -419,4 +422,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
diff --git a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java
index 2a222d23e07..2ae89e9c0c5 100644
--- a/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java
+++ b/lucene/core/src/java/org/apache/lucene/util/hnsw/HnswGraph.java
@@ -144,6 +144,28 @@ public abstract class HnswGraph {
this.size = size;
}
+ /**
+ * Consume integers from the iterator and place them into the `dest` array.
+ *
+ * @param dest where to put the integers
+ * @return The number of integers written to `dest`
+ */
+ public int consume(int[] dest) {
+ if (hasNext() == false) {
+ throw new NoSuchElementException();
+ }
+ int numToCopy = Math.min(size - cur, dest.length);
+ if (nodes == null) {
+ for (int i = 0; i < numToCopy; i++) {
+ dest[i] = cur + i;
+ }
+ return numToCopy;
+ }
+ System.arraycopy(nodes, cur, dest, 0, numToCopy);
+ cur += numToCopy;
+ return numToCopy;
+ }
+
@Override
public int nextInt() {
if (hasNext() == false) {
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 11b662ce3a6..946e496ab94 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene94.Lucene94Codec
+org.apache.lucene.codecs.lucene95.Lucene95Codec
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
index 5327e21d5fe..ecbe03ee497 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat
+org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
index 23f4c3d4343..ec72b6582a6 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
@@ -18,8 +18,8 @@ package org.apache.lucene.codecs.lucene90;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DirectoryReader;
@@ -31,7 +31,7 @@ import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene94Codec(Mode.BEST_COMPRESSION);
+ return new Lucene95Codec(Mode.BEST_COMPRESSION);
}
/**
@@ -41,7 +41,8 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
Directory dir = newDirectory();
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
- iwc.setCodec(new Lucene94Codec(RandomPicks.randomFrom(random(), Mode.values())));
+ iwc.setCodec(
+ new Lucene95Codec(RandomPicks.randomFrom(random(), Lucene95Codec.Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -70,7 +71,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
expectThrows(
NullPointerException.class,
() -> {
- new Lucene94Codec(null);
+ new Lucene95Codec(null);
});
expectThrows(
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestLucene95HnswVectorsFormat.java
similarity index 61%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestLucene95HnswVectorsFormat.java
index 3590d3c168d..1dd7c0de553 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene94/TestLucene94HnswVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene95/TestLucene95HnswVectorsFormat.java
@@ -14,29 +14,38 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene94;
+package org.apache.lucene.codecs.lucene95;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
-public class TestLucene94HnswVectorsFormat extends BaseKnnVectorsFormatTestCase {
+public class TestLucene95HnswVectorsFormat extends BaseKnnVectorsFormatTestCase {
@Override
protected Codec getCodec() {
return TestUtil.getDefaultCodec();
}
public void testToString() {
- Lucene94Codec customCodec =
- new Lucene94Codec() {
+ Lucene95Codec customCodec =
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(10, 20);
+ return new Lucene95HnswVectorsFormat(10, 20);
}
};
String expectedString =
- "Lucene94HnswVectorsFormat(name=Lucene94HnswVectorsFormat, maxConn=10, beamWidth=20)";
+ "Lucene95HnswVectorsFormat(name=Lucene95HnswVectorsFormat, maxConn=10, beamWidth=20)";
assertEquals(expectedString, customCodec.getKnnVectorsFormatForField("bogus_field").toString());
}
+
+ public void testLimits() {
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(-1, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(0, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(20, 0));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(20, -1));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(512 + 1, 20));
+ expectThrows(IllegalArgumentException.class, () -> new Lucene95HnswVectorsFormat(20, 3201));
+ }
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
index c17217f4a1f..4f78bdff088 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestKnnGraph.java
@@ -30,9 +30,9 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsReader;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -64,7 +64,7 @@ public class TestKnnGraph extends LuceneTestCase {
private static final String KNN_GRAPH_FIELD = "vector";
- private static int M = Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
+ private static int M = Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN;
private Codec codec;
private Codec float32Codec;
@@ -79,10 +79,10 @@ public class TestKnnGraph extends LuceneTestCase {
}
codec =
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
+ return new Lucene95HnswVectorsFormat(M, Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
}
};
@@ -91,10 +91,10 @@ public class TestKnnGraph extends LuceneTestCase {
vectorEncoding = randomVectorEncoding();
codec =
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
+ return new Lucene95HnswVectorsFormat(M, Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
}
};
@@ -102,10 +102,10 @@ public class TestKnnGraph extends LuceneTestCase {
float32Codec = codec;
} else {
float32Codec =
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, Lucene94HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
+ return new Lucene95HnswVectorsFormat(M, Lucene95HnswVectorsFormat.DEFAULT_BEAM_WIDTH);
}
};
}
@@ -117,7 +117,7 @@ public class TestKnnGraph extends LuceneTestCase {
@After
public void cleanup() {
- M = Lucene94HnswVectorsFormat.DEFAULT_MAX_CONN;
+ M = Lucene95HnswVectorsFormat.DEFAULT_MAX_CONN;
}
/** Basic test of creating documents in a graph */
@@ -265,8 +265,8 @@ public class TestKnnGraph extends LuceneTestCase {
PerFieldKnnVectorsFormat.FieldsReader perFieldReader =
(PerFieldKnnVectorsFormat.FieldsReader)
((CodecReader) getOnlyLeafReader(reader)).getVectorReader();
- Lucene94HnswVectorsReader vectorReader =
- (Lucene94HnswVectorsReader) perFieldReader.getFieldReader(KNN_GRAPH_FIELD);
+ Lucene95HnswVectorsReader vectorReader =
+ (Lucene95HnswVectorsReader) perFieldReader.getFieldReader(KNN_GRAPH_FIELD);
graph = copyGraph(vectorReader.getGraph(KNN_GRAPH_FIELD));
}
}
@@ -474,8 +474,8 @@ public class TestKnnGraph extends LuceneTestCase {
if (perFieldReader == null) {
continue;
}
- Lucene94HnswVectorsReader vectorReader =
- (Lucene94HnswVectorsReader) perFieldReader.getFieldReader(vectorField);
+ Lucene95HnswVectorsReader vectorReader =
+ (Lucene95HnswVectorsReader) perFieldReader.getFieldReader(vectorField);
if (vectorReader == null) {
continue;
}
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/KnnGraphTester.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/KnnGraphTester.java
index 95961ceda64..dd44898af75 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/KnnGraphTester.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/KnnGraphTester.java
@@ -39,9 +39,9 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsReader;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
@@ -295,7 +295,7 @@ public class KnnGraphTester {
KnnVectorsReader vectorsReader =
((PerFieldKnnVectorsFormat.FieldsReader) ((CodecReader) leafReader).getVectorReader())
.getFieldReader(KNN_FIELD);
- HnswGraph knnValues = ((Lucene94HnswVectorsReader) vectorsReader).getGraph(KNN_FIELD);
+ HnswGraph knnValues = ((Lucene95HnswVectorsReader) vectorsReader).getGraph(KNN_FIELD);
System.out.printf("Leaf %d has %d documents\n", context.ord, leafReader.maxDoc());
printGraphFanout(knnValues, leafReader.maxDoc());
}
@@ -691,10 +691,10 @@ public class KnnGraphTester {
private int createIndex(Path docsPath, Path indexPath) throws IOException {
IndexWriterConfig iwc = new IndexWriterConfig().setOpenMode(IndexWriterConfig.OpenMode.CREATE);
iwc.setCodec(
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(maxConn, beamWidth);
+ return new Lucene95HnswVectorsFormat(maxConn, beamWidth);
}
});
// iwc.setMergePolicy(NoMergePolicy.INSTANCE);
diff --git a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
index 1fe585da8cb..03f13195aa2 100644
--- a/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
+++ b/lucene/core/src/test/org/apache/lucene/util/hnsw/TestHnswGraph.java
@@ -31,9 +31,9 @@ import java.util.List;
import java.util.Random;
import java.util.Set;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsReader;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsReader;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnVectorField;
@@ -101,10 +101,10 @@ public class TestHnswGraph extends LuceneTestCase {
IndexWriterConfig iwc =
new IndexWriterConfig()
.setCodec(
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, beamWidth);
+ return new Lucene95HnswVectorsFormat(M, beamWidth);
}
});
try (IndexWriter iw = new IndexWriter(dir, iwc)) {
@@ -131,7 +131,7 @@ public class TestHnswGraph extends LuceneTestCase {
assertEquals(indexedDoc, ctx.reader().numDocs());
assertVectorsEqual(v3, values);
HnswGraph graphValues =
- ((Lucene94HnswVectorsReader)
+ ((Lucene95HnswVectorsReader)
((PerFieldKnnVectorsFormat.FieldsReader)
((CodecReader) ctx.reader()).getVectorReader())
.getFieldReader("field"))
@@ -161,19 +161,19 @@ public class TestHnswGraph extends LuceneTestCase {
IndexWriterConfig iwc =
new IndexWriterConfig()
.setCodec(
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, beamWidth);
+ return new Lucene95HnswVectorsFormat(M, beamWidth);
}
});
IndexWriterConfig iwc2 =
new IndexWriterConfig()
.setCodec(
- new Lucene94Codec() {
+ new Lucene95Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return new Lucene94HnswVectorsFormat(M, beamWidth);
+ return new Lucene95HnswVectorsFormat(M, beamWidth);
}
})
.setIndexSort(new Sort(new SortField("sortkey", SortField.Type.LONG)));
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
index 77d10d8de0f..c8d95995401 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
@@ -40,7 +40,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntPoint;
@@ -959,7 +959,7 @@ public class TestSuggestField extends LuceneTestCase {
IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
iwc.setMergePolicy(newLogMergePolicy());
Codec filterCodec =
- new Lucene94Codec() {
+ new Lucene95Codec() {
CompletionPostingsFormat.FSTLoadMode fstLoadMode =
RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
PostingsFormat postingsFormat = new Completion90PostingsFormat(fstLoadMode);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
index 89d0e6a970d..5e88b866c46 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/vector/ConfigurableMCodec.java
@@ -18,8 +18,8 @@ package org.apache.lucene.tests.codecs.vector;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
/**
* This codec allows customization of the number of connections made for an hnsw index. Increasing
@@ -31,13 +31,13 @@ public class ConfigurableMCodec extends FilterCodec {
private final KnnVectorsFormat knnVectorsFormat;
public ConfigurableMCodec() {
- super("ConfigurableMCodec", new Lucene94Codec());
- knnVectorsFormat = new Lucene94HnswVectorsFormat(128, 100);
+ super("ConfigurableMCodec", new Lucene95Codec());
+ knnVectorsFormat = new Lucene95HnswVectorsFormat(128, 100);
}
public ConfigurableMCodec(int maxConn) {
- super("ConfigurableMCodec", new Lucene94Codec());
- knnVectorsFormat = new Lucene94HnswVectorsFormat(maxConn, 100);
+ super("ConfigurableMCodec", new Lucene95Codec());
+ knnVectorsFormat = new Lucene95HnswVectorsFormat(maxConn, 100);
}
@Override
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
index f467d4a2cae..49881465ec2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
@@ -38,7 +38,7 @@ import java.util.TimeZone;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
@@ -195,9 +195,9 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {
} else if ("Compressing".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {
codec = CompressingCodec.randomInstance(random);
- } else if ("Lucene94".equals(TEST_CODEC)
- || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene94"))) {
- codec = new Lucene94Codec(RandomPicks.randomFrom(random, Lucene94Codec.Mode.values()));
+ } else if ("Lucene95".equals(TEST_CODEC)
+ || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene95"))) {
+ codec = new Lucene95Codec(RandomPicks.randomFrom(random, Lucene95Codec.Mode.values()));
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
index a6a7d9c1e4e..dc08e37f095 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
@@ -55,8 +55,8 @@ import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene90.Lucene90PostingsFormat;
-import org.apache.lucene.codecs.lucene94.Lucene94Codec;
-import org.apache.lucene.codecs.lucene94.Lucene94HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene95.Lucene95Codec;
+import org.apache.lucene.codecs.lucene95.Lucene95HnswVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -1236,7 +1236,7 @@ public final class TestUtil {
* different than {@link Codec#getDefault()} because that is randomized.
*/
public static Codec getDefaultCodec() {
- return new Lucene94Codec();
+ return new Lucene95Codec();
}
/**
@@ -1322,7 +1322,7 @@ public final class TestUtil {
* Lucene.
*/
public static KnnVectorsFormat getDefaultKnnVectorsFormat() {
- return new Lucene94HnswVectorsFormat();
+ return new Lucene95HnswVectorsFormat();
}
public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {