You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/06/25 00:19:44 UTC
(pinot) branch master updated: [index] Add json creator initializer for VectorIndexConfig (#16191)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a070bc0ce4 [index] Add json creator initializer for VectorIndexConfig (#16191)
a070bc0ce4 is described below
commit a070bc0ce4dc68a20b9860c50b840667367a1b26
Author: Anshul Singh <an...@gmail.com>
AuthorDate: Wed Jun 25 05:49:38 2025 +0530
[index] Add json creator initializer for VectorIndexConfig (#16191)
---
.../segment/local/utils/TableConfigUtils.java | 3 +-
.../segment/local/segment/index/H3IndexTest.java | 29 +++++
.../segment/local/segment/index/JsonIndexTest.java | 4 +-
.../local/segment/index/RangeIndexTest.java | 5 +-
.../segment/local/segment/index/TextIndexTest.java | 5 +-
.../{TextIndexTest.java => VectorIndexTest.java} | 45 ++++---
.../spi/index/creator/VectorIndexConfig.java | 17 +++
.../pinot/segment/spi/index/VectorConfigTest.java | 145 +++++++++++++++++++++
8 files changed, 231 insertions(+), 22 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index 3901c4e5a8..05eab7e8d5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -1456,7 +1456,8 @@ public final class TableConfigUtils {
if (clone.getFieldConfigList() != null) {
List<FieldConfig> cleanFieldConfigList = new ArrayList<>();
for (FieldConfig fieldConfig : clone.getFieldConfigList()) {
- cleanFieldConfigList.add(new FieldConfig.Builder(fieldConfig).withIndexTypes(null).build());
+ cleanFieldConfigList.add(new FieldConfig.Builder(fieldConfig)
+ .withIndexTypes(null).withProperties(null).build());
}
clone.setFieldConfigList(cleanFieldConfigList);
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
index db6b2337e9..d7e027d426 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.segment.index;
import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
@@ -52,8 +53,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.apache.pinot.spi.config.table.FieldConfig.EncodingType.RAW;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
@@ -206,5 +209,31 @@ public class H3IndexTest implements PinotBuffersAfterMethodCheckRule {
assertNotNull(fieldConfig.getIndexes().get(H3IndexType.INDEX_DISPLAY_NAME));
assertTrue(fieldConfig.getIndexTypes().isEmpty());
}
+
+ @Test
+ public void testConvertToUpdatedFormat()
+ throws IOException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"location_st_point\",\n"
+ + " \"encodingType\": \"RAW\",\n"
+ + " \"indexTypes\": [\n"
+ + " \"H3\"\n"
+ + " ],\n"
+ + " \"properties\": {\n"
+ + " \"resolutions\": \"13,5,6\"\n"
+ + " }\n"
+ + "}");
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("location_st_point"))
+ .collect(Collectors.toList()).get(0);
+ Assert.assertEquals(fieldConfig.getEncodingType(), RAW);
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(fieldConfig.getProperties());
+ JsonNode node = fieldConfig.getIndexes().get(H3IndexType.INDEX_DISPLAY_NAME);
+ Assert.assertEquals(node.toString(), "{\"disabled\":false,\"resolution\":[5,6,13]}");
+ }
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
index dc0b8bc31e..8f9b477ae2 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
@@ -974,9 +974,11 @@ public class JsonIndexTest implements PinotBuffersAfterMethodCheckRule {
FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
.filter(fc -> fc.getName().equals("dimStr"))
.collect(Collectors.toList()).get(0);
- assertNotNull(fieldConfig.getIndexes().get(JsonIndexType.INDEX_DISPLAY_NAME));
assertNull(_tableConfig.getIndexingConfig().getJsonIndexColumns());
assertNull(_tableConfig.getIndexingConfig().getJsonIndexConfigs());
+ assertNotNull(fieldConfig.getIndexes().get(JsonIndexType.INDEX_DISPLAY_NAME));
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(fieldConfig.getProperties());
}
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java
index 0308956871..ca3ab4259e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java
@@ -31,7 +31,8 @@ import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
-
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
public class RangeIndexTest {
public static class ConfTest extends AbstractSerdeIndexContract {
@@ -47,6 +48,8 @@ public class RangeIndexTest {
JsonNode indexConfig = fieldConfig.getIndexes().get(RangeIndexType.INDEX_DISPLAY_NAME);
assertNotNull(indexConfig);
assertEquals(RangeIndexConfig.DEFAULT.getVersion(), indexConfig.get("version").asInt());
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(fieldConfig.getProperties());
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
index 918de11390..4d49d51ebf 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
@@ -30,7 +30,8 @@ import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
-
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
public class TextIndexTest {
public static class ConfTest extends AbstractSerdeIndexContract {
@@ -50,6 +51,8 @@ public class TextIndexTest {
JsonNode indexConfig = fieldConfig.getIndexes().get(TextIndexType.INDEX_DISPLAY_NAME);
assertNotNull(indexConfig);
assertFalse(indexConfig.get("disabled").asBoolean());
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(fieldConfig.getProperties());
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/VectorIndexTest.java
similarity index 54%
copy from pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
copy to pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/VectorIndexTest.java
index 918de11390..9775e3ca7a 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/VectorIndexTest.java
@@ -21,41 +21,50 @@ package org.apache.pinot.segment.local.segment.index;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.stream.Collectors;
-import org.apache.pinot.segment.local.segment.index.text.TextIndexPlugin;
-import org.apache.pinot.segment.local.segment.index.text.TextIndexType;
-import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.segment.local.segment.index.vector.VectorIndexType;
import org.apache.pinot.spi.config.table.FieldConfig;
+import org.testng.Assert;
import org.testng.annotations.Test;
-import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
-
-public class TextIndexTest {
+public class VectorIndexTest {
public static class ConfTest extends AbstractSerdeIndexContract {
@Test
- public void oldToNewConfConversion()
+ public void testConvertToUpdatedFormat()
throws JsonProcessingException {
addFieldIndexConfig("{\n"
- + " \"name\": \"dimStr\",\n"
- + " \"indexTypes\" : [\"TEXT\"]\n"
- + " }");
+ + " \"name\": \"studentID\",\n"
+ + " \"encodingType\": \"DICTIONARY\",\n"
+ + " \"indexTypes\": [\n"
+ + " \"VECTOR\"\n"
+ + " ],\n"
+ + " \"properties\": {\n"
+ + " \"vectorIndexType\": \"HNSW\",\n"
+ + " \"vectorDimension\": 1536,\n"
+ + " \"vectorDistanceFunction\": \"COSINE\",\n"
+ + " \"version\": 1\n"
+ + " },\n"
+ + " \"tierOverwrites\": null\n"
+ + "}");
convertToUpdatedFormat();
assertNotNull(_tableConfig.getFieldConfigList());
assertFalse(_tableConfig.getFieldConfigList().isEmpty());
FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
- .filter(fc -> fc.getName().equals("dimStr")).collect(Collectors.toList()).get(0);
- JsonNode indexConfig = fieldConfig.getIndexes().get(TextIndexType.INDEX_DISPLAY_NAME);
+ .filter(fc -> fc.getName().equals("studentID")).collect(Collectors.toList()).get(0);
+ JsonNode indexConfig = fieldConfig.getIndexes().get(VectorIndexType.INDEX_DISPLAY_NAME);
assertNotNull(indexConfig);
assertFalse(indexConfig.get("disabled").asBoolean());
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(fieldConfig.getProperties());
+ Assert.assertEquals(indexConfig.toString(),
+ "{\"disabled\":false,\"vectorIndexType\":\"HNSW\",\"vectorDimension\":1536,"
+ + "\"version\":1,\"vectorDistanceFunction\":\"COSINE\",\"properties\":{\"vectorIndexType\":"
+ + "\"HNSW\",\"vectorDimension\":\"1536\",\"vectorDistanceFunction\":\"COSINE\",\"version\":\"1\"}}");
}
}
-
- @Test
- public void testStandardIndex() {
- assertEquals(StandardIndexes.text(), new TextIndexPlugin().getIndexType(),
- "Standard index should be equal to the instance returned by the plugin");
- }
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexConfig.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexConfig.java
index 65efd841bb..595e5b3ba7 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexConfig.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/creator/VectorIndexConfig.java
@@ -18,6 +18,8 @@
*/
package org.apache.pinot.segment.spi.index.creator;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import java.util.Map;
import javax.annotation.Nullable;
@@ -68,6 +70,21 @@ public class VectorIndexConfig extends IndexConfig {
_properties = properties;
}
+ @JsonCreator
+ public VectorIndexConfig(@JsonProperty("disabled") @Nullable Boolean disabled,
+ @JsonProperty("vectorIndexType") @Nullable String vectorIndexType,
+ @JsonProperty("vectorDimension") int vectorDimension,
+ @JsonProperty("version") int version,
+ @JsonProperty("vectorDistanceFunction") @Nullable VectorDistanceFunction vectorDistanceFunction,
+ @JsonProperty("properties") @Nullable Map<String, String> properties) {
+ super(disabled);
+ _vectorIndexType = vectorIndexType;
+ _vectorDimension = vectorDimension;
+ _version = version;
+ _vectorDistanceFunction = vectorDistanceFunction;
+ _properties = properties;
+ }
+
public String getVectorIndexType() {
return _vectorIndexType;
}
diff --git a/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/VectorConfigTest.java b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/VectorConfigTest.java
new file mode 100644
index 0000000000..904422fa29
--- /dev/null
+++ b/pinot-segment-spi/src/test/java/org/apache/pinot/segment/spi/index/VectorConfigTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.spi.index;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import java.util.Map;
+import org.apache.pinot.segment.spi.index.creator.VectorIndexConfig;
+import org.apache.pinot.spi.utils.JsonUtils;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import static org.apache.pinot.segment.spi.index.creator.VectorIndexConfig.VectorDistanceFunction.COSINE;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+public class VectorConfigTest {
+
+ @Test
+ public void withEmptyConf()
+ throws JsonProcessingException {
+ String confStr = "{}";
+ VectorIndexConfig config = JsonUtils.stringToObject(confStr, VectorIndexConfig.class);
+
+ assertFalse(config.isDisabled(), "Unexpected disabled");
+ assertNull(config.getVectorIndexType(), "Unexpected vectorIndexType");
+ assertNull(config.getVectorDistanceFunction(), "Unexpected vectorDistanceFunction");
+ assertNull(config.getProperties(), "unexpected properties");
+ assertEquals(config.getVectorDimension(), 0);
+ assertEquals(config.getVersion(), 0);
+ }
+
+ @Test
+ public void withDisabledNull()
+ throws JsonProcessingException {
+ String confStr = "{\"disabled\": null}";
+ VectorIndexConfig config = JsonUtils.stringToObject(confStr, VectorIndexConfig.class);
+
+ assertFalse(config.isDisabled(), "Unexpected disabled");
+ assertNull(config.getVectorIndexType(), "Unexpected vectorIndexType");
+ assertNull(config.getVectorDistanceFunction(), "Unexpected vectorDistanceFunction");
+ assertNull(config.getProperties(), "unexpected properties");
+ assertEquals(config.getVectorDimension(), 0);
+ assertEquals(config.getVersion(), 0);
+ }
+
+ @Test
+ public void withDisabledFalse()
+ throws JsonProcessingException {
+ String confStr = "{\"disabled\": false}";
+ VectorIndexConfig config = JsonUtils.stringToObject(confStr, VectorIndexConfig.class);
+
+ assertFalse(config.isDisabled(), "Unexpected disabled");
+ assertNull(config.getVectorIndexType(), "Unexpected vectorIndexType");
+ assertNull(config.getVectorDistanceFunction(), "Unexpected vectorDistanceFunction");
+ assertNull(config.getProperties(), "unexpected properties");
+ assertEquals(config.getVectorDimension(), 0);
+ assertEquals(config.getVersion(), 0);
+ }
+
+ @Test
+ public void withDisabledTrue()
+ throws JsonProcessingException {
+ String confStr = "{\"disabled\": true}";
+ VectorIndexConfig config = JsonUtils.stringToObject(confStr, VectorIndexConfig.class);
+
+ assertTrue(config.isDisabled(), "Unexpected disabled");
+ assertNull(config.getVectorIndexType(), "Unexpected vectorIndexType");
+ assertNull(config.getVectorDistanceFunction(), "Unexpected vectorDistanceFunction");
+ assertNull(config.getProperties(), "unexpected properties");
+ assertEquals(config.getVectorDimension(), 0);
+ assertEquals(config.getVersion(), 0);
+ }
+
+ @Test
+ public void withSomeData()
+ throws JsonProcessingException {
+ String confStr = "{\n"
+ + " \"version\": 1,\n"
+ + " \"properties\": {\n"
+ + " \"vectorIndexType\": \"HNSW\",\n"
+ + " \"vectorDimension\": \"1536\",\n"
+ + " \"vectorDistanceFunction\": \"COSINE\",\n"
+ + " \"version\": \"1\"\n"
+ + " },\n"
+ + " \"vectorDistanceFunction\": \"COSINE\",\n"
+ + " \"vectorIndexType\": \"HNSW\",\n"
+ + " \"vectorDimension\": 1536,\n"
+ + " \"disabled\": false\n"
+ + "}";
+ VectorIndexConfig config = JsonUtils.stringToObject(confStr, VectorIndexConfig.class);
+
+ assertFalse(config.isDisabled(), "Unexpected disabled");
+ assertEquals(config.getVectorDimension(), 1536);
+ assertEquals(config.getVersion(), 1);
+ assertEquals(config.getVectorIndexType(), "HNSW");
+ assertEquals(config.getVectorDistanceFunction(), COSINE);
+
+ Map<String, String> properties = config.getProperties();
+ assertEquals(properties.getOrDefault("vectorDimension", "0"), "1536");
+ assertEquals(properties.getOrDefault("version", "0"), "1");
+ assertEquals(properties.getOrDefault("vectorIndexType", ""), "HNSW");
+ assertEquals(properties.getOrDefault("vectorDistanceFunction", ""), COSINE.toString());
+ }
+
+ @Test
+ public void serde()
+ throws JsonProcessingException {
+ VectorIndexConfig initialConf = new VectorIndexConfig(false);
+
+ String confAsJson = JsonUtils.objectToString(initialConf);
+ Assert.assertEquals(confAsJson, "{\"disabled\":false,\"vectorDimension\":0,\"version\":0}");
+
+ VectorIndexConfig readConf = JsonUtils.stringToObject(confAsJson, VectorIndexConfig.class);
+ Assert.assertEquals(readConf, initialConf, "Unexpected configuration after serialization and deserialization");
+ }
+
+ @Test
+ public void deserializeShort()
+ throws JsonProcessingException {
+ VectorIndexConfig initialConf = new VectorIndexConfig(false);
+
+ String serialized = "{\"disabled\":false,\"vectorDimension\":0,\"version\":0}";
+ VectorIndexConfig vectorIndexConfig = JsonUtils.stringToObject(serialized, VectorIndexConfig.class);
+
+ Assert.assertEquals(vectorIndexConfig, initialConf, "Unexpected configuration after reading " + serialized);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org