You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by sa...@apache.org on 2023/04/21 10:25:50 UTC
[pinot] branch master updated: Utility to convert table config into updated format (#10623)
This is an automated email from the ASF dual-hosted git repository.
saurabhd336 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4481e1b1ab Utility to convert table config into updated format (#10623)
4481e1b1ab is described below
commit 4481e1b1ab44e6fac53049a4d6612da79be0b92b
Author: Shounak kulkarni <sh...@gmail.com>
AuthorDate: Fri Apr 21 15:55:41 2023 +0530
Utility to convert table config into updated format (#10623)
* new method in FieldIndexConfigsUtil to convert into new table config format
* Expose a flag on the tableConfig endpoint to get the payload in newer format
* checkstyle fixes
* refactored code not compatible with java 8
* moved the utility to TableConfigUtils
* rename flag to updatedFormat
* pulled out the INDEX_DISPLAY_NAME as a constant
* moved createTableConfigFromOldFormat
* Test cases on each index type to validate the conversion
Test are only added to the index types which already had the AbstractSerdeIndexContract tests. Test for other indexes will be added in next commit.
* changes based on upstream changes
* minor code cleanup on existing index tests
* added ConfTest to remaining index types
* created handleIndexSpecificCleanup to handle index type specific cleanup of old configs
* revert the flag introduced on get table config API
* checkstyle fix
* made name nonnull in FieldConfig Builder
* refactor redundant code runs
* Optimised the list iterations by converting it to map
---
.../local/segment/index/bloom/BloomIndexType.java | 14 +++-
.../index/dictionary/DictionaryIndexType.java | 16 +++-
.../segment/index/forward/ForwardIndexType.java | 8 +-
.../local/segment/index/fst/FstIndexType.java | 13 ++-
.../local/segment/index/h3/H3IndexType.java | 8 +-
.../segment/index/inverted/InvertedIndexType.java | 13 ++-
.../local/segment/index/json/JsonIndexType.java | 14 +++-
.../index/nullvalue/NullValueIndexType.java | 8 +-
.../local/segment/index/range/RangeIndexType.java | 13 ++-
.../local/segment/index/text/TextIndexType.java | 9 +-
.../segment/local/utils/TableConfigUtils.java | 29 +++++++
.../segment/index/AbstractSerdeIndexContract.java | 5 ++
.../segment/local/segment/index/H3IndexTest.java | 97 ++++++++++++++++++++++
.../segment/local/segment/index/JsonIndexTest.java | 32 +++++++
.../local/segment/index/NullValueIndexTest.java | 55 ++++++++++++
.../local/segment/index/RangeIndexTest.java | 58 +++++++++++++
.../segment/local/segment/index/TextIndexTest.java | 61 ++++++++++++++
.../segment/index/bloom/BloomIndexTypeTest.java | 24 +++++-
.../index/dictionary/DictionaryIndexTypeTest.java | 35 +++++++-
.../index/forward/ForwardIndexTypeTest.java | 22 ++++-
.../local/segment/index/fst/FstIndexTypeTest.java | 24 +++++-
.../index/inverted/InvertedIndexTypeTest.java | 20 ++++-
.../pinot/segment/spi/index/AbstractIndexType.java | 40 +++++++++
.../apache/pinot/segment/spi/index/IndexType.java | 12 +++
.../apache/pinot/spi/config/table/FieldConfig.java | 73 ++++++++++++++++
.../apache/pinot/spi/config/table/TableConfig.java | 24 ++++++
26 files changed, 711 insertions(+), 16 deletions(-)
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java
index c78d4ae218..84a00a3802 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java
@@ -50,6 +50,7 @@ import org.apache.pinot.spi.data.Schema;
public class BloomIndexType
extends AbstractIndexType<BloomFilterConfig, BloomFilterReader, BloomFilterCreator>
implements ConfigurableFromIndexLoadingConfig<BloomFilterConfig> {
+ public static final String INDEX_DISPLAY_NAME = "bloom";
protected BloomIndexType() {
super(StandardIndexes.BLOOM_FILTER_ID);
@@ -70,9 +71,14 @@ public class BloomIndexType
return BloomFilterConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<BloomFilterConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("bloom", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(
IndexConfigDeserializer.ifIndexingConfig(// reads tableConfig.indexingConfig.bloomFilterConfigs
IndexConfigDeserializer.fromMap(tableConfig -> tableConfig.getIndexingConfig().getBloomFilterConfigs())
@@ -133,4 +139,10 @@ public class BloomIndexType
return BloomFilterReaderFactory.getBloomFilterReader(dataBuffer, indexConfig.isLoadOnHeap());
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ tableConfig.getIndexingConfig().setBloomFilterColumns(null);
+ tableConfig.getIndexingConfig().setBloomFilterConfigs(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
index 9ffd9dfb6a..3d1f726bb5 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java
@@ -110,6 +110,11 @@ public class DictionaryIndexType
return DictionaryIndexConfig.DEFAULT;
}
+ @Override
+ public String getPrettyName() {
+ return getId();
+ }
+
@Override
public ColumnConfigDeserializer<DictionaryIndexConfig> createDeserializer() {
// reads tableConfig.indexingConfig.noDictionaryConfig
@@ -155,7 +160,7 @@ public class DictionaryIndexType
.withFallbackAlternative(fromNoDictCol)
.withFallbackAlternative(fromFieldConfigList)
.withFallbackAlternative(fromIndexingConfig)
- .withExclusiveAlternative(IndexConfigDeserializer.fromIndexes(getId(), getIndexConfigClass()));
+ .withExclusiveAlternative(IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()));
}
@Override
@@ -339,4 +344,13 @@ public class DictionaryIndexType
return DictionaryIndexType.read(dataBuffer, metadata, indexConfig);
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
+ indexingConfig.setNoDictionaryConfig(null);
+ indexingConfig.setNoDictionaryColumns(null);
+ indexingConfig.setOnHeapDictionaryColumns(null);
+ indexingConfig.setVarLengthDictionaryColumns(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
index cf56b4cb8b..26fcb652ff 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexType.java
@@ -56,6 +56,7 @@ import org.apache.pinot.spi.data.Schema;
public class ForwardIndexType
extends AbstractIndexType<ForwardIndexConfig, ForwardIndexReader, ForwardIndexCreator>
implements ConfigurableFromIndexLoadingConfig<ForwardIndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "forward";
protected ForwardIndexType() {
super(StandardIndexes.FORWARD_ID);
@@ -125,6 +126,11 @@ public class ForwardIndexType
return ForwardIndexConfig.DEFAULT;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<ForwardIndexConfig> createDeserializer() {
// reads tableConfig.fieldConfigList and decides what to create using the FieldConfig properties and encoding
@@ -144,7 +150,7 @@ public class ForwardIndexType
// the default `fromIndexes` deserializer.
}
);
- return IndexConfigDeserializer.fromIndexes("forward", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(fromFieldConfig);
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
index 5d4fb5168f..297a44fffc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java
@@ -60,6 +60,7 @@ import org.apache.pinot.spi.data.Schema;
public class FstIndexType extends AbstractIndexType<FstIndexConfig, TextIndexReader, FSTIndexCreator>
implements ConfigurableFromIndexLoadingConfig<FstIndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "fst";
protected FstIndexType() {
super(StandardIndexes.FST_ID);
@@ -90,9 +91,14 @@ public class FstIndexType extends AbstractIndexType<FstIndexConfig, TextIndexRea
return FstIndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<FstIndexConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("fst", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(IndexConfigDeserializer.fromIndexTypes(
FieldConfig.IndexType.FST,
(tableConfig, fieldConfig) -> {
@@ -167,4 +173,9 @@ public class FstIndexType extends AbstractIndexType<FstIndexConfig, TextIndexRea
return createIndexReader(dataBuffer, metadata);
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ tableConfig.getIndexingConfig().setFSTIndexType(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
index bb73d6208f..518e69d394 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java
@@ -55,6 +55,7 @@ import org.apache.pinot.spi.data.Schema;
public class H3IndexType extends AbstractIndexType<H3IndexConfig, H3IndexReader, GeoSpatialIndexCreator>
implements ConfigurableFromIndexLoadingConfig<H3IndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "h3";
protected H3IndexType() {
super(StandardIndexes.H3_ID);
@@ -75,9 +76,14 @@ public class H3IndexType extends AbstractIndexType<H3IndexConfig, H3IndexReader,
return H3IndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<H3IndexConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("h3", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(IndexConfigDeserializer.fromIndexTypes(
FieldConfig.IndexType.H3,
((tableConfig, fieldConfig) -> new H3IndexConfig(fieldConfig.getProperties()))));
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java
index 69e2f335b4..d7dbf97c75 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java
@@ -57,6 +57,7 @@ import org.apache.pinot.spi.data.Schema;
public class InvertedIndexType
extends AbstractIndexType<IndexConfig, InvertedIndexReader, DictionaryBasedInvertedIndexCreator>
implements ConfigurableFromIndexLoadingConfig<IndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "inverted";
protected InvertedIndexType() {
super(StandardIndexes.INVERTED_ID);
@@ -78,12 +79,17 @@ public class InvertedIndexType
return IndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<IndexConfig> createDeserializer() {
ColumnConfigDeserializer<IndexConfig> fromInvertedCols = IndexConfigDeserializer.fromCollection(
tableConfig -> tableConfig.getIndexingConfig().getInvertedIndexColumns(),
(acum, column) -> acum.put(column, IndexConfig.ENABLED));
- return IndexConfigDeserializer.fromIndexes("inverted", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(IndexConfigDeserializer.ifIndexingConfig(fromInvertedCols));
}
@@ -182,4 +188,9 @@ public class InvertedIndexType
return new BitmapInvertedIndexReader(dataBuffer, metadata.getCardinality());
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ tableConfig.getIndexingConfig().setInvertedIndexColumns(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java
index a61a8e3b66..a9b4f28320 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java
@@ -53,6 +53,7 @@ import org.apache.pinot.spi.data.Schema;
public class JsonIndexType extends AbstractIndexType<JsonIndexConfig, JsonIndexReader, JsonIndexCreator>
implements ConfigurableFromIndexLoadingConfig<JsonIndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "json";
protected JsonIndexType() {
super(StandardIndexes.JSON_ID);
@@ -73,6 +74,11 @@ public class JsonIndexType extends AbstractIndexType<JsonIndexConfig, JsonIndexR
return JsonIndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<JsonIndexConfig> createDeserializer() {
// reads tableConfig.indexingConfig.jsonIndexConfigs
@@ -83,7 +89,7 @@ public class JsonIndexType extends AbstractIndexType<JsonIndexConfig, JsonIndexR
IndexConfigDeserializer.fromCollection(
tableConfig -> tableConfig.getIndexingConfig().getJsonIndexColumns(),
(accum, column) -> accum.put(column, new JsonIndexConfig()));
- return IndexConfigDeserializer.fromIndexes("json", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative(
IndexConfigDeserializer.ifIndexingConfig(fromJsonIndexCols.withExclusiveAlternative(fromJsonIndexConf)));
}
@@ -152,4 +158,10 @@ public class JsonIndexType extends AbstractIndexType<JsonIndexConfig, JsonIndexR
return new ImmutableJsonIndexReader(dataBuffer, metadata.getTotalDocs());
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ tableConfig.getIndexingConfig().setJsonIndexColumns(null);
+ tableConfig.getIndexingConfig().setJsonIndexConfigs(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
index 99a4c9aeb1..0eb60a2935 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/nullvalue/NullValueIndexType.java
@@ -44,6 +44,7 @@ import org.apache.pinot.spi.data.Schema;
public class NullValueIndexType extends AbstractIndexType<IndexConfig, NullValueVectorReader, NullValueVectorCreator> {
+ public static final String INDEX_DISPLAY_NAME = "null";
protected NullValueIndexType() {
super(StandardIndexes.NULL_VALUE_VECTOR_ID);
@@ -65,9 +66,14 @@ public class NullValueIndexType extends AbstractIndexType<IndexConfig, NullValue
return IndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<IndexConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("null", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withFallbackAlternative(
IndexConfigDeserializer.ifIndexingConfig(
IndexConfigDeserializer.alwaysCall((TableConfig tableConfig, Schema schema) ->
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java
index 36b57dfc59..56ee6b25df 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java
@@ -58,6 +58,7 @@ import org.apache.pinot.spi.data.Schema;
public class RangeIndexType extends AbstractIndexType<RangeIndexConfig, RangeIndexReader, CombinedInvertedIndexCreator>
implements ConfigurableFromIndexLoadingConfig<RangeIndexConfig> {
+ public static final String INDEX_DISPLAY_NAME = "range";
protected RangeIndexType() {
super(StandardIndexes.RANGE_ID);
@@ -82,9 +83,14 @@ public class RangeIndexType extends AbstractIndexType<RangeIndexConfig, RangeInd
return RangeIndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<RangeIndexConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("range", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative((tableConfig, schema) -> {
if (tableConfig.getIndexingConfig() == null) {
return Collections.emptyMap();
@@ -173,4 +179,9 @@ public class RangeIndexType extends AbstractIndexType<RangeIndexConfig, RangeInd
"Unknown range index version " + version);
}
}
+
+ @Override
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ tableConfig.getIndexingConfig().setRangeIndexColumns(null);
+ }
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
index fecdcbd294..844bdcfb8b 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java
@@ -65,6 +65,8 @@ public class TextIndexType extends AbstractIndexType<TextIndexConfig, TextIndexR
implements ConfigurableFromIndexLoadingConfig<TextIndexConfig> {
protected static final Logger LOGGER = LoggerFactory.getLogger(TextIndexType.class);
+ public static final String INDEX_DISPLAY_NAME = "text";
+
protected TextIndexType() {
super(StandardIndexes.TEXT_ID);
}
@@ -90,9 +92,14 @@ public class TextIndexType extends AbstractIndexType<TextIndexConfig, TextIndexR
return TextIndexConfig.DISABLED;
}
+ @Override
+ public String getPrettyName() {
+ return INDEX_DISPLAY_NAME;
+ }
+
@Override
public ColumnConfigDeserializer<TextIndexConfig> createDeserializer() {
- return IndexConfigDeserializer.fromIndexes("text", getIndexConfigClass())
+ return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass())
.withExclusiveAlternative((tableConfig, schema) -> {
List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList();
if (fieldConfigList == null) {
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index 713fafa74f..c679759b87 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
@@ -45,6 +46,8 @@ import org.apache.pinot.segment.local.function.FunctionEvaluator;
import org.apache.pinot.segment.local.function.FunctionEvaluatorFactory;
import org.apache.pinot.segment.local.segment.creator.impl.inv.BitSlicedRangeIndexCreator;
import org.apache.pinot.segment.spi.AggregationFunctionType;
+import org.apache.pinot.segment.spi.index.IndexService;
+import org.apache.pinot.segment.spi.index.IndexType;
import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.IndexingConfig;
@@ -1176,4 +1179,30 @@ public final class TableConfigUtils {
String instanceSelectorType = routingConfig.getInstanceSelectorType();
return UPSERT_DEDUP_ALLOWED_ROUTING_STRATEGIES.stream().anyMatch(x -> x.equalsIgnoreCase(instanceSelectorType));
}
+
+ /**
+ * Helper method to extract TableConfig in updated syntax from current TableConfig.
+ * <ul>
+ * <li>Moves all index configs to FieldConfig.indexes</li>
+ * <li>Clean up index related configs from IndexingConfig and FieldConfig.IndexTypes</li>
+ * </ul>
+ */
+ public static TableConfig createTableConfigFromOldFormat(TableConfig tableConfig, Schema schema) {
+ TableConfig clone = new TableConfig(tableConfig);
+ for (IndexType<?, ?, ?> indexType : IndexService.getInstance().getAllIndexes()) {
+ // get all the index data in new format
+ indexType.convertToNewFormat(clone, schema);
+ }
+ // cleanup the indexTypes field from all FieldConfig items
+ if (clone.getFieldConfigList() != null) {
+ List<FieldConfig> cleanFieldConfigList = new ArrayList<>();
+ for (FieldConfig fieldConfig : clone.getFieldConfigList()) {
+ cleanFieldConfigList.add(new FieldConfig.Builder(fieldConfig)
+ .withIndexTypes(null)
+ .build());
+ }
+ clone.setFieldConfigList(cleanFieldConfigList);
+ }
+ return clone;
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/AbstractSerdeIndexContract.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/AbstractSerdeIndexContract.java
index caf20edb4b..86eb33ac20 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/AbstractSerdeIndexContract.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/AbstractSerdeIndexContract.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.pinot.segment.local.utils.TableConfigUtils;
import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
import org.apache.pinot.segment.spi.index.FieldIndexConfigsUtil;
import org.apache.pinot.segment.spi.index.IndexType;
@@ -124,4 +125,8 @@ public class AbstractSerdeIndexContract {
throws IOException {
return JsonUtils.stringToObject(json, _stringListTypeRef);
}
+
+ protected void convertToUpdatedFormat() {
+ _tableConfig = TableConfigUtils.createTableConfigFromOldFormat(_tableConfig, _schema);
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
index 6b25b5e8c6..ce43146c86 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java
@@ -18,23 +18,31 @@
*/
package org.apache.pinot.segment.local.segment.index;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.google.common.collect.Lists;
import java.io.File;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
+import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.realtime.impl.geospatial.MutableH3Index;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.OffHeapH3IndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.inv.geospatial.OnHeapH3IndexCreator;
+import org.apache.pinot.segment.local.segment.index.h3.H3IndexType;
import org.apache.pinot.segment.local.segment.index.readers.geospatial.ImmutableH3IndexReader;
import org.apache.pinot.segment.local.utils.GeometryUtils;
import org.apache.pinot.segment.local.utils.H3Utils;
import org.apache.pinot.segment.spi.V1Constants;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
import org.apache.pinot.segment.spi.index.creator.GeoSpatialIndexCreator;
+import org.apache.pinot.segment.spi.index.creator.H3IndexConfig;
import org.apache.pinot.segment.spi.index.reader.H3IndexReader;
import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.geom.Point;
import org.testng.Assert;
@@ -42,6 +50,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
public class H3IndexTest {
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "H3IndexCreatorTest");
@@ -107,4 +119,89 @@ public class H3IndexTest {
}
}
}
+
+ public static class ConfTest extends AbstractSerdeIndexContract {
+
+ protected void assertEquals(H3IndexConfig expected) {
+ Assert.assertEquals(getActualConfig("dimStr", StandardIndexes.h3()), expected);
+ }
+
+ @Test
+ public void oldFieldConfigNull()
+ throws JsonProcessingException {
+ _tableConfig.setFieldConfigList(null);
+
+ assertEquals(H3IndexConfig.DISABLED);
+ }
+
+ @Test
+ public void oldEmptyFieldConfig()
+ throws JsonProcessingException {
+ cleanFieldConfig();
+
+ assertEquals(H3IndexConfig.DISABLED);
+ }
+
+ @Test
+ public void oldFieldConfigNotH3()
+ throws JsonProcessingException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexTypes\" : []\n"
+ + " }");
+
+ assertEquals(H3IndexConfig.DISABLED);
+ }
+
+ @Test
+ public void oldFieldConfigH3Resolution()
+ throws JsonProcessingException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexTypes\" : [\"H3\"],\n"
+ + " \"properties\": {\n"
+ + " \"resolutions\": \"3\""
+ + " }\n"
+ + " }");
+
+ assertEquals(new H3IndexConfig(new H3IndexResolution(Lists.newArrayList(3))));
+ }
+
+ @Test
+ public void newConfEnabled()
+ throws JsonProcessingException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexes\" : {\n"
+ + " \"h3\": {\n"
+ + " \"enabled\": \"true\",\n"
+ + " \"resolution\": [3]\n"
+ + " }\n"
+ + " }\n"
+ + " }");
+ assertEquals(new H3IndexConfig(new H3IndexResolution(Lists.newArrayList(3))));
+ }
+
+ @Test
+ public void oldToNewConfConversion()
+ throws IOException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexes\" : {\n"
+ + " \"h3\": {\n"
+ + " \"enabled\": \"true\",\n"
+ + " \"resolution\": [3]\n"
+ + " }\n"
+ + " }\n"
+ + " }");
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimStr"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(H3IndexType.INDEX_DISPLAY_NAME));
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ }
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
index 656b47a9c5..53a98c5d55 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java
@@ -18,17 +18,23 @@
*/
package org.apache.pinot.segment.local.segment.index;
+import com.google.common.collect.Lists;
import java.io.File;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.segment.local.realtime.impl.json.MutableJsonIndexImpl;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OffHeapJsonIndexCreator;
import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OnHeapJsonIndexCreator;
+import org.apache.pinot.segment.local.segment.index.json.JsonIndexType;
import org.apache.pinot.segment.local.segment.index.readers.json.ImmutableJsonIndexReader;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.creator.JsonIndexCreator;
import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.JsonIndexConfig;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
import org.testng.Assert;
@@ -36,6 +42,10 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
/**
* Unit test for {@link JsonIndexCreator} and {@link JsonIndexReader}.
@@ -226,4 +236,26 @@ public class JsonIndexTest {
private MutableRoaringBitmap getMatchingDocIds(JsonIndexReader indexReader, String filter) {
return indexReader.getMatchingDocIds(filter);
}
+
+ public static class ConfTest extends AbstractSerdeIndexContract {
+
+ @Test
+ public void oldToNewConfConversion() {
+ Map<String, JsonIndexConfig> configs = new HashMap<>();
+ JsonIndexConfig config = new JsonIndexConfig();
+ config.setMaxLevels(2);
+ configs.put("dimStr", config);
+ _tableConfig.getIndexingConfig().setJsonIndexConfigs(configs);
+ _tableConfig.getIndexingConfig().setJsonIndexColumns(Lists.newArrayList("dimStr2"));
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimStr"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(JsonIndexType.INDEX_DISPLAY_NAME));
+ assertNull(_tableConfig.getIndexingConfig().getJsonIndexColumns());
+ assertNull(_tableConfig.getIndexingConfig().getJsonIndexConfigs());
+ }
+ }
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/NullValueIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/NullValueIndexTest.java
new file mode 100644
index 0000000000..a4be3ee2fd
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/NullValueIndexTest.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.pinot.segment.local.segment.index.nullvalue.NullValueIndexPlugin;
+import org.apache.pinot.segment.local.segment.index.nullvalue.NullValueIndexType;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+
+
+public class NullValueIndexTest {
+ public static class ConfTest extends AbstractSerdeIndexContract {
+
+ @Test
+ public void oldToNewConfConversion() {
+ _tableConfig.getIndexingConfig().setNullHandlingEnabled(true);
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ _tableConfig.getFieldConfigList()
+ .forEach(fieldConfig -> {
+ JsonNode indexConfig = fieldConfig.getIndexes().get(NullValueIndexType.INDEX_DISPLAY_NAME);
+ assertNotNull(indexConfig);
+ assertFalse(indexConfig.get("disabled").asBoolean());
+ });
+ }
+ }
+
+ @Test
+ public void testStandardIndex() {
+ assertEquals(StandardIndexes.nullValueVector(), new NullValueIndexPlugin().getIndexType(),
+ "Standard index should be equal to the instance returned by the plugin");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java
new file mode 100644
index 0000000000..0308956871
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.Lists;
+import java.util.stream.Collectors;
+import org.apache.pinot.segment.local.segment.index.range.RangeIndexPlugin;
+import org.apache.pinot.segment.local.segment.index.range.RangeIndexType;
+import org.apache.pinot.segment.spi.index.RangeIndexConfig;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+
+
+public class RangeIndexTest {
+ public static class ConfTest extends AbstractSerdeIndexContract {
+
+ @Test
+ public void oldToNewConfConversion() {
+ _tableConfig.getIndexingConfig().setRangeIndexColumns(Lists.newArrayList("dimInt"));
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimInt")).collect(Collectors.toList()).get(0);
+ JsonNode indexConfig = fieldConfig.getIndexes().get(RangeIndexType.INDEX_DISPLAY_NAME);
+ assertNotNull(indexConfig);
+ assertEquals(RangeIndexConfig.DEFAULT.getVersion(), indexConfig.get("version").asInt());
+ }
+ }
+
+ @Test
+ public void testStandardIndex() {
+ assertEquals(StandardIndexes.range(), new RangeIndexPlugin().getIndexType(),
+ "Standard index should be equal to the instance returned by the plugin");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
new file mode 100644
index 0000000000..918de11390
--- /dev/null
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.segment.index;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.stream.Collectors;
+import org.apache.pinot.segment.local.segment.index.text.TextIndexPlugin;
+import org.apache.pinot.segment.local.segment.index.text.TextIndexType;
+import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.spi.config.table.FieldConfig;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+
+
+public class TextIndexTest {
+ public static class ConfTest extends AbstractSerdeIndexContract {
+
+ @Test
+ public void oldToNewConfConversion()
+ throws JsonProcessingException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexTypes\" : [\"TEXT\"]\n"
+ + " }");
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimStr")).collect(Collectors.toList()).get(0);
+ JsonNode indexConfig = fieldConfig.getIndexes().get(TextIndexType.INDEX_DISPLAY_NAME);
+ assertNotNull(indexConfig);
+ assertFalse(indexConfig.get("disabled").asBoolean());
+ }
+ }
+
+ @Test
+ public void testStandardIndex() {
+ assertEquals(StandardIndexes.text(), new TextIndexPlugin().getIndexType(),
+ "Standard index should be equal to the instance returned by the plugin");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexTypeTest.java
index 6841caf93e..16479b4065 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexTypeTest.java
@@ -23,9 +23,11 @@ import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
import org.apache.pinot.segment.spi.index.StandardIndexes;
import org.apache.pinot.spi.config.table.BloomFilterConfig;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
@@ -51,7 +53,7 @@ public class BloomIndexTypeTest {
};
}
- public class ConfTest extends AbstractSerdeIndexContract {
+ public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(BloomFilterConfig expected) {
Assert.assertEquals(getActualConfig("dimInt", StandardIndexes.bloomFilter()), expected);
@@ -160,6 +162,26 @@ public class BloomIndexTypeTest {
assertEquals(config);
}
+
+ @Test(dataProvider = "allConfigs", dataProviderClass = BloomIndexTypeTest.class)
+ public void oldToNewConfConversion(String confStr)
+ throws IOException {
+ _tableConfig.getIndexingConfig().setBloomFilterColumns(
+ JsonUtils.stringToObject("[\"dimInt\"]", _stringListTypeRef)
+ );
+ BloomFilterConfig config =
+ JsonUtils.stringToObject(confStr, BloomFilterConfig.class);
+ _tableConfig.getIndexingConfig().setBloomFilterConfigs(Collections.singletonMap("dimInt", config));
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimInt"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(BloomIndexType.INDEX_DISPLAY_NAME));
+ assertNull(_tableConfig.getIndexingConfig().getBloomFilterColumns());
+ assertNull(_tableConfig.getIndexingConfig().getBloomFilterConfigs());
+ }
}
@Test
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexTypeTest.java
index c7b34d68c5..819812496e 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexTypeTest.java
@@ -22,9 +22,11 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
import org.apache.pinot.segment.spi.index.DictionaryIndexConfig;
import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -33,7 +35,7 @@ import static org.testng.Assert.*;
public class DictionaryIndexTypeTest {
- public class ConfTest extends AbstractSerdeIndexContract {
+ public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(DictionaryIndexConfig expected) {
Assert.assertEquals(getActualConfig("dimInt", StandardIndexes.dictionary()), expected);
@@ -75,7 +77,8 @@ public class DictionaryIndexTypeTest {
assertEquals(DictionaryIndexConfig.DISABLED);
}
- public void oldRawEncondingType()
+ @Test
+ public void oldRawEncodingType()
throws IOException {
_tableConfig.getIndexingConfig().setNoDictionaryConfig(
JsonUtils.stringToObject("{\"dimInt\": \"RAW\"}",
@@ -219,6 +222,34 @@ public class DictionaryIndexTypeTest {
+ " }");
assertEquals(new DictionaryIndexConfig(false, true));
}
+
+ @Test
+ public void oldToNewConfConversion()
+ throws IOException {
+ _tableConfig.getIndexingConfig().setNoDictionaryColumns(
+ JsonUtils.stringToObject("[\"dimInt\"]", _stringListTypeRef)
+ );
+ _tableConfig.getIndexingConfig()
+ .setOnHeapDictionaryColumns(JsonUtils.stringToObject("[\"dimInt\"]", _stringListTypeRef));
+ _tableConfig.getIndexingConfig()
+ .setVarLengthDictionaryColumns(JsonUtils.stringToObject("[\"dimInt\"]", _stringListTypeRef));
+ _tableConfig.getIndexingConfig().setNoDictionaryConfig(
+ JsonUtils.stringToObject("{\"dimInt\": \"RAW\"}",
+ new TypeReference<Map<String, String>>() {
+ })
+ );
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimInt"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(new DictionaryIndexType().getPrettyName()));
+ assertNull(_tableConfig.getIndexingConfig().getNoDictionaryColumns());
+ assertNull(_tableConfig.getIndexingConfig().getOnHeapDictionaryColumns());
+ assertNull(_tableConfig.getIndexingConfig().getVarLengthDictionaryColumns());
+ assertNull(_tableConfig.getIndexingConfig().getNoDictionaryConfig());
+ }
}
@Test
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
index 2bf4001c84..bcfece6be5 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/forward/ForwardIndexTypeTest.java
@@ -21,10 +21,12 @@ package org.apache.pinot.segment.local.segment.index.forward;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
+import java.util.stream.Collectors;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
import org.apache.pinot.segment.spi.index.ForwardIndexConfig;
import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
@@ -46,7 +48,7 @@ public class ForwardIndexTypeTest {
};
}
- public class ConfTest extends AbstractSerdeIndexContract {
+ public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(ForwardIndexConfig expected) {
Assert.assertEquals(getActualConfig("dimInt", StandardIndexes.forward()), expected);
@@ -270,6 +272,24 @@ public class ForwardIndexTypeTest {
.build()
);
}
+
+ @Test
+ public void oldToNewConfConversion()
+ throws JsonProcessingException {
+ addFieldIndexConfig(""
+ + " {\n"
+ + " \"name\": \"dimInt\","
+ + " \"encodingType\": \"RAW\"\n"
+ + " }"
+ );
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimInt"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(ForwardIndexType.INDEX_DISPLAY_NAME));
+ }
}
@Test
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexTypeTest.java
index c0a1f6310d..c07d5ff910 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexTypeTest.java
@@ -19,10 +19,13 @@
package org.apache.pinot.segment.local.segment.index.fst;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.io.IOException;
+import java.util.stream.Collectors;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
import org.apache.pinot.segment.spi.index.FstIndexConfig;
import org.apache.pinot.segment.spi.index.StandardIndexes;
import org.apache.pinot.spi.config.table.FSTType;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -31,7 +34,7 @@ import static org.testng.Assert.*;
public class FstIndexTypeTest {
- public class ConfTest extends AbstractSerdeIndexContract {
+ public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(FstIndexConfig expected) {
Assert.assertEquals(getActualConfig("dimStr", StandardIndexes.fst()), expected);
@@ -151,6 +154,25 @@ public class FstIndexTypeTest {
+ " }");
assertEquals(new FstIndexConfig(null));
}
+
+ @Test
+ public void oldToNewConfConversion()
+ throws IOException {
+ addFieldIndexConfig("{\n"
+ + " \"name\": \"dimStr\",\n"
+ + " \"indexTypes\" : [\"FST\"]\n"
+ + " }");
+ _tableConfig.getIndexingConfig().setFSTIndexType(FSTType.NATIVE);
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimStr"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(FstIndexType.INDEX_DISPLAY_NAME));
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ assertNull(_tableConfig.getIndexingConfig().getFSTIndexType());
+ }
}
@Test
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexTypeTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexTypeTest.java
index 20d3e4b40e..cbdd49f8ae 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexTypeTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexTypeTest.java
@@ -20,8 +20,10 @@ package org.apache.pinot.segment.local.segment.index.inverted;
import com.fasterxml.jackson.core.JsonProcessingException;
import java.io.IOException;
+import java.util.stream.Collectors;
import org.apache.pinot.segment.local.segment.index.AbstractSerdeIndexContract;
import org.apache.pinot.segment.spi.index.StandardIndexes;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.IndexConfig;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -32,7 +34,7 @@ import static org.testng.Assert.*;
public class InvertedIndexTypeTest {
- public class ConfTest extends AbstractSerdeIndexContract {
+ public static class ConfTest extends AbstractSerdeIndexContract {
protected void assertEquals(IndexConfig expected) {
Assert.assertEquals(getActualConfig("dimInt", StandardIndexes.inverted()), expected);
@@ -110,6 +112,22 @@ public class InvertedIndexTypeTest {
);
assertEquals(IndexConfig.ENABLED);
}
+
+ @Test
+ public void oldToNewConfConversion()
+ throws IOException {
+ _tableConfig.getIndexingConfig()
+ .setInvertedIndexColumns(parseStringList("[\"dimInt\"]"));
+ convertToUpdatedFormat();
+ assertNotNull(_tableConfig.getFieldConfigList());
+ assertFalse(_tableConfig.getFieldConfigList().isEmpty());
+ FieldConfig fieldConfig = _tableConfig.getFieldConfigList().stream()
+ .filter(fc -> fc.getName().equals("dimInt"))
+ .collect(Collectors.toList()).get(0);
+ assertNotNull(fieldConfig.getIndexes().get(InvertedIndexType.INDEX_DISPLAY_NAME));
+ assertNull(_tableConfig.getIndexingConfig().getInvertedIndexColumns());
+ assertTrue(fieldConfig.getIndexTypes().isEmpty());
+ }
}
@Test
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/AbstractIndexType.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/AbstractIndexType.java
index 299612a72d..e3c2afbedd 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/AbstractIndexType.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/AbstractIndexType.java
@@ -19,8 +19,16 @@
package org.apache.pinot.segment.spi.index;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.IndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.data.Schema;
@@ -37,6 +45,9 @@ public abstract class AbstractIndexType<C extends IndexConfig, IR extends IndexR
protected abstract IndexReaderFactory<IR> createReaderFactory();
+ protected void handleIndexSpecificCleanup(TableConfig tableConfig) {
+ }
+
public AbstractIndexType(String id) {
_id = id;
}
@@ -62,6 +73,35 @@ public abstract class AbstractIndexType<C extends IndexConfig, IR extends IndexR
return _readerFactory;
}
+ public void convertToNewFormat(TableConfig tableConfig, Schema schema) {
+ Map<String, C> deserialize = getConfig(tableConfig, schema);
+ List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList() == null
+ ? new ArrayList<>()
+ : tableConfig.getFieldConfigList();
+ Map<String, FieldConfig> fieldConfigMap = fieldConfigList.stream()
+ .collect(Collectors.toMap(FieldConfig::getName, Function.identity()));
+ for (Map.Entry<String, C> entry : deserialize.entrySet()) {
+ FieldConfig fieldConfig = fieldConfigMap.get(entry.getKey());
+ if (fieldConfig != null) {
+ ObjectNode currentIndexes = fieldConfig.getIndexes().isNull()
+ ? new ObjectMapper().createObjectNode()
+ : new ObjectMapper().valueToTree(fieldConfig.getIndexes());
+ JsonNode indexes = currentIndexes.set(getPrettyName(), entry.getValue().toJsonNode());
+ FieldConfig.Builder builder = new FieldConfig.Builder(fieldConfig);
+ builder.withIndexes(indexes);
+ fieldConfigList.remove(fieldConfig);
+ fieldConfigList.add(builder.build());
+ } else {
+ JsonNode indexes = new ObjectMapper().createObjectNode().set(getPrettyName(), entry.getValue().toJsonNode());
+ FieldConfig.Builder builder = new FieldConfig.Builder(entry.getKey());
+ builder.withIndexes(indexes);
+ fieldConfigList.add(builder.build());
+ }
+ }
+ tableConfig.setFieldConfigList(fieldConfigList);
+ handleIndexSpecificCleanup(tableConfig);
+ }
+
@Override
public String toString() {
return _id;
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
index 89a0e23666..ca78711b06 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/IndexType.java
@@ -61,6 +61,8 @@ public interface IndexType<C extends IndexConfig, IR extends IndexReader, IC ext
Map<String, C> getConfig(TableConfig tableConfig, Schema schema);
+ String getPrettyName();
+
/**
* Returns the {@link IndexCreator} that can should be used to create an index of this type with the given context
* and configuration.
@@ -93,4 +95,14 @@ public interface IndexType<C extends IndexConfig, IR extends IndexReader, IC ext
IndexHandler createIndexHandler(SegmentDirectory segmentDirectory, Map<String, FieldIndexConfigs> configsByCol,
@Nullable Schema schema, @Nullable TableConfig tableConfig);
+
+ /**
+ * This method is used to perform in place conversion of provided {@link TableConfig} to newer format
+ * related to the IndexType that implements it.
+ *
+ * {@link AbstractIndexType#convertToNewFormat(TableConfig, Schema)} ensures all the index information from old format
+ * is made available in the new format while it depends on the individual index types to handle the data cleanup from
+ * old format.
+ */
+ void convertToNewFormat(TableConfig tableConfig, Schema schema);
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
index 50b5ac7f91..e567bb6f2e 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/FieldConfig.java
@@ -26,6 +26,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.util.List;
import java.util.Map;
+import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.pinot.spi.config.BaseJsonConfig;
@@ -159,4 +160,76 @@ public class FieldConfig extends BaseJsonConfig {
public Map<String, String> getProperties() {
return _properties;
}
+
+ public static class Builder {
+ @Nonnull
+ private String _name;
+ private EncodingType _encodingType;
+ private List<IndexType> _indexTypes;
+ private JsonNode _indexes;
+ private CompressionCodec _compressionCodec;
+ private Map<String, String> _properties;
+ private TimestampConfig _timestampConfig;
+ private JsonNode _tierOverwrites;
+
+ public Builder(@Nonnull String name) {
+ _name = name;
+ }
+
+ public Builder(FieldConfig other) {
+ _name = other._name;
+ _encodingType = other._encodingType;
+ _indexTypes = other._indexTypes;
+ _indexes = other._indexes;
+ _compressionCodec = other._compressionCodec;
+ _properties = other._properties;
+ _timestampConfig = other._timestampConfig;
+ _tierOverwrites = other._tierOverwrites;
+ }
+
+ public Builder withIndexes(JsonNode indexes) {
+ _indexes = indexes;
+ return this;
+ }
+
+ public Builder withName(String name) {
+ _name = name;
+ return this;
+ }
+
+ public Builder withEncodingType(EncodingType encodingType) {
+ _encodingType = encodingType;
+ return this;
+ }
+
+ public Builder withIndexTypes(List<IndexType> indexTypes) {
+ _indexTypes = indexTypes;
+ return this;
+ }
+
+ public Builder withCompressionCodec(CompressionCodec compressionCodec) {
+ _compressionCodec = compressionCodec;
+ return this;
+ }
+
+ public Builder withProperties(Map<String, String> properties) {
+ _properties = properties;
+ return this;
+ }
+
+ public Builder withTimestampConfig(TimestampConfig timestampConfig) {
+ _timestampConfig = timestampConfig;
+ return this;
+ }
+
+ public Builder withTierOverwrites(JsonNode tierOverwrites) {
+ _tierOverwrites = tierOverwrites;
+ return this;
+ }
+
+ public FieldConfig build() {
+ return new FieldConfig(_name, _encodingType, null, _indexTypes, _compressionCodec, _timestampConfig,
+ _indexes, _properties, _tierOverwrites);
+ }
+ }
}
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TableConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TableConfig.java
index 39e50322bb..41a5bfeb71 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TableConfig.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/TableConfig.java
@@ -175,6 +175,30 @@ public class TableConfig extends BaseJsonConfig {
_segmentAssignmentConfigMap = segmentAssignmentConfigMap;
}
+ public TableConfig(TableConfig tableConfig) {
+ _tableType = tableConfig.getTableType();
+ _tableName = tableConfig.getTableName();
+ _validationConfig = tableConfig.getValidationConfig();
+ _tenantConfig = tableConfig.getTenantConfig();
+ _indexingConfig = tableConfig.getIndexingConfig();
+ _customConfig = tableConfig.getCustomConfig();
+ _quotaConfig = tableConfig.getQuotaConfig();
+ _taskConfig = tableConfig.getTaskConfig();
+ _routingConfig = tableConfig.getRoutingConfig();
+ _queryConfig = tableConfig.getQueryConfig();
+ _instanceAssignmentConfigMap = tableConfig.getInstanceAssignmentConfigMap();
+ _fieldConfigList = tableConfig.getFieldConfigList();
+ _upsertConfig = tableConfig.getUpsertConfig();
+ _dedupConfig = tableConfig.getDedupConfig();
+ _dimensionTableConfig = tableConfig.getDimensionTableConfig();
+ _ingestionConfig = tableConfig.getIngestionConfig();
+ _tierConfigsList = tableConfig.getTierConfigsList();
+ _dimTable = tableConfig.isDimTable();
+ _tunerConfigList = tableConfig.getTunerConfigsList();
+ _instancePartitionsMap = tableConfig.getInstancePartitionsMap();
+ _segmentAssignmentConfigMap = tableConfig.getSegmentAssignmentConfigMap();
+ }
+
@JsonProperty(TABLE_NAME_KEY)
public String getTableName() {
return _tableName;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org