You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by go...@apache.org on 2024/02/27 07:26:29 UTC
(pinot) branch master updated: Adds per-column, query-time index skip option (#12414)
This is an automated email from the ASF dual-hosted git repository.
gortiz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5dc807bb0a Adds per-column, query-time index skip option (#12414)
5dc807bb0a is described below
commit 5dc807bb0a11035598c764c49bd2b481bbaff591
Author: Evan Galpin <eg...@users.noreply.github.com>
AuthorDate: Mon Feb 26 23:26:23 2024 -0800
Adds per-column, query-time index skip option (#12414)
* Adds per-column, query-time index skip option
* Adds tests
* Throw ParseException if indexSkipConfig parsing fails
* Throw RuntimeException instead of ParseException
* Empty commit to re-run tests
---
.../common/utils/config/QueryOptionsUtils.java | 32 +++++++
.../common/utils/config/QueryOptionsUtilsTest.java | 24 ++++++
.../core/operator/filter/FilterOperatorUtils.java | 22 +++--
.../org/apache/pinot/core/plan/FilterPlanNode.java | 10 ++-
.../core/plan/maker/InstancePlanMakerImplV2.java | 2 +
.../core/query/request/context/QueryContext.java | 19 +++++
.../tests/OfflineClusterIntegrationTest.java | 97 ++++++++++++++++++++--
.../pinot/segment/spi/datasource/DataSource.java | 7 ++
.../apache/pinot/spi/utils/CommonConstants.java | 1 +
9 files changed, 195 insertions(+), 19 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
index f31b41b63f..881aa30b8a 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/QueryOptionsUtils.java
@@ -23,8 +23,11 @@ import com.google.common.collect.ImmutableMap;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import javax.annotation.Nullable;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey;
import org.apache.pinot.spi.utils.CommonConstants.MultiStageQueryRunner.JoinOverFlowMode;
@@ -34,6 +37,7 @@ import org.apache.pinot.spi.utils.CommonConstants.MultiStageQueryRunner.JoinOver
* Utils to parse query options.
*/
public class QueryOptionsUtils {
+
private QueryOptionsUtils() {
}
@@ -145,6 +149,34 @@ public class QueryOptionsUtils {
return "false".equalsIgnoreCase(queryOptions.get(QueryOptionKey.USE_SCAN_REORDER_OPTIMIZATION));
}
+ @Nullable
+ public static Map<String, Set<FieldConfig.IndexType>> getIndexSkipConfig(Map<String, String> queryOptions) {
+ // Example config: indexSkipConfig='col1=inverted,range&col2=inverted'
+ String indexSkipConfigStr = queryOptions.get(QueryOptionKey.INDEX_SKIP_CONFIG);
+ if (indexSkipConfigStr == null) {
+ return null;
+ }
+
+ String[] perColumnIndexSkip = indexSkipConfigStr.split("&");
+ Map<String, Set<FieldConfig.IndexType>> indexSkipConfig = new HashMap<>();
+ for (String columnConf : perColumnIndexSkip) {
+ String[] conf = columnConf.split("=");
+ if (conf.length != 2) {
+ throw new RuntimeException("Invalid format for " + QueryOptionKey.INDEX_SKIP_CONFIG
+ + ". Example of valid format: SET indexSkipConfig='col1=inverted,range&col2=inverted'");
+ }
+ String columnName = conf[0];
+ String[] indexTypes = conf[1].split(",");
+
+ for (String indexType : indexTypes) {
+ indexSkipConfig.computeIfAbsent(columnName, k -> new HashSet<>())
+ .add(FieldConfig.IndexType.valueOf(indexType.toUpperCase()));
+ }
+ }
+
+ return indexSkipConfig;
+ }
+
@Nullable
public static Boolean isUseFixedReplica(Map<String, String> queryOptions) {
String useFixedReplica = queryOptions.get(CommonConstants.Broker.Request.QueryOptionKey.USE_FIXED_REPLICA);
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
index 1564fcc15e..d23b98fa7b 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/QueryOptionsUtilsTest.java
@@ -21,7 +21,10 @@ package org.apache.pinot.common.utils.config;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
+import java.util.Set;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.utils.CommonConstants;
+import org.apache.pinot.sql.parsers.parser.ParseException;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -43,4 +46,25 @@ public class QueryOptionsUtilsTest {
Assert.assertEquals(resolved.get(CommonConstants.Broker.Request.QueryOptionKey.ENABLE_NULL_HANDLING), "true");
Assert.assertEquals(resolved.get(CommonConstants.Broker.Request.QueryOptionKey.USE_MULTISTAGE_ENGINE), "false");
}
+
+ @Test
+ public void testIndexSkipConfigParsing()
+ throws ParseException {
+ String indexSkipConfigStr = "col1=inverted,range&col2=sorted";
+ Map<String, String> queryOptions =
+ Map.of(CommonConstants.Broker.Request.QueryOptionKey.INDEX_SKIP_CONFIG, indexSkipConfigStr);
+ Map<String, Set<FieldConfig.IndexType>> indexSkipConfig = QueryOptionsUtils.getIndexSkipConfig(queryOptions);
+ Assert.assertEquals(indexSkipConfig.get("col1"),
+ Set.of(FieldConfig.IndexType.RANGE, FieldConfig.IndexType.INVERTED));
+ Assert.assertEquals(indexSkipConfig.get("col2"),
+ Set.of(FieldConfig.IndexType.SORTED));
+ }
+
+ @Test(expectedExceptions = RuntimeException.class)
+ public void testIndexSkipConfigParsingInvalid() {
+ String indexSkipConfigStr = "col1=inverted,range&col2";
+ Map<String, String> queryOptions =
+ Map.of(CommonConstants.Broker.Request.QueryOptionKey.INDEX_SKIP_CONFIG, indexSkipConfigStr);
+ QueryOptionsUtils.getIndexSkipConfig(queryOptions);
+ }
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
index c0aa629cb6..ac30591c60 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/FilterOperatorUtils.java
@@ -27,6 +27,7 @@ import org.apache.pinot.core.operator.filter.predicate.PredicateEvaluator;
import org.apache.pinot.core.query.request.context.QueryContext;
import org.apache.pinot.segment.spi.datasource.DataSource;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.roaringbitmap.buffer.ImmutableRoaringBitmap;
@@ -95,29 +96,36 @@ public class FilterOperatorUtils {
// operator is used only if the column is sorted and has dictionary.
Predicate.Type predicateType = predicateEvaluator.getPredicateType();
if (predicateType == Predicate.Type.RANGE) {
- if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null) {
+ if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.SORTED)) {
return new SortedIndexBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
- if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
+ if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.RANGE)) {
return new RangeIndexBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
return new ScanBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
} else if (predicateType == Predicate.Type.REGEXP_LIKE) {
- if (dataSource.getFSTIndex() != null && dataSource.getDataSourceMetadata().isSorted()) {
+ if (dataSource.getFSTIndex() != null && dataSource.getDataSourceMetadata().isSorted()
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.SORTED)) {
return new SortedIndexBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
- if (dataSource.getFSTIndex() != null && dataSource.getInvertedIndex() != null) {
+ if (dataSource.getFSTIndex() != null && dataSource.getInvertedIndex() != null
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.INVERTED)) {
return new InvertedIndexFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
return new ScanBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
} else {
- if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null) {
+ if (dataSource.getDataSourceMetadata().isSorted() && dataSource.getDictionary() != null
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.SORTED)) {
return new SortedIndexBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
- if (dataSource.getInvertedIndex() != null) {
+ if (dataSource.getInvertedIndex() != null
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.INVERTED)) {
return new InvertedIndexFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
- if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)) {
+ if (RangeIndexBasedFilterOperator.canEvaluate(predicateEvaluator, dataSource)
+ && queryContext.isIndexUseAllowed(dataSource, FieldConfig.IndexType.RANGE)) {
return new RangeIndexBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
}
return new ScanBasedFilterOperator(queryContext, predicateEvaluator, dataSource, numDocs);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
index 84c53374e2..9a6d17cc9f 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java
@@ -59,6 +59,7 @@ import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
import org.apache.pinot.segment.spi.index.reader.NullValueVectorReader;
import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
+import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.exception.BadQueryRequestException;
import org.roaringbitmap.buffer.MutableRoaringBitmap;
@@ -152,7 +153,8 @@ public class FilterPlanNode implements PlanNode {
findLiteral = true;
}
}
- return columnName != null && _indexSegment.getDataSource(columnName).getH3Index() != null && findLiteral;
+ return columnName != null && _indexSegment.getDataSource(columnName).getH3Index() != null && findLiteral
+ && _queryContext.isIndexUseAllowed(columnName, FieldConfig.IndexType.H3);
}
/**
@@ -182,14 +184,16 @@ public class FilterPlanNode implements PlanNode {
if (arguments.get(0).getType() == ExpressionContext.Type.IDENTIFIER
&& arguments.get(1).getType() == ExpressionContext.Type.LITERAL) {
String columnName = arguments.get(0).getIdentifier();
- return _indexSegment.getDataSource(columnName).getH3Index() != null;
+ return _indexSegment.getDataSource(columnName).getH3Index() != null
+ && _queryContext.isIndexUseAllowed(columnName, FieldConfig.IndexType.H3);
}
return false;
} else {
if (arguments.get(1).getType() == ExpressionContext.Type.IDENTIFIER
&& arguments.get(0).getType() == ExpressionContext.Type.LITERAL) {
String columnName = arguments.get(1).getIdentifier();
- return _indexSegment.getDataSource(columnName).getH3Index() != null;
+ return _indexSegment.getDataSource(columnName).getH3Index() != null
+ && _queryContext.isIndexUseAllowed(columnName, FieldConfig.IndexType.H3);
}
return false;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
index 85b159a51e..5d06a79b10 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/plan/maker/InstancePlanMakerImplV2.java
@@ -175,6 +175,8 @@ public class InstancePlanMakerImplV2 implements PlanMaker {
// Set skipScanFilterReorder
queryContext.setSkipScanFilterReorder(QueryOptionsUtils.isSkipScanFilterReorder(queryOptions));
+ queryContext.setIndexSkipConfig(QueryOptionsUtils.getIndexSkipConfig(queryOptions));
+
// Set maxExecutionThreads
int maxExecutionThreads;
Integer maxExecutionThreadsFromQuery = QueryOptionsUtils.getMaxExecutionThreads(queryOptions);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
index 5d5e9a3718..c2829b8175 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/request/context/QueryContext.java
@@ -40,6 +40,8 @@ import org.apache.pinot.core.plan.maker.InstancePlanMakerImplV2;
import org.apache.pinot.core.query.aggregation.function.AggregationFunction;
import org.apache.pinot.core.query.aggregation.function.AggregationFunctionFactory;
import org.apache.pinot.core.util.MemoizedClassAssociation;
+import org.apache.pinot.segment.spi.datasource.DataSource;
+import org.apache.pinot.spi.config.table.FieldConfig;
/**
@@ -123,6 +125,8 @@ public class QueryContext {
private boolean _nullHandlingEnabled;
// Whether server returns the final result
private boolean _serverReturnFinalResult;
+ // Collection of index types to skip per column
+ private Map<String, Set<FieldConfig.IndexType>> _indexSkipConfig;
private QueryContext(@Nullable String tableName, @Nullable QueryContext subquery,
List<ExpressionContext> selectExpressions, boolean distinct, List<String> aliasList,
@@ -428,6 +432,21 @@ public class QueryContext {
+ ", _expressionOverrideHints=" + _expressionOverrideHints + ", _explain=" + _explain + '}';
}
+ public void setIndexSkipConfig(Map<String, Set<FieldConfig.IndexType>> indexSkipConfig) {
+ _indexSkipConfig = indexSkipConfig;
+ }
+
+ public boolean isIndexUseAllowed(String columnName, FieldConfig.IndexType indexType) {
+ if (_indexSkipConfig == null) {
+ return true;
+ }
+ return !_indexSkipConfig.getOrDefault(columnName, Collections.EMPTY_SET).contains(indexType);
+ }
+
+ public boolean isIndexUseAllowed(DataSource dataSource, FieldConfig.IndexType indexType) {
+ return isIndexUseAllowed(dataSource.getColumnName(), indexType);
+ }
+
public static class Builder {
private String _tableName;
private QueryContext _subquery;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index d58712b308..8c62a0e4dd 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -94,6 +94,7 @@ import org.testng.annotations.Test;
import static org.apache.pinot.common.function.scalar.StringFunctions.*;
import static org.apache.pinot.controller.helix.core.PinotHelixResourceManager.EXTERNAL_VIEW_CHECK_INTERVAL_MS;
import static org.apache.pinot.controller.helix.core.PinotHelixResourceManager.EXTERNAL_VIEW_ONLINE_SEGMENTS_MAX_WAIT_MS;
+import static org.apache.pinot.spi.utils.CommonConstants.Broker.Request.QueryOptionKey.INDEX_SKIP_CONFIG;
import static org.testng.Assert.*;
@@ -598,7 +599,7 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
assertEquals(getTableSize(getTableName()), DISK_SIZE_IN_BYTES);
}
- private void addInvertedIndex()
+ private void addInvertedIndex(boolean shouldReload)
throws Exception {
// Update table config to add inverted index on DivActualElapsedTime column, and
// reload the table to get config change into effect and add the inverted index.
@@ -610,8 +611,36 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
// After all segments are reloaded, the inverted index is added on DivActualElapsedTime.
// It's expected to have numEntriesScannedInFilter equal to 0, i.e. no docs is scanned
// at filtering stage when inverted index can answer the predicate directly.
- reloadAllSegments(TEST_UPDATED_INVERTED_INDEX_QUERY, false, getCountStarResult());
- assertEquals(postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ if (shouldReload) {
+ reloadAllSegments(TEST_UPDATED_INVERTED_INDEX_QUERY, false, getCountStarResult());
+ assertEquals(postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ }
+ }
+ private void addInvertedIndex()
+ throws Exception {
+ addInvertedIndex(true);
+ }
+
+ private void addRangeIndex(boolean shouldReload)
+ throws Exception {
+ // Update table config to add Range index on DivActualElapsedTime column, and
+ // reload the table to get config change into effect and add the Range index.
+ TableConfig tableConfig = getOfflineTableConfig();
+ tableConfig.getIndexingConfig().setRangeIndexColumns(UPDATED_RANGE_INDEX_COLUMNS);
+ updateTableConfig(tableConfig);
+
+ // It takes a while to reload multiple segments, thus we retry the query for some time.
+ // After all segments are reloaded, the range index is added on DivActualElapsedTime.
+ // It's expected to have numEntriesScannedInFilter equal to 0, i.e. no docs is scanned
+ // at filtering stage when inverted index can answer the predicate directly.
+ if (shouldReload) {
+ reloadAllSegments(TEST_UPDATED_RANGE_INDEX_QUERY, false, getCountStarResult());
+ assertEquals(postQuery(TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ }
+ }
+
+ private void addRangeIndex() throws Exception {
+ addRangeIndex(true);
}
@Test(dataProvider = "useBothQueryEngines")
@@ -1277,14 +1306,10 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
assertEquals(postQuery(TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs);
// Update table config and trigger reload
- TableConfig tableConfig = getOfflineTableConfig();
- tableConfig.getIndexingConfig().setRangeIndexColumns(UPDATED_RANGE_INDEX_COLUMNS);
- updateTableConfig(tableConfig);
- reloadAllSegments(TEST_UPDATED_RANGE_INDEX_QUERY, false, numTotalDocs);
- assertEquals(postQuery(TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ addRangeIndex();
// Update table config to remove the new range index, and check if the new range index is removed
- tableConfig = getOfflineTableConfig();
+ TableConfig tableConfig = getOfflineTableConfig();
tableConfig.getIndexingConfig().setRangeIndexColumns(getRangeIndexColumns());
updateTableConfig(tableConfig);
reloadAllSegments(TEST_UPDATED_RANGE_INDEX_QUERY, true, numTotalDocs);
@@ -3261,4 +3286,58 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
testQuery("SELECT BOOL_AND(CAST(Cancelled AS BOOLEAN)) FROM mytable");
testQuery("SELECT BOOL_OR(CAST(Diverted AS BOOLEAN)) FROM mytable");
}
+
+ private String buildIndexSkipConfig(String columnsAndIndexes) {
+ return "SET " + INDEX_SKIP_CONFIG + "='" + columnsAndIndexes + "'; ";
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testIndexSkipConfig(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ long numTotalDocs = getCountStarResult();
+ assertEquals(postQuery(TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs);
+
+ // Update table config to add range and inverted index, and trigger reload
+ addRangeIndex(false); // skip segment reload and instead reload after also adding inverted index
+ addInvertedIndex();
+
+ // Ensure inv index is operational
+ assertEquals(postQuery(TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+
+ // disallow use of range index on DivActualElapsedTime, inverted should be unaffected
+ String indexSkipConf = buildIndexSkipConfig("DivActualElapsedTime=range");
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs);
+
+ // disallow use of inverted index on DivActualElapsedTime, range should be unaffected
+ indexSkipConf = buildIndexSkipConfig("DivActualElapsedTime=inverted");
+ // Confirm that inverted index is not used
+ assertFalse(postQuery(indexSkipConf + " EXPLAIN PLAN FOR " + TEST_UPDATED_INVERTED_INDEX_QUERY).toString()
+ .contains("FILTER_INVERTED_INDEX"));
+
+ // EQ predicate type allows for using range index if one exists, even if inverted index is skipped. That is why
+ // we still see no docs scanned even though we skip the inverted index. This is a good test to show that using
+ // the indexSkipConfig can allow fine-grained experimentation of index usage at query time.
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), 0L);
+
+ // disallow use of both range and inverted indexes on DivActualElapsedTime, neither should be used at query time
+ indexSkipConf = buildIndexSkipConfig("DivActualElapsedTime=inverted,range");
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_INVERTED_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs);
+ assertEquals(postQuery(
+ indexSkipConf + TEST_UPDATED_RANGE_INDEX_QUERY).get("numEntriesScannedInFilter").asLong(), numTotalDocs);
+
+ // Update table config to remove the new indexes, and check if the new indexes are removed
+ TableConfig tableConfig = getOfflineTableConfig();
+ tableConfig.getIndexingConfig().setRangeIndexColumns(getRangeIndexColumns());
+ tableConfig.getIndexingConfig().setInvertedIndexColumns(getInvertedIndexColumns());
+ updateTableConfig(tableConfig);
+ reloadAllSegments(TEST_UPDATED_RANGE_INDEX_QUERY, true, numTotalDocs);
+ }
}
diff --git a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/DataSource.java b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/DataSource.java
index 8775c14e12..73f294e458 100644
--- a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/DataSource.java
+++ b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/datasource/DataSource.java
@@ -50,6 +50,13 @@ public interface DataSource {
*/
ForwardIndexReader<?> getForwardIndex();
+ /**
+ * Returns the column name to which this data source pertains
+ */
+ default String getColumnName() {
+ return getDataSourceMetadata().getFieldSpec().getName();
+ }
+
/**
* Returns the dictionary for the column if it is dictionary-encoded, or {@code null} if not.
*/
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 2c3b5e8cd2..3131e2b42b 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -360,6 +360,7 @@ public class CommonConstants {
public static final String SERVER_RETURN_FINAL_RESULT = "serverReturnFinalResult";
// Reorder scan based predicates based on cardinality and number of selected values
public static final String AND_SCAN_REORDERING = "AndScanReordering";
+ public static final String INDEX_SKIP_CONFIG = "indexSkipConfig";
public static final String ORDER_BY_ALGORITHM = "orderByAlgorithm";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org