You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ma...@apache.org on 2023/03/27 21:07:24 UTC
[pinot] branch master updated: Adding fields to enable/disable dictionary optimization. (#10484)
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new fd08b906c0 Adding fields to enable/disable dictionary optimization. (#10484)
fd08b906c0 is described below
commit fd08b906c09ca89fb7d9c5c8ab5a5ceaf2d18fe4
Author: swaminathanmanish <12...@users.noreply.github.com>
AuthorDate: Mon Mar 27 14:07:13 2023 -0700
Adding fields to enable/disable dictionary optimization. (#10484)
---
.../pinot/common/utils/config/TableConfigSerDeTest.java | 7 +++++--
.../pinot/spi/utils/builder/TableConfigBuilder.java | 16 ++++++++++++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
index 30dbfe80b4..6436058b59 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeTest.java
@@ -64,11 +64,12 @@ import static org.testng.Assert.*;
public class TableConfigSerDeTest {
-
+ private static final double NO_DICTIONARY_THRESHOLD_RATIO = 0.72;
@Test
public void testSerDe()
throws IOException {
- TableConfigBuilder tableConfigBuilder = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable");
+ TableConfigBuilder tableConfigBuilder = new TableConfigBuilder(TableType.OFFLINE).setTableName("testTable")
+ .setNoDictionarySizeRatioThreshold(NO_DICTIONARY_THRESHOLD_RATIO).setOptimizeDictionaryForMetrics(true);
{
// Default table config
TableConfig tableConfig = tableConfigBuilder.build();
@@ -367,6 +368,8 @@ public class TableConfigSerDeTest {
assertNull(tableConfig.getInstanceAssignmentConfigMap());
assertNull(tableConfig.getSegmentAssignmentConfigMap());
assertNull(tableConfig.getFieldConfigList());
+ assertEquals(tableConfig.getIndexingConfig().isOptimizeDictionaryForMetrics(), true);
+ assertEquals(tableConfig.getIndexingConfig().getNoDictionarySizeRatioThreshold(), NO_DICTIONARY_THRESHOLD_RATIO);
// Serialize
ObjectNode tableConfigJson = (ObjectNode) tableConfig.toJsonNode();
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
index 4916668eed..ba87dee5c9 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/TableConfigBuilder.java
@@ -103,6 +103,10 @@ public class TableConfigBuilder {
private List<StarTreeIndexConfig> _starTreeIndexConfigs;
private List<String> _jsonIndexColumns;
private boolean _aggregateMetrics;
+ private boolean _optimizeDictionaryForMetrics;
+ // This threshold determines if dictionary should be enabled or not for a metric column and is relevant
+ // only when _optimizeDictionaryForMetrics is set to true.
+ private double _noDictionarySizeRatioThreshold;
private TableCustomConfig _customConfig;
private QuotaConfig _quotaConfig;
@@ -256,6 +260,16 @@ public class TableConfigBuilder {
return this;
}
+ public TableConfigBuilder setOptimizeDictionaryForMetrics(boolean optimizeDictionaryForMetrics) {
+ _optimizeDictionaryForMetrics = optimizeDictionaryForMetrics;
+ return this;
+ }
+
+ public TableConfigBuilder setNoDictionarySizeRatioThreshold(double noDictionarySizeRatioThreshold) {
+ _noDictionarySizeRatioThreshold = noDictionarySizeRatioThreshold;
+ return this;
+ }
+
public TableConfigBuilder setCreateInvertedIndexDuringSegmentGeneration(
boolean createInvertedIndexDuringSegmentGeneration) {
_createInvertedIndexDuringSegmentGeneration = createInvertedIndexDuringSegmentGeneration;
@@ -444,6 +458,8 @@ public class TableConfigBuilder {
indexingConfig.setStarTreeIndexConfigs(_starTreeIndexConfigs);
indexingConfig.setJsonIndexColumns(_jsonIndexColumns);
indexingConfig.setAggregateMetrics(_aggregateMetrics);
+ indexingConfig.setOptimizeDictionaryForMetrics(_optimizeDictionaryForMetrics);
+ indexingConfig.setNoDictionarySizeRatioThreshold(_noDictionarySizeRatioThreshold);
if (_customConfig == null) {
_customConfig = new TableCustomConfig(null);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org