You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2025/02/05 22:37:34 UTC
(pinot) branch master updated: Add segment StarTree index rebuild throttler configurable via ZK cluster configs (#14943)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e9ee474e1e Add segment StarTree index rebuild throttler configurable via ZK cluster configs (#14943)
e9ee474e1e is described below
commit e9ee474e1e9b38ade49c66141d6241c0225372f1
Author: Sonam Mandal <so...@startree.ai>
AuthorDate: Wed Feb 5 14:37:27 2025 -0800
Add segment StarTree index rebuild throttler configurable via ZK cluster configs (#14943)
---
.../BaseTableDataManagerAcquireSegmentTest.java | 5 +-
.../data/manager/BaseTableDataManagerTest.java | 6 +-
.../pinot/core/data/manager/TableIndexingTest.java | 6 +-
.../offline/DimensionTableDataManagerTest.java | 6 +-
.../tests/BaseRealtimeClusterIntegrationTest.java | 3 +
.../tests/HybridClusterIntegrationTest.java | 3 +
.../tests/OfflineClusterIntegrationTest.java | 3 +
.../StarTreeFunctionParametersIntegrationTest.java | 9 +
.../immutable/ImmutableSegmentLoader.java | 12 +-
.../segment/index/loader/SegmentPreProcessor.java | 46 +-
.../utils/BaseSegmentPreprocessThrottler.java | 215 ++++++
.../utils/SegmentAllIndexPreprocessThrottler.java | 61 ++
.../local/utils/SegmentPreprocessThrottler.java | 208 +-----
.../utils/SegmentStarTreePreprocessThrottler.java | 63 ++
.../local/segment/index/loader/LoaderTest.java | 7 +-
.../index/loader/SegmentPreProcessorTest.java | 21 +-
.../utils/SegmentPreprocessThrottlerTest.java | 807 +++++++++++++--------
.../server/starter/helix/BaseServerStarter.java | 21 +-
.../helix/HelixInstanceDataManagerConfig.java | 2 +-
.../apache/pinot/spi/utils/CommonConstants.java | 8 +
20 files changed, 989 insertions(+), 523 deletions(-)
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
index c4114ada52..93c490fd4f 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerAcquireSegmentTest.java
@@ -35,8 +35,10 @@ import org.apache.pinot.core.data.manager.offline.ImmutableSegmentDataManager;
import org.apache.pinot.core.data.manager.offline.OfflineTableDataManager;
import org.apache.pinot.segment.local.data.manager.SegmentDataManager;
import org.apache.pinot.segment.local.data.manager.TableDataManager;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentLocks;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.config.instance.InstanceDataManagerConfig;
@@ -118,7 +120,8 @@ public class BaseTableDataManagerAcquireSegmentTest {
when(instanceDataManagerConfig.getDeletedSegmentsCacheSize()).thenReturn(DELETED_SEGMENTS_CACHE_SIZE);
when(instanceDataManagerConfig.getDeletedSegmentsCacheTtlMinutes()).thenReturn(DELETED_SEGMENTS_TTL_MINUTES);
TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).build();
- SegmentPreprocessThrottler segmentPreprocessThrottler = new SegmentPreprocessThrottler(8, 10, true);
+ SegmentPreprocessThrottler segmentPreprocessThrottler = new SegmentPreprocessThrottler(
+ new SegmentAllIndexPreprocessThrottler(8, 10, true), new SegmentStarTreePreprocessThrottler(4, 8, true));
TableDataManager tableDataManager = new OfflineTableDataManager();
tableDataManager.init(instanceDataManagerConfig, mock(HelixManager.class), new SegmentLocks(), tableConfig, null,
null, segmentPreprocessThrottler);
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
index 8d4993b9e5..bff09b7ea8 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java
@@ -40,8 +40,10 @@ import org.apache.pinot.core.data.manager.offline.OfflineTableDataManager;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentLocks;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.V1Constants;
@@ -97,8 +99,8 @@ public class BaseTableDataManagerTest {
private static final Schema SCHEMA =
new Schema.SchemaBuilder().setSchemaName(RAW_TABLE_NAME).addSingleValueDimension(STRING_COLUMN, DataType.STRING)
.addMetric(LONG_COLUMN, DataType.LONG).build();
- static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(2, 4,
- true);
+ static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(
+ new SegmentAllIndexPreprocessThrottler(2, 4, true), new SegmentStarTreePreprocessThrottler(2, 4, true));
@BeforeClass
public void setUp()
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java
index f9f93e6032..d878bc44ba 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/TableIndexingTest.java
@@ -38,7 +38,9 @@ import org.apache.pinot.segment.local.indexsegment.immutable.ImmutableSegmentLoa
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.local.utils.TableConfigUtils;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
@@ -81,8 +83,8 @@ public class TableIndexingTest {
private static final File TEMP_DIR = new File(FileUtils.getTempDirectory(), "TableIndexingTest");
private static final String TABLE_NAME = "mytable";
private static final String OFFLINE_TABLE_NAME = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME);
- private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(2, 4,
- true);
+ private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(
+ new SegmentAllIndexPreprocessThrottler(2, 4, true), new SegmentStarTreePreprocessThrottler(1, 2, true));
public static final String COLUMN_NAME = "col";
public static final String COLUMN_DAY_NAME = "$col$DAY";
public static final String COLUMN_MONTH_NAME = "$col$MONTH";
diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java
index 8c605be2cc..db80535192 100644
--- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/offline/DimensionTableDataManagerTest.java
@@ -40,8 +40,10 @@ import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentCreationDriverFactory;
import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig;
import org.apache.pinot.segment.local.segment.index.loader.LoaderTest;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentLocks;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.segment.spi.creator.SegmentGeneratorConfig;
import org.apache.pinot.segment.spi.creator.SegmentIndexCreationDriver;
@@ -77,8 +79,8 @@ public class DimensionTableDataManagerTest {
private static final String CSV_DATA_PATH = "data/dimBaseballTeams.csv";
private static final String SCHEMA_PATH = "data/dimBaseballTeams_schema.json";
private static final String TABLE_CONFIG_PATH = "data/dimBaseballTeams_config.json";
- private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(1, 2,
- true);
+ private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(
+ new SegmentAllIndexPreprocessThrottler(1, 2, true), new SegmentStarTreePreprocessThrottler(1, 2, true));
private File _indexDir;
private IndexLoadingConfig _indexLoadingConfig;
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java
index b8574cf003..fe29e7e4e4 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/BaseRealtimeClusterIntegrationTest.java
@@ -56,6 +56,9 @@ public abstract class BaseRealtimeClusterIntegrationTest extends BaseClusterInte
// Set max segment preprocess parallelism to 8
_helixManager.getConfigAccessor()
.set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, Integer.toString(8));
+ // Set max segment startree preprocess parallelism to 6
+ _helixManager.getConfigAccessor()
+ .set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM, Integer.toString(6));
startBroker();
startServer();
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
index 010dd9b6c6..73c5d91c0a 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/HybridClusterIntegrationTest.java
@@ -131,6 +131,9 @@ public class HybridClusterIntegrationTest extends BaseClusterIntegrationTestSet
// Set max segment preprocess parallelism to 10
_helixManager.getConfigAccessor()
.set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, Integer.toString(10));
+ // Set max segment startree preprocess parallelism to 6
+ _helixManager.getConfigAccessor()
+ .set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM, Integer.toString(6));
startBroker();
startServers(2);
startKafka();
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
index daa826de22..c31c644985 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/OfflineClusterIntegrationTest.java
@@ -196,6 +196,9 @@ public class OfflineClusterIntegrationTest extends BaseClusterIntegrationTestSet
// Set max segment preprocess parallelism to 8 to test that all segments can be processed
_helixManager.getConfigAccessor()
.set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, Integer.toString(8));
+ // Set max segment startree preprocess parallelism to 6 to test that all segments can be processed
+ _helixManager.getConfigAccessor()
+ .set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM, Integer.toString(6));
startBrokers();
startServers();
diff --git a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeFunctionParametersIntegrationTest.java b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeFunctionParametersIntegrationTest.java
index 04e38cc1a6..f79fba2194 100644
--- a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeFunctionParametersIntegrationTest.java
+++ b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/StarTreeFunctionParametersIntegrationTest.java
@@ -26,12 +26,15 @@ import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.apache.commons.io.FileUtils;
+import org.apache.helix.model.HelixConfigScope;
+import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.pinot.segment.spi.Constants;
import org.apache.pinot.spi.config.table.StarTreeAggregationConfig;
import org.apache.pinot.spi.config.table.StarTreeIndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.apache.pinot.util.TestUtils;
import org.testng.annotations.AfterClass;
@@ -53,6 +56,12 @@ public class StarTreeFunctionParametersIntegrationTest extends BaseClusterIntegr
// Start the Pinot cluster
startZk();
startController();
+ HelixConfigScope scope =
+ new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(getHelixClusterName())
+ .build();
+ // Set max segment startree preprocess parallelism to 6
+ _helixManager.getConfigAccessor()
+ .set(scope, CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM, Integer.toString(8));
startBroker();
startServer();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
index 4650c832e3..40c943abbc 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/indexsegment/immutable/ImmutableSegmentLoader.java
@@ -202,14 +202,15 @@ public class ImmutableSegmentLoader {
SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir);
if (segmentMetadata.getTotalDocs() > 0) {
if (segmentPreprocessThrottler != null) {
- segmentPreprocessThrottler.acquire();
+ segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().acquire();
}
try {
convertSegmentFormat(indexDir, indexLoadingConfig, segmentMetadata);
- preprocessSegment(indexDir, segmentMetadata.getName(), segmentMetadata.getCrc(), indexLoadingConfig, schema);
+ preprocessSegment(indexDir, segmentMetadata.getName(), segmentMetadata.getCrc(), indexLoadingConfig, schema,
+ segmentPreprocessThrottler);
} finally {
if (segmentPreprocessThrottler != null) {
- segmentPreprocessThrottler.release();
+ segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().release();
}
}
}
@@ -322,7 +323,8 @@ public class ImmutableSegmentLoader {
}
private static void preprocessSegment(File indexDir, String segmentName, String segmentCrc,
- IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema)
+ IndexLoadingConfig indexLoadingConfig, @Nullable Schema schema,
+ @Nullable SegmentPreprocessThrottler segmentPreprocessThrottler)
throws Exception {
PinotConfiguration segmentDirectoryConfigs = indexLoadingConfig.getSegmentDirectoryConfigs();
SegmentDirectoryLoaderContext segmentLoaderContext =
@@ -337,7 +339,7 @@ public class ImmutableSegmentLoader {
SegmentDirectory segmentDirectory =
SegmentDirectoryLoaderRegistry.getDefaultSegmentDirectoryLoader().load(indexDir.toURI(), segmentLoaderContext);
try (SegmentPreProcessor preProcessor = new SegmentPreProcessor(segmentDirectory, indexLoadingConfig, schema)) {
- preProcessor.process();
+ preProcessor.process(segmentPreprocessThrottler);
}
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
index 197924473e..34e9d20325 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessor.java
@@ -33,6 +33,7 @@ import org.apache.pinot.segment.local.segment.index.loader.invertedindex.Inverte
import org.apache.pinot.segment.local.startree.StarTreeBuilderUtils;
import org.apache.pinot.segment.local.startree.v2.builder.MultipleTreesBuilder;
import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
+import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.index.IndexHandler;
import org.apache.pinot.segment.spi.index.IndexService;
@@ -81,6 +82,11 @@ public class SegmentPreProcessor implements AutoCloseable {
public void process()
throws Exception {
+ process(null);
+ }
+
+ public void process(@Nullable SegmentPreprocessThrottler segmentPreprocessThrottler)
+ throws Exception {
if (_segmentMetadata.getTotalDocs() == 0) {
LOGGER.info("Skip preprocessing empty segment: {}", _segmentMetadata.getName());
return;
@@ -154,7 +160,7 @@ public class SegmentPreProcessor implements AutoCloseable {
// that the other required indices (e.g. forward index) are up-to-date.
try (SegmentDirectory.Writer segmentWriter = _segmentDirectory.createWriter()) {
// Create/modify/remove star-trees if required.
- processStarTrees(indexDir);
+ processStarTrees(indexDir, segmentPreprocessThrottler);
_segmentDirectory.reloadMetadata();
segmentWriter.save();
}
@@ -238,7 +244,8 @@ public class SegmentPreProcessor implements AutoCloseable {
return !starTreeBuilderConfigs.isEmpty();
}
- private void processStarTrees(File indexDir)
+ private void processStarTrees(File indexDir,
+ @Nullable SegmentPreprocessThrottler segmentPreprocessThrottler)
throws Exception {
// Create/modify/remove star-trees if required
if (_indexLoadingConfig.isEnableDynamicStarTreeCreation()) {
@@ -246,14 +253,13 @@ public class SegmentPreProcessor implements AutoCloseable {
.generateBuilderConfigs(_indexLoadingConfig.getStarTreeIndexConfigs(),
_indexLoadingConfig.isEnableDefaultStarTree(), _segmentMetadata);
boolean shouldGenerateStarTree = !starTreeBuilderConfigs.isEmpty();
+ boolean shouldRemoveStarTree = false;
List<StarTreeV2Metadata> starTreeMetadataList = _segmentMetadata.getStarTreeV2MetadataList();
if (starTreeMetadataList != null) {
// There are existing star-trees
if (!shouldGenerateStarTree) {
// Newer config does not have star-trees. Delete all existing star-trees.
- LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName());
- StarTreeBuilderUtils.removeStarTrees(indexDir);
- _segmentMetadata = new SegmentMetadataImpl(indexDir);
+ shouldRemoveStarTree = true;
} else if (StarTreeBuilderUtils.shouldModifyExistingStarTrees(starTreeBuilderConfigs, starTreeMetadataList)) {
// Existing and newer both have star-trees, but they don't match. Rebuild the star-trees.
LOGGER.info("Change detected in star-trees for segment: {}", _segmentMetadata.getName());
@@ -262,14 +268,30 @@ public class SegmentPreProcessor implements AutoCloseable {
shouldGenerateStarTree = false;
}
}
- // Generate the star-trees if needed
- if (shouldGenerateStarTree) {
- // NOTE: Always use OFF_HEAP mode on server side.
- try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, indexDir,
- MultipleTreesBuilder.BuildMode.OFF_HEAP)) {
- builder.build();
+ // Generate/remove the star-trees if needed
+ if (shouldGenerateStarTree || shouldRemoveStarTree) {
+ if (segmentPreprocessThrottler != null) {
+ segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().acquire();
+ }
+
+ try {
+ if (shouldRemoveStarTree) {
+ // 'shouldGenerateStarTree' should be false if they need to be removed
+ LOGGER.info("Removing star-trees from segment: {}", _segmentMetadata.getName());
+ StarTreeBuilderUtils.removeStarTrees(indexDir);
+ } else {
+ // NOTE: Always use OFF_HEAP mode on server side.
+ try (MultipleTreesBuilder builder = new MultipleTreesBuilder(starTreeBuilderConfigs, indexDir,
+ MultipleTreesBuilder.BuildMode.OFF_HEAP)) {
+ builder.build();
+ }
+ }
+ _segmentMetadata = new SegmentMetadataImpl(indexDir);
+ } finally {
+ if (segmentPreprocessThrottler != null) {
+ segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().release();
+ }
}
- _segmentMetadata = new SegmentMetadataImpl(indexDir);
}
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/BaseSegmentPreprocessThrottler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/BaseSegmentPreprocessThrottler.java
new file mode 100644
index 0000000000..500aac9871
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/BaseSegmentPreprocessThrottler.java
@@ -0,0 +1,215 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.util.Map;
+import java.util.Set;
+import org.apache.pinot.common.concurrency.AdjustableSemaphore;
+import org.apache.pinot.spi.config.provider.PinotClusterConfigChangeListener;
+import org.slf4j.Logger;
+
+
+/**
+ * Base class for segment preprocess throttlers, contains the common logic for the semaphore and handling the pre and
+ * post query serving values. The semaphore cannot be null and must contain > 0 total permits
+ */
+public abstract class BaseSegmentPreprocessThrottler implements PinotClusterConfigChangeListener {
+
+ protected AdjustableSemaphore _semaphore;
+ /**
+ * _maxPreprocessConcurrency and _maxPreprocessConcurrencyBeforeServingQueries must be > 0. To effectively disable
+ * throttling, this can be set to a very high value
+ */
+ protected int _maxPreprocessConcurrency;
+ protected int _maxPreprocessConcurrencyBeforeServingQueries;
+ protected boolean _isServingQueries;
+ private final Logger _logger;
+
+ /**
+ * Base segment preprocess throttler constructor
+ * @param maxPreprocessConcurrency configured index preprocessing concurrency
+ * @param maxPreprocessConcurrencyBeforeServingQueries configured preprocessing concurrency before serving queries
+ * @param isServingQueries whether the server is ready to serve queries or not
+ * @param logger logger to use
+ */
+ public BaseSegmentPreprocessThrottler(int maxPreprocessConcurrency, int maxPreprocessConcurrencyBeforeServingQueries,
+ boolean isServingQueries, Logger logger) {
+ _logger = logger;
+ _logger.info("Initializing SegmentPreprocessThrottler, maxPreprocessConcurrency: {}, "
+ + "maxPreprocessConcurrencyBeforeServingQueries: {}, isServingQueries: {}",
+ maxPreprocessConcurrency, maxPreprocessConcurrencyBeforeServingQueries, isServingQueries);
+ Preconditions.checkArgument(maxPreprocessConcurrency > 0,
+ "Max preprocess parallelism must be > 0, but found to be: " + maxPreprocessConcurrency);
+ Preconditions.checkArgument(maxPreprocessConcurrencyBeforeServingQueries > 0,
+ "Max preprocess parallelism before serving queries must be > 0, but found to be: "
+ + maxPreprocessConcurrencyBeforeServingQueries);
+
+ _maxPreprocessConcurrency = maxPreprocessConcurrency;
+ _maxPreprocessConcurrencyBeforeServingQueries = maxPreprocessConcurrencyBeforeServingQueries;
+ _isServingQueries = isServingQueries;
+
+ // maxPreprocessConcurrencyBeforeServingQueries is only used prior to serving queries and once the server is
+ // ready to serve queries this is not used again. This too is configurable via ZK CLUSTER config updates while the
+ // server is starting up.
+ if (!isServingQueries) {
+ logger.info("Serving queries is disabled, using preprocess concurrency as: {}",
+ _maxPreprocessConcurrencyBeforeServingQueries);
+ }
+
+ _semaphore = new AdjustableSemaphore(
+ _isServingQueries ? _maxPreprocessConcurrency : _maxPreprocessConcurrencyBeforeServingQueries, true);
+ _logger.info("Created semaphore with total permits: {}, available permits: {}", totalPermits(),
+ availablePermits());
+ }
+
+ public synchronized void startServingQueries() {
+ _logger.info("Serving queries is to be enabled, reset throttling threshold for segment preprocess concurrency, "
+ + "total permits: {}, available permits: {}", totalPermits(), availablePermits());
+ _isServingQueries = true;
+ _semaphore.setPermits(_maxPreprocessConcurrency);
+ _logger.info("Reset throttling completed, new concurrency: {}, total permits: {}, available permits: {}",
+ _maxPreprocessConcurrency, totalPermits(), availablePermits());
+ }
+
+ protected void handleMaxPreprocessConcurrencyChange(Set<String> changedConfigs, Map<String, String> clusterConfigs,
+ String configName, String defaultConfigValue) {
+ if (!changedConfigs.contains(configName)) {
+ _logger.info("changedConfigs list indicates config: {} was not updated, skipping updates", configName);
+ return;
+ }
+
+ String maxParallelSegmentPreprocessesStr =
+ clusterConfigs == null ? defaultConfigValue : clusterConfigs.getOrDefault(configName, defaultConfigValue);
+
+ int maxPreprocessConcurrency;
+ try {
+ maxPreprocessConcurrency = Integer.parseInt(maxParallelSegmentPreprocessesStr);
+ } catch (Exception e) {
+ _logger.warn("Invalid config {} set to: {}, not making change, fix config and try again", configName,
+ maxParallelSegmentPreprocessesStr);
+ return;
+ }
+
+ if (maxPreprocessConcurrency <= 0) {
+ _logger.warn("config {}: {} must be > 0, not making change, fix config and try again", configName,
+ maxPreprocessConcurrency);
+ return;
+ }
+
+ if (maxPreprocessConcurrency == _maxPreprocessConcurrency) {
+ _logger.info("No ZK update for config {}, value: {}, total permits: {}", configName, _maxPreprocessConcurrency,
+ totalPermits());
+ return;
+ }
+
+ _logger.info("Updated config: {} from: {} to: {}", configName, _maxPreprocessConcurrency,
+ maxPreprocessConcurrency);
+ _maxPreprocessConcurrency = maxPreprocessConcurrency;
+
+ if (!_isServingQueries) {
+ _logger.info("Serving queries hasn't been enabled yet, not updating the permits with config {}", configName);
+ return;
+ }
+ _semaphore.setPermits(_maxPreprocessConcurrency);
+ _logger.info("Updated total permits: {}", totalPermits());
+ }
+
+ protected void handleMaxPreprocessConcurrencyBeforeServingQueriesChange(Set<String> changedConfigs,
+ Map<String, String> clusterConfigs, String configName, String defaultConfigValue) {
+ if (!changedConfigs.contains(configName)) {
+ _logger.info("changedConfigs list indicates config: {} was not updated, skipping updates", configName);
+ return;
+ }
+
+ String maxParallelSegmentPreprocessesBeforeServingQueriesStr =
+ clusterConfigs == null ? defaultConfigValue : clusterConfigs.getOrDefault(configName, defaultConfigValue);
+
+ int maxPreprocessConcurrencyBeforeServingQueries;
+ try {
+ maxPreprocessConcurrencyBeforeServingQueries =
+ Integer.parseInt(maxParallelSegmentPreprocessesBeforeServingQueriesStr);
+ } catch (Exception e) {
+ _logger.warn("Invalid config {} set to: {}, not making change, fix config and try again", configName,
+ maxParallelSegmentPreprocessesBeforeServingQueriesStr);
+ return;
+ }
+
+ if (maxPreprocessConcurrencyBeforeServingQueries <= 0) {
+ _logger.warn("config {}: {} must be > 0, not making change, fix config and try again", configName,
+ maxPreprocessConcurrencyBeforeServingQueries);
+ return;
+ }
+
+ if (maxPreprocessConcurrencyBeforeServingQueries == _maxPreprocessConcurrencyBeforeServingQueries) {
+ _logger.info("No ZK update for config: {} value: {}, total permits: {}", configName,
+ _maxPreprocessConcurrencyBeforeServingQueries, totalPermits());
+ return;
+ }
+
+ _logger.info("Updated config: {} from: {} to: {}", configName, _maxPreprocessConcurrencyBeforeServingQueries,
+ maxPreprocessConcurrencyBeforeServingQueries);
+ _maxPreprocessConcurrencyBeforeServingQueries = maxPreprocessConcurrencyBeforeServingQueries;
+ if (!_isServingQueries) {
+ _logger.info("config: {} was updated before serving queries was enabled, updating the permits", configName);
+ _semaphore.setPermits(_maxPreprocessConcurrencyBeforeServingQueries);
+ _logger.info("Updated total permits: {}", totalPermits());
+ }
+ }
+
+ /**
+ * Block trying to acquire the semaphore to perform the segment StarTree index rebuild steps unless interrupted.
+ * <p>
+ * {@link #release()} should be called after the segment preprocess completes. It is the responsibility of the caller
+ * to ensure that {@link #release()} is called exactly once for each call to this method.
+ *
+ * @throws InterruptedException if the current thread is interrupted
+ */
+ public void acquire()
+ throws InterruptedException {
+ _semaphore.acquire();
+ }
+
+ /**
+ * Should be called after the segment StarTree index build completes. It is the responsibility of the caller to
+ * ensure that this method is called exactly once for each call to {@link #acquire()}.
+ */
+ public void release() {
+ _semaphore.release();
+ }
+
+ /**
+ * Get the number of available permits
+ * @return number of available permits
+ */
+ @VisibleForTesting
+ protected int availablePermits() {
+ return _semaphore.availablePermits();
+ }
+
+ /**
+ * Get the total number of permits
+ * @return total number of permits
+ */
+ @VisibleForTesting
+ protected int totalPermits() {
+ return _semaphore.getTotalPermits();
+ }
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentAllIndexPreprocessThrottler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentAllIndexPreprocessThrottler.java
new file mode 100644
index 0000000000..6b94945e31
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentAllIndexPreprocessThrottler.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.utils;
+
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Used to throttle the total concurrent index rebuilds (for all indexes) on a given Pinot server.
+ */
+public class SegmentAllIndexPreprocessThrottler extends BaseSegmentPreprocessThrottler {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SegmentAllIndexPreprocessThrottler.class);
+
+ /**
+ * @param maxPreprocessConcurrency configured preprocessing concurrency
+ * @param maxPreprocessConcurrencyBeforeServingQueries configured preprocessing concurrency before serving queries
+ * @param isServingQueries whether the server is ready to serve queries or not
+ */
+ public SegmentAllIndexPreprocessThrottler(int maxPreprocessConcurrency,
+ int maxPreprocessConcurrencyBeforeServingQueries, boolean isServingQueries) {
+ super(maxPreprocessConcurrency, maxPreprocessConcurrencyBeforeServingQueries, isServingQueries, LOGGER);
+ }
+
+ @Override
+ public synchronized void onChange(Set<String> changedConfigs, Map<String, String> clusterConfigs) {
+ if (CollectionUtils.isEmpty(changedConfigs)) {
+ LOGGER.info("Skip updating SegmentAllIndexPreprocessThrottler configs with unchanged clusterConfigs");
+ return;
+ }
+
+ LOGGER.info("Updating SegmentAllIndexPreprocessThrottler configs with latest clusterConfigs");
+ handleMaxPreprocessConcurrencyChange(changedConfigs, clusterConfigs,
+ CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM);
+ handleMaxPreprocessConcurrencyBeforeServingQueriesChange(changedConfigs, clusterConfigs,
+ CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ LOGGER.info("Updated SegmentAllIndexPreprocessThrottler configs with latest clusterConfigs");
+ }
+}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottler.java
index 072b229f45..4e9a68fd35 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottler.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottler.java
@@ -18,208 +18,52 @@
*/
package org.apache.pinot.segment.local.utils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.pinot.common.concurrency.AdjustableSemaphore;
import org.apache.pinot.spi.config.provider.PinotClusterConfigChangeListener;
-import org.apache.pinot.spi.utils.CommonConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
- * Used to throttle the total concurrent index rebuilds that can happen on a given Pinot server.
+ * Contains all the segment preprocess throttlers used to control the total index rebuilds that can happen on a given
+ * Pinot server. For now this class supports index rebuild throttling at the following levels:
+ * - All index throttling
+ * - StarTree index throttling
* Code paths that do no need to rebuild the index or which don't happen on the server need not utilize this throttler.
+ * The throttlers passed in for now cannot be 'null', instead for code paths that do not need throttling, this object
+ * itself will be passed in as 'null'.
*/
public class SegmentPreprocessThrottler implements PinotClusterConfigChangeListener {
- private static final Logger LOGGER = LoggerFactory.getLogger(SegmentPreprocessThrottler.class);
- /**
- * _maxPreprocessConcurrency and _maxConcurrentPreprocessesBeforeServingQueries must be > 0. To effectively disable
- * throttling, this can be set to a very high value
- */
- private int _maxPreprocessConcurrency;
- private int _maxPreprocessConcurrencyBeforeServingQueries;
- private boolean _isServingQueries;
- private final AdjustableSemaphore _semaphore;
+ private final SegmentAllIndexPreprocessThrottler _segmentAllIndexPreprocessThrottler;
+ private final SegmentStarTreePreprocessThrottler _segmentStarTreePreprocessThrottler;
/**
- * @param maxPreprocessConcurrency configured preprocessing concurrency
- * @param maxPreprocessConcurrencyBeforeServingQueries configured preprocessing concurrency before serving queries
- * @param isServingQueries whether the server is ready to serve queries or not
+ * Constructor for SegmentPreprocessThrottler
+ * @param segmentAllIndexPreprocessThrottler segment preprocess throttler to use for all indexes
+ * @param segmentStarTreePreprocessThrottler segment preprocess throttler to use for StarTree index
*/
- public SegmentPreprocessThrottler(int maxPreprocessConcurrency, int maxPreprocessConcurrencyBeforeServingQueries,
- boolean isServingQueries) {
- LOGGER.info("Initializing SegmentPreprocessThrottler, maxPreprocessConcurrency: {}, "
- + "maxPreprocessConcurrencyBeforeServingQueries: {}, isServingQueries: {}",
- maxPreprocessConcurrency, maxPreprocessConcurrencyBeforeServingQueries, isServingQueries);
- Preconditions.checkArgument(maxPreprocessConcurrency > 0,
- "Max preprocess parallelism must be > 0, but found to be: " + maxPreprocessConcurrency);
- Preconditions.checkArgument(maxPreprocessConcurrencyBeforeServingQueries > 0,
- "Max preprocess parallelism before serving queries must be > 0, but found to be: "
- + maxPreprocessConcurrencyBeforeServingQueries);
-
- _maxPreprocessConcurrency = maxPreprocessConcurrency;
- _maxPreprocessConcurrencyBeforeServingQueries = maxPreprocessConcurrencyBeforeServingQueries;
- _isServingQueries = isServingQueries;
-
- // maxConcurrentPreprocessesBeforeServingQueries is only used prior to serving queries and once the server is
- // ready to serve queries this is not used again. This too is configurable via ZK CLUSTER config updates while the
- // server is starting up.
- int preprocessConcurrency = _maxPreprocessConcurrency;
- if (!isServingQueries) {
- preprocessConcurrency = _maxPreprocessConcurrencyBeforeServingQueries;
- LOGGER.info("Serving queries is disabled, setting preprocess concurrency to: {}", preprocessConcurrency);
- }
- _semaphore = new AdjustableSemaphore(preprocessConcurrency, true);
- LOGGER.info("Created semaphore with total permits: {}, available permits: {}", totalPermits(),
- availablePermits());
- }
-
- public synchronized void startServingQueries() {
- LOGGER.info("Serving queries is to be enabled, reset throttling threshold for segment preprocess concurrency, "
- + "total permits: {}, available permits: {}", totalPermits(), availablePermits());
- _isServingQueries = true;
- _semaphore.setPermits(_maxPreprocessConcurrency);
- LOGGER.info("Reset throttling completed, new concurrency: {}, total permits: {}, available permits: {}",
- _maxPreprocessConcurrency, totalPermits(), availablePermits());
- }
-
- @Override
- public synchronized void onChange(Set<String> changedConfigs, Map<String, String> clusterConfigs) {
- if (CollectionUtils.isEmpty(changedConfigs)) {
- LOGGER.info("Skip updating SegmentPreprocessThrottler configs with unchanged clusterConfigs");
- return;
- }
-
- LOGGER.info("Updating SegmentPreprocessThrottler configs with latest clusterConfigs");
- handleMaxPreprocessConcurrencyChange(changedConfigs, clusterConfigs);
- handleMaxPreprocessConcurrencyBeforeServingQueriesChange(changedConfigs, clusterConfigs);
- LOGGER.info("Updated SegmentPreprocessThrottler configs with latest clusterConfigs");
- }
-
- private void handleMaxPreprocessConcurrencyChange(Set<String> changedConfigs, Map<String, String> clusterConfigs) {
- if (!changedConfigs.contains(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM)) {
- LOGGER.info("changedConfigs list indicates maxPreprocessConcurrency was not updated, skipping updates");
- return;
- }
-
- String configName = CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM;
- String defaultConfigValue = CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM;
- String maxParallelSegmentPreprocessesStr =
- clusterConfigs == null ? defaultConfigValue : clusterConfigs.getOrDefault(configName, defaultConfigValue);
-
- int maxPreprocessConcurrency;
- try {
- maxPreprocessConcurrency = Integer.parseInt(maxParallelSegmentPreprocessesStr);
- } catch (Exception e) {
- LOGGER.warn("Invalid maxPreprocessConcurrency set: {}, not making change, fix config and try again",
- maxParallelSegmentPreprocessesStr);
- return;
- }
-
- if (maxPreprocessConcurrency <= 0) {
- LOGGER.warn("maxPreprocessConcurrency: {} must be > 0, not making change, fix config and try again",
- maxPreprocessConcurrency);
- return;
- }
-
- if (maxPreprocessConcurrency == _maxPreprocessConcurrency) {
- LOGGER.info("No ZK update for maxPreprocessConcurrency {}, total permits: {}", _maxPreprocessConcurrency,
- totalPermits());
- return;
- }
-
- LOGGER.info("Updated maxPreprocessConcurrency from: {} to: {}", _maxPreprocessConcurrency,
- maxPreprocessConcurrency);
- _maxPreprocessConcurrency = maxPreprocessConcurrency;
-
- if (!_isServingQueries) {
- LOGGER.info("Serving queries hasn't been enabled yet, not updating the permits with maxPreprocessConcurrency");
- return;
- }
- _semaphore.setPermits(_maxPreprocessConcurrency);
- LOGGER.info("Updated total permits: {}", totalPermits());
- }
-
- private void handleMaxPreprocessConcurrencyBeforeServingQueriesChange(Set<String> changedConfigs,
- Map<String, String> clusterConfigs) {
- if (!changedConfigs.contains(
- CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES)) {
- LOGGER.info("changedConfigs list indicates maxPreprocessConcurrencyBeforeServingQueries was not updated, "
- + "skipping updates");
- return;
- }
-
- String configName = CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES;
- String defaultConfigValue = CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES;
- String maxParallelSegmentPreprocessesBeforeServingQueriesStr =
- clusterConfigs == null ? defaultConfigValue : clusterConfigs.getOrDefault(configName, defaultConfigValue);
-
- int maxPreprocessConcurrencyBeforeServingQueries;
- try {
- maxPreprocessConcurrencyBeforeServingQueries =
- Integer.parseInt(maxParallelSegmentPreprocessesBeforeServingQueriesStr);
- } catch (Exception e) {
- LOGGER.warn("Invalid maxPreprocessConcurrencyBeforeServingQueries set: {}, not making change, fix config and "
- + "try again", maxParallelSegmentPreprocessesBeforeServingQueriesStr);
- return;
- }
-
- if (maxPreprocessConcurrencyBeforeServingQueries <= 0) {
- LOGGER.warn("maxPreprocessConcurrencyBeforeServingQueries: {} must be > 0, not making change, fix config "
- + "and try again", maxPreprocessConcurrencyBeforeServingQueries);
- return;
- }
-
- if (maxPreprocessConcurrencyBeforeServingQueries == _maxPreprocessConcurrencyBeforeServingQueries) {
- LOGGER.info("No ZK update for maxPreprocessConcurrencyBeforeServingQueries {}, total permits: {}",
- _maxPreprocessConcurrencyBeforeServingQueries, totalPermits());
- return;
- }
-
- LOGGER.info("Updated maxPreprocessConcurrencyBeforeServingQueries from: {} to: {}",
- _maxPreprocessConcurrencyBeforeServingQueries, maxPreprocessConcurrencyBeforeServingQueries);
- _maxPreprocessConcurrencyBeforeServingQueries = maxPreprocessConcurrencyBeforeServingQueries;
- if (!_isServingQueries) {
- LOGGER.info("maxPreprocessConcurrencyBeforeServingQueries was updated before serving queries was enabled, "
- + "updating the permits");
- _semaphore.setPermits(_maxPreprocessConcurrencyBeforeServingQueries);
- LOGGER.info("Updated total permits: {}", totalPermits());
- }
+ public SegmentPreprocessThrottler(SegmentAllIndexPreprocessThrottler segmentAllIndexPreprocessThrottler,
+ SegmentStarTreePreprocessThrottler segmentStarTreePreprocessThrottler) {
+ _segmentAllIndexPreprocessThrottler = segmentAllIndexPreprocessThrottler;
+ _segmentStarTreePreprocessThrottler = segmentStarTreePreprocessThrottler;
}
- /**
- * Block trying to acquire the semaphore to perform the segment index rebuild steps unless interrupted.
- * <p>
- * {@link #release()} should be called after the segment preprocess completes. It is the responsibility of the caller
- * to ensure that {@link #release()} is called exactly once for each call to this method.
- *
- * @throws InterruptedException if the current thread is interrupted
- */
- public void acquire()
- throws InterruptedException {
- _semaphore.acquire();
+ public SegmentAllIndexPreprocessThrottler getSegmentAllIndexPreprocessThrottler() {
+ return _segmentAllIndexPreprocessThrottler;
}
- /**
- * Should be called after the segment index build completes. It is the responsibility of the caller to ensure that
- * this method is called exactly once for each call to {@link #acquire()}.
- */
- public void release() {
- _semaphore.release();
+ public SegmentStarTreePreprocessThrottler getSegmentStarTreePreprocessThrottler() {
+ return _segmentStarTreePreprocessThrottler;
}
- @VisibleForTesting
- int availablePermits() {
- return _semaphore.availablePermits();
+ public void startServingQueries() {
+ _segmentAllIndexPreprocessThrottler.startServingQueries();
+ _segmentStarTreePreprocessThrottler.startServingQueries();
}
- @VisibleForTesting
- int totalPermits() {
- return _semaphore.getTotalPermits();
+ @Override
+ public synchronized void onChange(Set<String> changedConfigs, Map<String, String> clusterConfigs) {
+ _segmentAllIndexPreprocessThrottler.onChange(changedConfigs, clusterConfigs);
+ _segmentStarTreePreprocessThrottler.onChange(changedConfigs, clusterConfigs);
}
}
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentStarTreePreprocessThrottler.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentStarTreePreprocessThrottler.java
new file mode 100644
index 0000000000..4ccaaf4a9c
--- /dev/null
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SegmentStarTreePreprocessThrottler.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.segment.local.utils;
+
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.pinot.spi.utils.CommonConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Used to throttle the total concurrent startree index rebuilds on a given Pinot server.
+ */
+public class SegmentStarTreePreprocessThrottler extends BaseSegmentPreprocessThrottler {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SegmentStarTreePreprocessThrottler.class);
+
+ /**
+ * @param maxStarTreePreprocessConcurrency configured StarTree index preprocessing concurrency
+ * @param maxStarTreePreprocessConcurrencyBeforeServingQueries configured preprocessing concurrency before serving
+ * queries
+ * @param isServingQueries whether the server is ready to serve queries or not
+ */
+ public SegmentStarTreePreprocessThrottler(int maxStarTreePreprocessConcurrency,
+ int maxStarTreePreprocessConcurrencyBeforeServingQueries, boolean isServingQueries) {
+ super(maxStarTreePreprocessConcurrency, maxStarTreePreprocessConcurrencyBeforeServingQueries, isServingQueries,
+ LOGGER);
+ }
+
+ @Override
+ public synchronized void onChange(Set<String> changedConfigs, Map<String, String> clusterConfigs) {
+ if (CollectionUtils.isEmpty(changedConfigs)) {
+ LOGGER.info("Skip updating SegmentStarTreePreprocessThrottler configs with unchanged clusterConfigs");
+ return;
+ }
+
+ LOGGER.info("Updating SegmentStarTreePreprocessThrottler configs with latest clusterConfigs");
+ handleMaxPreprocessConcurrencyChange(changedConfigs, clusterConfigs,
+ CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM);
+ handleMaxPreprocessConcurrencyBeforeServingQueriesChange(changedConfigs, clusterConfigs,
+ CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ LOGGER.info("Updated SegmentStarTreePreprocessThrottler configs with latest clusterConfigs");
+ }
+}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
index 9b39d9fc89..ea59295a02 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/LoaderTest.java
@@ -30,7 +30,9 @@ import org.apache.pinot.segment.local.segment.creator.SegmentTestUtils;
import org.apache.pinot.segment.local.segment.creator.impl.SegmentIndexCreationDriverImpl;
import org.apache.pinot.segment.local.segment.index.converter.SegmentV1V2ToV3FormatConverter;
import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.ImmutableSegment;
import org.apache.pinot.segment.spi.IndexSegment;
import org.apache.pinot.segment.spi.V1Constants;
@@ -72,8 +74,9 @@ public class LoaderTest {
private static final String VECTOR_INDEX_COL_NAME = "vector1";
private static final int VECTOR_DIM_SIZE = 512;
- private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER = new SegmentPreprocessThrottler(1, 2,
- true);
+ private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER =
+ new SegmentPreprocessThrottler(new SegmentAllIndexPreprocessThrottler(1, 2, true),
+ new SegmentStarTreePreprocessThrottler(1, 2, true));
private File _avroFile;
private File _vectorAvroFile;
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
index 012c2d920d..0a8a28ddb2 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/loader/SegmentPreProcessorTest.java
@@ -43,6 +43,9 @@ import org.apache.pinot.segment.local.segment.index.forward.ForwardIndexType;
import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode;
import org.apache.pinot.segment.local.segment.readers.GenericRowRecordReader;
import org.apache.pinot.segment.local.segment.store.SegmentLocalFSDirectory;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.ColumnMetadata;
import org.apache.pinot.segment.spi.V1Constants;
import org.apache.pinot.segment.spi.compression.ChunkCompressionType;
@@ -141,6 +144,10 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
private static final String NEW_HLL_BYTE_METRIC_COLUMN_NAME = "newHLLByteMetric";
private static final String NEW_TDIGEST_BYTE_METRIC_COLUMN_NAME = "newTDigestByteMetric";
+ private static final SegmentPreprocessThrottler SEGMENT_PREPROCESS_THROTTLER =
+ new SegmentPreprocessThrottler(new SegmentAllIndexPreprocessThrottler(2, 4, true),
+ new SegmentStarTreePreprocessThrottler(1, 2, true));
+
private final File _avroFile;
private final Schema _schema;
private final Schema _newColumnsSchema1;
@@ -307,7 +314,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
try (SegmentDirectory segmentDirectory = new SegmentLocalFSDirectory(INDEX_DIR, ReadMode.mmap);
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory, createIndexLoadingConfig(schema),
schema)) {
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
}
@@ -1492,7 +1499,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
createIndexLoadingConfig(_newColumnsSchemaWithH3Json), _newColumnsSchemaWithH3Json)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
verifyProcessNotNeeded();
}
@@ -1603,7 +1610,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
new IndexLoadingConfig(tableConfig, schema), schema)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
// Update table config to convert noDict to dict for longCol
@@ -1612,7 +1619,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
new IndexLoadingConfig(tableConfig, schema), schema)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
// Update table config to convert dict to noDict for longCol and add the Startree index config
@@ -1625,7 +1632,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
new IndexLoadingConfig(tableConfig, schema), schema)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
// Remove Startree index but keep the no dictionary for longCol
@@ -1634,7 +1641,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
new IndexLoadingConfig(tableConfig, schema), schema)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
// Update table config to convert noDict to dict for longCol and also add the Startree index
@@ -1644,7 +1651,7 @@ public class SegmentPreProcessorTest implements PinotBuffersAfterClassCheckRule
SegmentPreProcessor processor = new SegmentPreProcessor(segmentDirectory,
new IndexLoadingConfig(tableConfig, schema), schema)) {
assertTrue(processor.needProcess());
- processor.process();
+ processor.process(SEGMENT_PREPROCESS_THROTTLER);
}
}
diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottlerTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottlerTest.java
index 486ce4a89f..2b5c7f5a70 100644
--- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottlerTest.java
+++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/SegmentPreprocessThrottlerTest.java
@@ -18,431 +18,628 @@
*/
package org.apache.pinot.segment.local.utils;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.helix.HelixAdmin;
-import org.apache.helix.HelixManager;
import org.apache.pinot.spi.utils.CommonConstants;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
public class SegmentPreprocessThrottlerTest {
- private AutoCloseable _mocks;
- @Mock
- private HelixManager _helixManager;
- @Mock
- private HelixAdmin _helixAdmin;
- private SegmentPreprocessThrottler _segmentPreprocessThrottler;
-
- @BeforeMethod
- public void setUp() {
- _mocks = MockitoAnnotations.openMocks(this);
- when(_helixManager.getClusterManagmentTool()).thenReturn(_helixAdmin);
- when(_helixManager.getClusterName()).thenReturn("testCluster");
- }
-
- @AfterMethod
- public void tearDown()
- throws Exception {
- _mocks.close();
- }
@Test
public void testBasicAcquireRelease()
throws Exception {
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(4, 8, true);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 4);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), 4);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(4, 8, true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(4, 8, true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 4);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), 4);
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 3);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), 4);
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 3);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), 4);
- _segmentPreprocessThrottler.release();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 4);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), 4);
+ preprocessThrottler.release();
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 4);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), 4);
+ }
}
@Test
public void testBasicAcquireAllPermits()
throws Exception {
int totalPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(totalPermits, totalPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), totalPermits);
-
- for (int i = 0; i < totalPermits; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), totalPermits - i - 1);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), totalPermits);
- }
- for (int i = 0; i < totalPermits; i++) {
- _segmentPreprocessThrottler.release();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), i + 1);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), totalPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(totalPermits, totalPermits * 2, true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(totalPermits, totalPermits * 2, true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), totalPermits);
+
+ for (int i = 0; i < totalPermits; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.availablePermits(), totalPermits - i - 1);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), totalPermits);
+ }
+ for (int i = 0; i < totalPermits; i++) {
+ preprocessThrottler.release();
+ Assert.assertEquals(preprocessThrottler.availablePermits(), i + 1);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), totalPermits);
+ }
}
}
@Test
- public void testThrowExceptionOnSettingInvalidConfigValues()
- throws Exception {
- Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentPreprocessThrottler(-1, 4, true));
- Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentPreprocessThrottler(0, 4, true));
- Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentPreprocessThrottler(1, -4, true));
- Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentPreprocessThrottler(1, 0, true));
+ public void testThrowExceptionOnSettingInvalidConfigValues() {
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(-1, 4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(0, 4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(1, -4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(1, 0, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(-1, 4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(0, 4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(1, -4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentAllIndexPreprocessThrottler(1, 0, false));
+
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(-1, 4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(0, 4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(1, -4, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(1, 0, true));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(-1, 4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(0, 4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(1, -4, false));
+ Assert.assertThrows(IllegalArgumentException.class, () -> new SegmentStarTreePreprocessThrottler(1, 0, false));
}
@Test
public void testDisabledThrottlingBySettingDefault()
throws Exception {
// Default should be quite high. Should be able to essentially acquire as many permits as wanted
- int defaultPermits = Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM);
- int defaultPermitsBeforeQuery =
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(defaultPermits, defaultPermitsBeforeQuery, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermits);
-
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermits);
- for (int i = 0; i < 100; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermits - i - 1);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM), Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM), Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ int defaultPermits = preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM)
+ : Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermits);
+ for (int i = 0; i < 100; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermits - i - 1);
+ }
}
}
@Test
- public void testPositiveToNegativeThrottleChange()
- throws Exception {
+ public void testPositiveToNegativeThrottleChange() {
int initialPermits = 2;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
-
- // Change the value of cluster config for max segment preprocess parallelism to be a negative value
- // If maxConcurrentQueries is <= 0, this is an invalid configuration change. Do nothing other than log a warning
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, "-1");
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
-
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
+
+ // Change the value of cluster config for max segment preprocess parallelism to be a negative value
+ // If config is <= 0, this is an invalid configuration change. Do nothing other than log a warning
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, "-1");
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
+ }
}
@Test
public void testIncreaseSegmentPreprocessParallelism()
throws Exception {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
-
- for (int i = 0; i < initialPermits; i++) {
- _segmentPreprocessThrottler.acquire();
- }
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 0);
-
- // Increase the value of cluster config for max segment preprocess parallelism
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
- String.valueOf(initialPermits * 2));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
-
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
- for (int i = 0; i < initialPermits; i++) {
- _segmentPreprocessThrottler.acquire();
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ for (int i = 0; i < initialPermits; i++) {
+ preprocessThrottler.acquire();
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 0);
+
+ // Increase the value of cluster config for max segment preprocess parallelism
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits * 2));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
+ for (int i = 0; i < initialPermits; i++) {
+ preprocessThrottler.acquire();
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 0);
+ for (int i = 0; i < (initialPermits * 2); i++) {
+ preprocessThrottler.release();
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits * 2);
}
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 0);
- for (int i = 0; i < (initialPermits * 2); i++) {
- _segmentPreprocessThrottler.release();
- }
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits * 2);
}
@Test
public void testDecreaseSegmentPreprocessParallelism()
throws Exception {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
-
- for (int i = 0; i < initialPermits; i++) {
- _segmentPreprocessThrottler.acquire();
- }
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), 0);
-
- // Increase the value of cluster config for max segment preprocess parallelism
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
- String.valueOf(initialPermits / 2));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits / 2);
-
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), -(initialPermits / 2));
- for (int i = 0; i < 4; i++) {
- _segmentPreprocessThrottler.release();
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ for (int i = 0; i < initialPermits; i++) {
+ preprocessThrottler.acquire();
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), 0);
+
+ // Increase the value of cluster config for max segment preprocess parallelism
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits / 2));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits / 2);
+
+ Assert.assertEquals(preprocessThrottler.availablePermits(), -(initialPermits / 2));
+ for (int i = 0; i < 4; i++) {
+ preprocessThrottler.release();
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits / 2);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits / 2);
}
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits / 2);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits / 2);
}
@Test
public void testServingQueriesDisabled() {
int initialPermits = 4;
- int defaultPermitsBeforeQuery =
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, defaultPermitsBeforeQuery, false);
- // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
- // preprocessing more segments is acceptable and cannot affect the query performance
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
-
- // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
- _segmentPreprocessThrottler.startServingQueries();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ int defaultPermitsBeforeQuery = preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES)
+ : Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
+ // preprocessing more segments is acceptable and cannot affect the query performance
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
+
+ // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
+ preprocessThrottler.startServingQueries();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
+ }
}
@Test
public void testServingQueriesDisabledWithAcquireRelease()
throws InterruptedException {
int initialPermits = 4;
- int defaultPermitsBeforeQuery =
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, defaultPermitsBeforeQuery, false);
- // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
- // preprocessing more segments is acceptable and cannot affect the query performance
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
- }
-
- // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
- _segmentPreprocessThrottler.startServingQueries();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits - defaultPermitsBeforeQuery);
-
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.release();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(),
- (initialPermits - defaultPermitsBeforeQuery) + i + 1);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ int defaultPermitsBeforeQuery = preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES)
+ : Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
+ // preprocessing more segments is acceptable and cannot affect the query performance
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ }
+
+ // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
+ preprocessThrottler.startServingQueries();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ initialPermits - defaultPermitsBeforeQuery);
+
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.release();
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ (initialPermits - defaultPermitsBeforeQuery) + i + 1);
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
}
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
}
@Test
public void testServingQueriesDisabledWithAcquireReleaseWithConfigIncrease()
throws InterruptedException {
int initialPermits = 4;
- int defaultPermitsBeforeQuery =
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, defaultPermitsBeforeQuery, false);
- // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
- // preprocessing more segments is acceptable and cannot affect the query performance
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
- }
-
- // Double the permits for before serving queries config
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
- String.valueOf(defaultPermitsBeforeQuery * 2));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery * 2);
- // We doubled permits but took all of the previous ones
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
-
- // Take remaining permits
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery * 2);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ int defaultPermitsBeforeQuery = preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES)
+ : Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
+ // preprocessing more segments is acceptable and cannot affect the query performance
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ }
+
+ // Double the permits for before serving queries config
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ String.valueOf(defaultPermitsBeforeQuery * 2));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery * 2);
+ // We doubled permits but took all of the previous ones
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
+
+ // Take remaining permits
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery * 2);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ }
+
+ // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
+ preprocessThrottler.startServingQueries();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ initialPermits - (defaultPermitsBeforeQuery * 2));
+
+ for (int i = 0; i < defaultPermitsBeforeQuery * 2; i++) {
+ preprocessThrottler.release();
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ (initialPermits - defaultPermitsBeforeQuery * 2) + i + 1);
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
}
-
- // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
- _segmentPreprocessThrottler.startServingQueries();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(),
- initialPermits - (defaultPermitsBeforeQuery * 2));
-
- for (int i = 0; i < defaultPermitsBeforeQuery * 2; i++) {
- _segmentPreprocessThrottler.release();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(),
- (initialPermits - defaultPermitsBeforeQuery * 2) + i + 1);
- }
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
}
@Test
public void testServingQueriesDisabledWithAcquireReleaseWithConfigDecrease()
throws InterruptedException {
int initialPermits = 4;
- int defaultPermitsBeforeQuery =
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, defaultPermitsBeforeQuery, false);
- // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
- // preprocessing more segments is acceptable and cannot affect the query performance
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.acquire();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES), false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ int defaultPermitsBeforeQuery = preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES)
+ : Integer.parseInt(
+ CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ // We set isServingQueries to false when the server is not yet ready to server queries. In this scenario ideally
+ // preprocessing more segments is acceptable and cannot affect the query performance
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery);
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.acquire();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), defaultPermitsBeforeQuery - i - 1);
+ }
+
+ // Half the permits for before serving queries config
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ String.valueOf(defaultPermitsBeforeQuery / 2));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), defaultPermitsBeforeQuery / 2);
+ // We doubled permits but took all of the previous ones
+ Assert.assertEquals(preprocessThrottler.availablePermits(), -(defaultPermitsBeforeQuery / 2));
+
+ // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
+ preprocessThrottler.startServingQueries();
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ initialPermits - defaultPermitsBeforeQuery);
+
+ for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
+ preprocessThrottler.release();
+ Assert.assertEquals(preprocessThrottler.availablePermits(),
+ (initialPermits - defaultPermitsBeforeQuery) + i + 1);
+ }
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ Assert.assertEquals(preprocessThrottler.availablePermits(), initialPermits);
}
-
- // Half the permits for before serving queries config
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
- String.valueOf(defaultPermitsBeforeQuery / 2));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), defaultPermitsBeforeQuery / 2);
- // We doubled permits but took all of the previous ones
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), -(defaultPermitsBeforeQuery / 2));
-
- // Once the server is ready to server queries, we should reset the throttling configurations to be as configured
- _segmentPreprocessThrottler.startServingQueries();
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits - defaultPermitsBeforeQuery);
-
- for (int i = 0; i < defaultPermitsBeforeQuery; i++) {
- _segmentPreprocessThrottler.release();
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(),
- (initialPermits - defaultPermitsBeforeQuery) + i + 1);
- }
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
- Assert.assertEquals(_segmentPreprocessThrottler.availablePermits(), initialPermits);
}
@Test
public void testThrowException()
throws Exception {
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(1, 2, true);
- SegmentPreprocessThrottler spy = spy(_segmentPreprocessThrottler);
- spy.acquire();
- Assert.assertEquals(spy.availablePermits(), 0);
- doThrow(new InterruptedException("interrupt")).when(spy).acquire();
-
- Assert.assertThrows(InterruptedException.class, spy::acquire);
- Assert.assertEquals(spy.availablePermits(), 0);
- spy.release();
- Assert.assertEquals(spy.availablePermits(), 1);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(1, 2, true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(1, 2, true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ BaseSegmentPreprocessThrottler spy = spy(preprocessThrottler);
+ spy.acquire();
+ Assert.assertEquals(spy.availablePermits(), 0);
+ doThrow(new InterruptedException("interrupt")).when(spy).acquire();
+
+ Assert.assertThrows(InterruptedException.class, spy::acquire);
+ Assert.assertEquals(spy.availablePermits(), 0);
+ spy.release();
+ Assert.assertEquals(spy.availablePermits(), 1);
+ }
}
@Test
- public void testChangeConfigsEmpty()
- throws Exception {
+ public void testChangeConfigsEmpty() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
-
- // Add some random configs and call 'onChange'
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ // Add some random configs and call 'onChange'
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ }
}
@Test
- public void testChangeConfigDeletedConfigsEmpty()
- throws Exception {
+ public void testChangeConfigDeletedConfigsEmpty() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
-
- // Create a set of valid keys and pass clusterConfigs as null, the config should reset to the default
- Set<String> keys = new HashSet<>();
- keys.add(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM);
- _segmentPreprocessThrottler.onChange(keys, null);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(),
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM));
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ // Create a set of valid keys and pass clusterConfigs as null, the config should reset to the default
+ Set<String> keys = new HashSet<>();
+ keys.add(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM);
+ preprocessThrottler.onChange(keys, null);
+ Assert.assertEquals(preprocessThrottler.totalPermits(),
+ Integer.parseInt(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM));
+ }
}
@Test
- public void testChangeConfigDeletedConfigsEmptyQueriesDisabled()
- throws Exception {
+ public void testChangeConfigDeletedConfigsEmptyQueriesDisabled() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, false);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
-
- // Create a set of valid keys and pass clusterConfigs as null, the config should reset to the default
- Set<String> keys = new HashSet<>();
- keys.add(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM);
- keys.add(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
- _segmentPreprocessThrottler.onChange(keys, null);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(),
- Integer.parseInt(CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES));
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+
+ // Create a set of valid keys and pass clusterConfigs as null, the config should reset to the default
+ Set<String> keys = new HashSet<>();
+ keys.add(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM);
+ keys.add(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES);
+ preprocessThrottler.onChange(keys, null);
+ Assert.assertEquals(preprocessThrottler.totalPermits(),
+ Integer.parseInt(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES));
+ }
}
@Test
- public void testChangeConfigsOtherThanRelevant()
- throws Exception {
+ public void testChangeConfigsOtherThanRelevant() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ // Add some random configs and call 'onChange'
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put("random.config.key", "random.config.value");
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM, "42");
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+ }
+ }
- // Add some random configs and call 'onChange'
- Map<String, String> updatedClusterConfigs = new HashMap<>();
- updatedClusterConfigs.put("random.config.key", "random.config.value");
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
+ @Test
+ public void testChangeConfigs() {
+ int initialPermits = 4;
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ true));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits);
+
+ // Add random and relevant configs and call 'onChange'
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put("random.config.key", "random.config.value");
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits * 2));
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ String.valueOf(initialPermits * 4));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ // Since isServingQueries = false, new total should match CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+ }
}
@Test
- public void testChangeConfigs()
- throws Exception {
+ public void testChangeConfigsWithServingQueriesDisabled() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, true);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits);
+ List<BaseSegmentPreprocessThrottler> segmentPreprocessThrottlerList = new ArrayList<>();
+ segmentPreprocessThrottlerList.add(new SegmentAllIndexPreprocessThrottler(initialPermits, initialPermits * 2,
+ false));
+ segmentPreprocessThrottlerList.add(new SegmentStarTreePreprocessThrottler(initialPermits, initialPermits * 2,
+ false));
+
+ for (BaseSegmentPreprocessThrottler preprocessThrottler : segmentPreprocessThrottlerList) {
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 2);
+
+ // Add random and relevant configs and call 'onChange'
+ Map<String, String> updatedClusterConfigs = new HashMap<>();
+ updatedClusterConfigs.put("random.config.key", "random.config.value");
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits * 2));
+ updatedClusterConfigs.put(preprocessThrottler instanceof SegmentAllIndexPreprocessThrottler
+ ? CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES
+ : CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ String.valueOf(initialPermits * 4));
+ preprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ // Since isServingQueries = false, new total should match higher threshold of before serving queries
+ Assert.assertEquals(preprocessThrottler.totalPermits(), initialPermits * 4);
+ }
+ }
- // Add random configs and call 'onChange'
+ @Test
+ public void testChangeConfigsOnSegmentPreprocessThrottler() {
+ int initialPermits = 4;
+ SegmentAllIndexPreprocessThrottler allIndexPreprocessThrottler = new SegmentAllIndexPreprocessThrottler(
+ initialPermits, initialPermits * 2, true);
+ SegmentStarTreePreprocessThrottler starTreePreprocessThrottler = new SegmentStarTreePreprocessThrottler(
+ initialPermits, initialPermits * 2, true);
+ SegmentPreprocessThrottler segmentPreprocessThrottler = new SegmentPreprocessThrottler(allIndexPreprocessThrottler,
+ starTreePreprocessThrottler);
+
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().totalPermits(),
+ initialPermits);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().totalPermits(),
+ initialPermits);
+
+ // Add random and relevant configs and call 'onChange'
Map<String, String> updatedClusterConfigs = new HashMap<>();
updatedClusterConfigs.put("random.config.key", "random.config.value");
updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
String.valueOf(initialPermits * 2));
- updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
- String.valueOf(initialPermits * 4));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- // Since isServingQueries = false, new total should match CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
+ updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits * 2));
+ segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().totalPermits(),
+ initialPermits * 2);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().totalPermits(),
+ initialPermits * 2);
}
@Test
- public void testChangeConfigsWithServingQueriesDisabled()
- throws Exception {
+ public void testChangeConfigsOnSegmentPreprocessThrottlerQueriesDisabled() {
int initialPermits = 4;
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(initialPermits, initialPermits * 2, false);
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 2);
-
- // Add random configs and call 'onChange'
+ SegmentAllIndexPreprocessThrottler allIndexPreprocessThrottler = new SegmentAllIndexPreprocessThrottler(
+ initialPermits, initialPermits * 2, false);
+ SegmentStarTreePreprocessThrottler starTreePreprocessThrottler = new SegmentStarTreePreprocessThrottler(
+ initialPermits, initialPermits * 2, false);
+ SegmentPreprocessThrottler segmentPreprocessThrottler = new SegmentPreprocessThrottler(allIndexPreprocessThrottler,
+ starTreePreprocessThrottler);
+
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().totalPermits(),
+ initialPermits * 2);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().totalPermits(),
+ initialPermits * 2);
+
+ // Add random and relevant configs and call 'onChange'
Map<String, String> updatedClusterConfigs = new HashMap<>();
updatedClusterConfigs.put("random.config.key", "random.config.value");
updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
String.valueOf(initialPermits * 2));
+ updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ String.valueOf(initialPermits * 2));
updatedClusterConfigs.put(CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
String.valueOf(initialPermits * 4));
- _segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
- // Since isServingQueries = false, new total should match higher threshold of before serving queries
- Assert.assertEquals(_segmentPreprocessThrottler.totalPermits(), initialPermits * 4);
+ updatedClusterConfigs.put(
+ CommonConstants.Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ String.valueOf(initialPermits * 4));
+ segmentPreprocessThrottler.onChange(updatedClusterConfigs.keySet(), updatedClusterConfigs);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentAllIndexPreprocessThrottler().totalPermits(),
+ initialPermits * 4);
+ Assert.assertEquals(segmentPreprocessThrottler.getSegmentStarTreePreprocessThrottler().totalPermits(),
+ initialPermits * 4);
}
}
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
index e75c38fe0b..c6ba62f8e7 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/BaseServerStarter.java
@@ -80,7 +80,9 @@ import org.apache.pinot.core.transport.ListenerConfig;
import org.apache.pinot.core.util.ListenerConfigUtil;
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneIndexRefreshManager;
import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndexSearcherPool;
+import org.apache.pinot.segment.local.utils.SegmentAllIndexPreprocessThrottler;
import org.apache.pinot.segment.local.utils.SegmentPreprocessThrottler;
+import org.apache.pinot.segment.local.utils.SegmentStarTreePreprocessThrottler;
import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.segment.spi.memory.unsafe.MmapMemoryConfig;
import org.apache.pinot.server.access.AccessControlFactory;
@@ -626,6 +628,7 @@ public abstract class BaseServerStarter implements ServiceStartable {
ControllerLeaderLocator.create(_helixManager);
ServerSegmentCompletionProtocolHandler.init(
_serverConf.subset(SegmentCompletionProtocol.PREFIX_OF_CONFIG_OF_SEGMENT_UPLOADER));
+
int maxPreprocessConcurrency = Integer.parseInt(
_serverConf.getProperty(Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM,
Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM));
@@ -633,8 +636,22 @@ public abstract class BaseServerStarter implements ServiceStartable {
_serverConf.getProperty(Helix.CONFIG_OF_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
Helix.DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES));
// Relax throttling until the server is ready to serve queries
- _segmentPreprocessThrottler = new SegmentPreprocessThrottler(maxPreprocessConcurrency,
- maxPreprocessConcurrencyBeforeServingQueries, false);
+ SegmentAllIndexPreprocessThrottler segmentAllIndexPreprocessThrottler =
+ new SegmentAllIndexPreprocessThrottler(maxPreprocessConcurrency, maxPreprocessConcurrencyBeforeServingQueries,
+ false);
+ int maxStarTreePreprocessConcurrency = Integer.parseInt(
+ _serverConf.getProperty(Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM,
+ Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM));
+ int maxStarTreePreprocessConcurrencyBeforeServingQueries = Integer.parseInt(
+ _serverConf.getProperty(Helix.CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES,
+ Helix.DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES));
+ // Relax throttling until the server is ready to serve queries
+ SegmentStarTreePreprocessThrottler segmentStarTreePreprocessThrottler =
+ new SegmentStarTreePreprocessThrottler(maxStarTreePreprocessConcurrency,
+ maxStarTreePreprocessConcurrencyBeforeServingQueries, false);
+ _segmentPreprocessThrottler =
+ new SegmentPreprocessThrottler(segmentAllIndexPreprocessThrottler, segmentStarTreePreprocessThrottler);
+
ServerConf serverConf = new ServerConf(_serverConf);
_serverInstance = new ServerInstance(serverConf, _helixManager, _accessControlFactory, _segmentPreprocessThrottler);
ServerMetrics serverMetrics = _serverInstance.getServerMetrics();
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
index b666d990f0..ad190bcbde 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
@@ -96,7 +96,7 @@ public class HelixInstanceDataManagerConfig implements InstanceDataManagerConfig
// NOTE: While segment load can be faster, multiple threads will be taken up loading segments, so
// it is possible that the query latencies increase during that period.
//
- private static final String MAX_PARALLEL_REFRESH_THREADS = "max.parallel.refresh.threads";
+ public static final String MAX_PARALLEL_REFRESH_THREADS = "max.parallel.refresh.threads";
// To preload segments of table using upsert in parallel for fast upsert metadata recovery.
private static final String MAX_SEGMENT_PRELOAD_THREADS = "max.segment.preload.threads";
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index f1695e1119..4032042ca7 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -255,6 +255,14 @@ public class CommonConstants {
"pinot.server.max.segment.preprocess.parallelism.before.serving.queries";
// Use the below default before enabling queries on the server
public static final String DEFAULT_MAX_SEGMENT_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES = String.valueOf(100);
+ // Preprocess throttle config specifically for StarTree index rebuild
+ public static final String CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM =
+ "pinot.server.max.segment.startree.preprocess.parallelism";
+ public static final String DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM = String.valueOf(100);
+ public static final String CONFIG_OF_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES =
+ "pinot.server.max.segment.startree.preprocess.parallelism.before.serving.queries";
+ public static final String DEFAULT_MAX_SEGMENT_STARTREE_PREPROCESS_PARALLELISM_BEFORE_SERVING_QUERIES =
+ String.valueOf(100);
}
public static class Broker {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org