You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2024/07/08 22:09:06 UTC
(pinot) branch master updated: Make TableSizeReader and StorageQuotaChecker available as service (#13560)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0b253819a0 Make TableSizeReader and StorageQuotaChecker available as service (#13560)
0b253819a0 is described below
commit 0b253819a0fd921acd369af3714b39c9ba8d2405
Author: Shounak kulkarni <sh...@gmail.com>
AuthorDate: Tue Jul 9 03:09:00 2024 +0500
Make TableSizeReader and StorageQuotaChecker available as service (#13560)
---
.../pinot/controller/BaseControllerStarter.java | 13 +++++++-
.../controller/api/resources/DebugResource.java | 12 ++------
.../PinotSegmentUploadDownloadRestletResource.java | 15 ++-------
.../pinot/controller/api/resources/TableSize.java | 22 ++-----------
.../api/upload/SegmentValidationUtils.java | 16 ++--------
.../controller/helix/SegmentStatusChecker.java | 8 ++---
.../controller/validation/StorageQuotaChecker.java | 22 ++++++-------
.../controller/helix/SegmentStatusCheckerTest.java | 25 +++++++--------
.../validation/StorageQuotaCheckerTest.java | 36 ++++++++++++----------
9 files changed, 64 insertions(+), 105 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
index e2c8a84bfd..621ac490aa 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/BaseControllerStarter.java
@@ -105,9 +105,11 @@ import org.apache.pinot.controller.helix.core.statemodel.LeadControllerResourceM
import org.apache.pinot.controller.helix.core.util.HelixSetupUtils;
import org.apache.pinot.controller.helix.starter.HelixConfig;
import org.apache.pinot.controller.tuner.TableConfigTunerRegistry;
+import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.controller.validation.BrokerResourceValidationManager;
import org.apache.pinot.controller.validation.OfflineSegmentIntervalChecker;
import org.apache.pinot.controller.validation.RealtimeSegmentValidationManager;
+import org.apache.pinot.controller.validation.StorageQuotaChecker;
import org.apache.pinot.core.periodictask.PeriodicTask;
import org.apache.pinot.core.periodictask.PeriodicTaskScheduler;
import org.apache.pinot.core.query.executor.sql.SqlQueryExecutor;
@@ -194,6 +196,8 @@ public abstract class BaseControllerStarter implements ServiceStartable {
protected PoolingHttpClientConnectionManager _connectionManager;
protected TenantRebalancer _tenantRebalancer;
protected ExecutorService _tenantRebalanceExecutorService;
+ protected TableSizeReader _tableSizeReader;
+ protected StorageQuotaChecker _storageQuotaChecker;
@Override
public void init(PinotConfiguration pinotConfiguration)
@@ -483,6 +487,11 @@ public abstract class BaseControllerStarter implements ServiceStartable {
SocketConfig.custom()
.setSoTimeout(Timeout.of(_config.getServerAdminRequestTimeoutSeconds() * 1000, TimeUnit.MILLISECONDS))
.build());
+ _tableSizeReader =
+ new TableSizeReader(_executorService, _connectionManager, _controllerMetrics, _helixResourceManager,
+ _leadControllerManager);
+ _storageQuotaChecker = new StorageQuotaChecker(_tableSizeReader, _controllerMetrics, _leadControllerManager,
+ _helixResourceManager);
// Setting up periodic tasks
List<PeriodicTask> controllerPeriodicTasks = setupControllerPeriodicTasks();
@@ -533,6 +542,8 @@ public abstract class BaseControllerStarter implements ServiceStartable {
bind(_sqlQueryExecutor).to(SqlQueryExecutor.class);
bind(_pinotLLCRealtimeSegmentManager).to(PinotLLCRealtimeSegmentManager.class);
bind(_tenantRebalancer).to(TenantRebalancer.class);
+ bind(_tableSizeReader).to(TableSizeReader.class);
+ bind(_storageQuotaChecker).to(StorageQuotaChecker.class);
bind(controllerStartTime).named(ControllerAdminApiApplication.START_TIME);
String loggerRootDir = _config.getProperty(CommonConstants.Controller.CONFIG_OF_LOGGER_ROOT_DIR);
if (loggerRootDir != null) {
@@ -846,7 +857,7 @@ public abstract class BaseControllerStarter implements ServiceStartable {
periodicTasks.add(_brokerResourceValidationManager);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
periodicTasks.add(_segmentStatusChecker);
_rebalanceChecker = new RebalanceChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
_executorService);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
index a9ad7bea80..945ebcce46 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
@@ -59,13 +59,11 @@ import org.apache.helix.PropertyKey;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.pinot.common.exception.InvalidConfigException;
-import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.restlet.resources.SegmentConsumerInfo;
import org.apache.pinot.common.restlet.resources.SegmentErrorInfo;
import org.apache.pinot.common.restlet.resources.SegmentServerDebugInfo;
import org.apache.pinot.common.utils.DatabaseUtils;
import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.api.debug.TableDebugInfo;
import org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
@@ -118,14 +116,11 @@ public class DebugResource {
@Inject
HttpClientConnectionManager _connectionManager;
- @Inject
- ControllerMetrics _controllerMetrics;
-
@Inject
ControllerConf _controllerConf;
@Inject
- LeadControllerManager _leadControllerManager;
+ TableSizeReader _tableSizeReader;
@GET
@Path("tables/{tableName}")
@@ -241,12 +236,9 @@ public class DebugResource {
}
private TableDebugInfo.TableSizeSummary getTableSize(String tableNameWithType) {
- TableSizeReader tableSizeReader =
- new TableSizeReader(_executor, _connectionManager, _controllerMetrics, _pinotHelixResourceManager,
- _leadControllerManager);
TableSizeReader.TableSizeDetails tableSizeDetails;
try {
- tableSizeDetails = tableSizeReader
+ tableSizeDetails = _tableSizeReader
.getTableSizeDetails(tableNameWithType, _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
} catch (Throwable t) {
tableSizeDetails = null;
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
index 94101d1081..b2c9541b93 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
@@ -40,7 +40,6 @@ import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.Executor;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.ws.rs.Consumes;
@@ -65,7 +64,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
-import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMeter;
import org.apache.pinot.common.metrics.ControllerMetrics;
@@ -78,7 +76,6 @@ import org.apache.pinot.common.utils.FileUploadDownloadClient.FileUploadType;
import org.apache.pinot.common.utils.URIUtils;
import org.apache.pinot.common.utils.fetcher.SegmentFetcherFactory;
import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.api.access.AccessControl;
import org.apache.pinot.controller.api.access.AccessControlFactory;
import org.apache.pinot.controller.api.access.AccessType;
@@ -87,6 +84,7 @@ import org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.controller.api.upload.SegmentValidationUtils;
import org.apache.pinot.controller.api.upload.ZKOperator;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
+import org.apache.pinot.controller.validation.StorageQuotaChecker;
import org.apache.pinot.core.auth.Actions;
import org.apache.pinot.core.auth.Authorize;
import org.apache.pinot.core.auth.TargetType;
@@ -137,17 +135,11 @@ public class PinotSegmentUploadDownloadRestletResource {
ControllerMetrics _controllerMetrics;
@Inject
- HttpClientConnectionManager _connectionManager;
-
- @Inject
- Executor _executor;
+ StorageQuotaChecker _storageQuotaChecker;
@Inject
AccessControlFactory _accessControlFactory;
- @Inject
- LeadControllerManager _leadControllerManager;
-
@GET
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@Path("/segments/{tableName}/{segmentName}")
@@ -362,8 +354,7 @@ public class PinotSegmentUploadDownloadRestletResource {
untarredSegmentSizeInBytes = FileUtils.sizeOfDirectory(tempSegmentDir);
}
SegmentValidationUtils.checkStorageQuota(segmentName, untarredSegmentSizeInBytes, tableConfig,
- _pinotHelixResourceManager, _controllerConf, _controllerMetrics, _connectionManager, _executor,
- _leadControllerManager);
+ _controllerConf, _storageQuotaChecker);
// Encrypt segment
String crypterNameInTableConfig = tableConfig.getValidationConfig().getCrypterClassName();
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
index b6de77e96b..bf7000a0c3 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableSize.java
@@ -27,7 +27,6 @@ import io.swagger.annotations.ApiResponses;
import io.swagger.annotations.Authorization;
import io.swagger.annotations.SecurityDefinition;
import io.swagger.annotations.SwaggerDefinition;
-import java.util.concurrent.Executor;
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -37,13 +36,9 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.hc.client5.http.io.HttpClientConnectionManager;
-import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.utils.DatabaseUtils;
import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.api.exception.ControllerApplicationException;
-import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.core.auth.Actions;
import org.apache.pinot.core.auth.Authorize;
@@ -70,17 +65,7 @@ public class TableSize {
@Inject
ControllerConf _controllerConf;
@Inject
- PinotHelixResourceManager _pinotHelixResourceManager;
- @Inject
- Executor _executor;
- @Inject
- HttpClientConnectionManager _connectionManager;
-
- @Inject
- ControllerMetrics _controllerMetrics;
-
- @Inject
- LeadControllerManager _leadControllerManager;
+ TableSizeReader _tableSizeReader;
@GET
@Path("/tables/{tableName}/size")
@@ -96,13 +81,10 @@ public class TableSize {
@ApiParam(value = "Table name without type", required = true, example = "myTable | myTable_OFFLINE")
@PathParam("tableName") String tableName, @Context HttpHeaders headers) {
tableName = DatabaseUtils.translateTableName(tableName, headers);
- TableSizeReader tableSizeReader =
- new TableSizeReader(_executor, _connectionManager, _controllerMetrics, _pinotHelixResourceManager,
- _leadControllerManager);
TableSizeReader.TableSizeDetails tableSizeDetails = null;
try {
tableSizeDetails =
- tableSizeReader.getTableSizeDetails(tableName, _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
+ _tableSizeReader.getTableSizeDetails(tableName, _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
} catch (Throwable t) {
throw new ControllerApplicationException(LOGGER, String.format("Failed to read table size for %s", tableName),
Response.Status.INTERNAL_SERVER_ERROR, t);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java
index f7dda577dc..dff3dd3d11 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/upload/SegmentValidationUtils.java
@@ -18,15 +18,9 @@
*/
package org.apache.pinot.controller.api.upload;
-import java.util.concurrent.Executor;
import javax.ws.rs.core.Response;
-import org.apache.hc.client5.http.io.HttpClientConnectionManager;
-import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.controller.ControllerConf;
-import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.api.exception.ControllerApplicationException;
-import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
-import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.controller.validation.StorageQuotaChecker;
import org.apache.pinot.segment.spi.SegmentMetadata;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -65,19 +59,13 @@ public class SegmentValidationUtils {
}
public static void checkStorageQuota(String segmentName, long segmentSizeInBytes, TableConfig tableConfig,
- PinotHelixResourceManager resourceManager, ControllerConf controllerConf, ControllerMetrics controllerMetrics,
- HttpClientConnectionManager connectionManager, Executor executor, LeadControllerManager leadControllerManager) {
+ ControllerConf controllerConf, StorageQuotaChecker quotaChecker) {
if (!controllerConf.getEnableStorageQuotaCheck()) {
return;
}
- boolean isLeaderForTable = leadControllerManager.isLeaderForTable(tableConfig.getTableName());
- TableSizeReader tableSizeReader =
- new TableSizeReader(executor, connectionManager, controllerMetrics, resourceManager, leadControllerManager);
- StorageQuotaChecker quotaChecker =
- new StorageQuotaChecker(tableConfig, tableSizeReader, controllerMetrics, isLeaderForTable, resourceManager);
StorageQuotaChecker.QuotaCheckerResponse response;
try {
- response = quotaChecker.isSegmentStorageWithinQuota(segmentName, segmentSizeInBytes,
+ response = quotaChecker.isSegmentStorageWithinQuota(tableConfig, segmentName, segmentSizeInBytes,
controllerConf.getServerAdminRequestTimeoutSeconds() * 1000);
} catch (Exception e) {
throw new ControllerApplicationException(LOGGER,
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index 508b00d109..6488ecb022 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -25,9 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
-import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.store.zk.ZkHelixPropertyStore;
@@ -89,15 +87,13 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
*/
public SegmentStatusChecker(PinotHelixResourceManager pinotHelixResourceManager,
LeadControllerManager leadControllerManager, ControllerConf config, ControllerMetrics controllerMetrics,
- ExecutorService executorService) {
+ TableSizeReader tableSizeReader) {
super("SegmentStatusChecker", config.getStatusCheckerFrequencyInSeconds(),
config.getStatusCheckerInitialDelayInSeconds(), pinotHelixResourceManager, leadControllerManager,
controllerMetrics);
_waitForPushTimeSeconds = config.getStatusCheckerWaitForPushTimeInSeconds();
- _tableSizeReader =
- new TableSizeReader(executorService, new PoolingHttpClientConnectionManager(), _controllerMetrics,
- _pinotHelixResourceManager, leadControllerManager);
+ _tableSizeReader = tableSizeReader;
}
@Override
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/StorageQuotaChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/StorageQuotaChecker.java
index 4920cf5e60..ec7d34bd64 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/StorageQuotaChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/StorageQuotaChecker.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMetrics;
+import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.spi.config.table.QuotaConfig;
@@ -39,18 +40,16 @@ public class StorageQuotaChecker {
private static final Logger LOGGER = LoggerFactory.getLogger(StorageQuotaChecker.class);
private final TableSizeReader _tableSizeReader;
- private final TableConfig _tableConfig;
private final ControllerMetrics _controllerMetrics;
- private final boolean _isLeaderForTable;
+ private final LeadControllerManager _leadControllerManager;
private final PinotHelixResourceManager _pinotHelixResourceManager;
- public StorageQuotaChecker(TableConfig tableConfig, TableSizeReader tableSizeReader,
- ControllerMetrics controllerMetrics, boolean isLeaderForTable,
+ public StorageQuotaChecker(TableSizeReader tableSizeReader,
+ ControllerMetrics controllerMetrics, LeadControllerManager leadControllerManager,
PinotHelixResourceManager pinotHelixResourceManager) {
- _tableConfig = tableConfig;
_tableSizeReader = tableSizeReader;
_controllerMetrics = controllerMetrics;
- _isLeaderForTable = isLeaderForTable;
+ _leadControllerManager = leadControllerManager;
_pinotHelixResourceManager = pinotHelixResourceManager;
}
@@ -75,7 +74,8 @@ public class StorageQuotaChecker {
/**
* Returns whether the new added segment is within the storage quota.
*/
- public QuotaCheckerResponse isSegmentStorageWithinQuota(String segmentName, long segmentSizeInBytes, int timeoutMs)
+ public QuotaCheckerResponse isSegmentStorageWithinQuota(TableConfig tableConfig, String segmentName,
+ long segmentSizeInBytes, int timeoutMs)
throws InvalidConfigException {
Preconditions.checkArgument(timeoutMs > 0, "Timeout value must be > 0, input: %s", timeoutMs);
@@ -83,10 +83,10 @@ public class StorageQuotaChecker {
// 2. read table size from all the servers
// 3. update predicted segment sizes
// 4. is the updated size within quota
- QuotaConfig quotaConfig = _tableConfig.getQuotaConfig();
- int numReplicas = _pinotHelixResourceManager.getNumReplicas(_tableConfig);
+ QuotaConfig quotaConfig = tableConfig.getQuotaConfig();
+ int numReplicas = _pinotHelixResourceManager.getNumReplicas(tableConfig);
- final String tableNameWithType = _tableConfig.getTableName();
+ final String tableNameWithType = tableConfig.getTableName();
if (quotaConfig == null || quotaConfig.getStorage() == null) {
// no quota configuration...so ignore for backwards compatibility
@@ -137,7 +137,7 @@ public class StorageQuotaChecker {
tableNameWithType, tableSubtypeSize._estimatedSizeInBytes, tableSubtypeSize._reportedSizeInBytes);
// Only emit the real percentage of storage quota usage by lead controller, otherwise emit 0L.
- if (_isLeaderForTable) {
+ if (_leadControllerManager.isLeaderForTable(tableNameWithType)) {
long existingStorageQuotaUtilization = tableSubtypeSize._estimatedSizeInBytes * 100 / allowedStorageBytes;
_controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.TABLE_STORAGE_QUOTA_UTILIZATION,
existingStorageQuotaUtilization);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
index 3161c9da20..429391e8e3 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/SegmentStatusCheckerTest.java
@@ -20,8 +20,6 @@ package org.apache.pinot.controller.helix;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.apache.helix.AccessOption;
import org.apache.helix.model.ExternalView;
@@ -61,7 +59,6 @@ import static org.testng.Assert.assertFalse;
public class SegmentStatusCheckerTest {
- private final ExecutorService _executorService = Executors.newFixedThreadPool(1);
private SegmentStatusChecker _segmentStatusChecker;
private PinotHelixResourceManager _helixResourceManager;
@@ -145,7 +142,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -237,7 +234,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -336,7 +333,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -395,7 +392,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -440,7 +437,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -534,7 +531,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -591,7 +588,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -643,7 +640,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ null);
// verify state before test
assertEquals(MetricValueUtils.getGlobalGaugeValue(_controllerMetrics, ControllerGauge.DISABLED_TABLE_COUNT), 0);
@@ -684,7 +681,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ null);
// verify state before test
assertFalse(MetricValueUtils.globalGaugeExists(_controllerMetrics, ControllerGauge.DISABLED_TABLE_COUNT));
@@ -756,7 +753,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
@@ -803,7 +800,7 @@ public class SegmentStatusCheckerTest {
_controllerMetrics = new ControllerMetrics(_metricsRegistry);
_segmentStatusChecker =
new SegmentStatusChecker(_helixResourceManager, _leadControllerManager, _config, _controllerMetrics,
- _executorService);
+ _tableSizeReader);
_segmentStatusChecker.setTableSizeReader(_tableSizeReader);
_segmentStatusChecker.start();
_segmentStatusChecker.run();
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/validation/StorageQuotaCheckerTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/validation/StorageQuotaCheckerTest.java
index 888db0e4cc..689fd622a1 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/validation/StorageQuotaCheckerTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/validation/StorageQuotaCheckerTest.java
@@ -23,6 +23,7 @@ import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.metrics.ControllerGauge;
import org.apache.pinot.common.metrics.ControllerMetrics;
import org.apache.pinot.common.metrics.MetricValueUtils;
+import org.apache.pinot.controller.LeadControllerManager;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.spi.config.table.QuotaConfig;
@@ -56,10 +57,10 @@ public class StorageQuotaCheckerTest {
new TableConfigBuilder(TableType.OFFLINE).setTableName(OFFLINE_TABLE_NAME).setNumReplicas(NUM_REPLICAS).build();
_tableSizeReader = mock(TableSizeReader.class);
ControllerMetrics controllerMetrics = new ControllerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
- _storageQuotaChecker = new StorageQuotaChecker(tableConfig, _tableSizeReader, controllerMetrics, true,
- mock(PinotHelixResourceManager.class));
+ _storageQuotaChecker = new StorageQuotaChecker(_tableSizeReader, controllerMetrics,
+ mock(LeadControllerManager.class), mock(PinotHelixResourceManager.class));
tableConfig.setQuotaConfig(null);
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
}
@Test
@@ -70,10 +71,10 @@ public class StorageQuotaCheckerTest {
.setNumReplicas(NUM_REPLICAS).build();
_tableSizeReader = mock(TableSizeReader.class);
ControllerMetrics controllerMetrics = new ControllerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
- _storageQuotaChecker = new StorageQuotaChecker(tableConfig, _tableSizeReader, controllerMetrics, true,
- mock(PinotHelixResourceManager.class));
+ _storageQuotaChecker = new StorageQuotaChecker(_tableSizeReader, controllerMetrics,
+ mock(LeadControllerManager.class), mock(PinotHelixResourceManager.class));
tableConfig.setQuotaConfig(null);
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
}
@Test
@@ -83,10 +84,10 @@ public class StorageQuotaCheckerTest {
new TableConfigBuilder(TableType.OFFLINE).setTableName(OFFLINE_TABLE_NAME).setNumReplicas(NUM_REPLICAS).build();
_tableSizeReader = mock(TableSizeReader.class);
ControllerMetrics controllerMetrics = new ControllerMetrics(PinotMetricUtils.getPinotMetricsRegistry());
- _storageQuotaChecker = new StorageQuotaChecker(tableConfig, _tableSizeReader, controllerMetrics, true,
- mock(PinotHelixResourceManager.class));
+ _storageQuotaChecker = new StorageQuotaChecker(_tableSizeReader, controllerMetrics,
+ mock(LeadControllerManager.class), mock(PinotHelixResourceManager.class));
tableConfig.setQuotaConfig(new QuotaConfig(null, null));
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
}
@Test
@@ -99,19 +100,20 @@ public class StorageQuotaCheckerTest {
PinotHelixResourceManager pinotHelixResourceManager = mock(PinotHelixResourceManager.class);
when(pinotHelixResourceManager.getNumReplicas(eq(tableConfig))).thenReturn(NUM_REPLICAS);
_storageQuotaChecker =
- new StorageQuotaChecker(tableConfig, _tableSizeReader, controllerMetrics, true, pinotHelixResourceManager);
+ new StorageQuotaChecker(_tableSizeReader, controllerMetrics, mock(LeadControllerManager.class),
+ pinotHelixResourceManager);
tableConfig.setQuotaConfig(new QuotaConfig("2.8K", null));
// No response from server, should pass without updating metrics
mockTableSizeResult(-1, 0);
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
assertFalse(
MetricValueUtils.tableGaugeExists(controllerMetrics, OFFLINE_TABLE_NAME,
ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE));
// Within quota but with missing segments, should pass without updating metrics
mockTableSizeResult(4 * 1024, 1);
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
assertFalse(
MetricValueUtils.tableGaugeExists(controllerMetrics, OFFLINE_TABLE_NAME,
ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE));
@@ -119,30 +121,30 @@ public class StorageQuotaCheckerTest {
// Exceed quota and with missing segments, should fail without updating metrics
mockTableSizeResult(8 * 1024, 1);
- assertFalse(isSegmentWithinQuota());
+ assertFalse(isSegmentWithinQuota(tableConfig));
assertFalse(
MetricValueUtils.tableGaugeExists(controllerMetrics, OFFLINE_TABLE_NAME,
ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE));
// Within quota without missing segments, should pass and update metrics
mockTableSizeResult(3 * 1024, 0);
- assertTrue(isSegmentWithinQuota());
+ assertTrue(isSegmentWithinQuota(tableConfig));
assertEquals(
MetricValueUtils.getTableGaugeValue(controllerMetrics, OFFLINE_TABLE_NAME,
ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE), 3 * 1024);
// Exceed quota without missing segments, should fail and update metrics
mockTableSizeResult(4 * 1024, 0);
- assertFalse(isSegmentWithinQuota());
+ assertFalse(isSegmentWithinQuota(tableConfig));
assertEquals(
MetricValueUtils.getTableGaugeValue(controllerMetrics, OFFLINE_TABLE_NAME,
ControllerGauge.OFFLINE_TABLE_ESTIMATED_SIZE), 4 * 1024);
}
- private boolean isSegmentWithinQuota()
+ private boolean isSegmentWithinQuota(TableConfig tableConfig)
throws InvalidConfigException {
return _storageQuotaChecker
- .isSegmentStorageWithinQuota(SEGMENT_NAME, SEGMENT_SIZE_IN_BYTES, 1000)._isSegmentWithinQuota;
+ .isSegmentStorageWithinQuota(tableConfig, SEGMENT_NAME, SEGMENT_SIZE_IN_BYTES, 1000)._isSegmentWithinQuota;
}
public void mockTableSizeResult(long tableSizeInBytes, int numMissingSegments)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org