You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by cb...@apache.org on 2026/03/12 01:20:43 UTC
(pinot) branch master updated: Enhance /tableConfigs/validate endpoint with cluster-aware validations (#16675)
This is an automated email from the ASF dual-hosted git repository.
cbalci pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9c38cc1b732 Enhance /tableConfigs/validate endpoint with cluster-aware validations (#16675)
9c38cc1b732 is described below
commit 9c38cc1b7320fdb0bbd145c7a087ee4847ef49f5
Author: Rekha Seethamraju <re...@gmail.com>
AuthorDate: Wed Mar 11 18:20:36 2026 -0700
Enhance /tableConfigs/validate endpoint with cluster-aware validations (#16675)
---
.../api/resources/TableConfigsRestletResource.java | 39 +++++++++++++--
.../api/TableConfigsRestletResourceTest.java | 55 ++++++++++++++++++++++
.../segment/local/utils/TableConfigUtils.java | 4 +-
3 files changed, 93 insertions(+), 5 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 744d5a1dd22..9acef500a64 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -32,6 +32,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.ws.rs.DELETE;
@@ -196,7 +197,8 @@ public class TableConfigsRestletResource {
@ManualAuthorization // performed after parsing table configs
public ConfigSuccessResponse addConfig(
String tableConfigsStr,
- @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: (ALL|TASK|UPSERT)")
+ @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: "
+ + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip,
@DefaultValue("false") @QueryParam("ignoreActiveTasks") boolean ignoreActiveTasks,
@Context HttpHeaders httpHeaders, @Context Request request)
@@ -363,7 +365,8 @@ public class TableConfigsRestletResource {
public ConfigSuccessResponse updateConfig(
@ApiParam(value = "TableConfigs name i.e. raw table name", required = true) @PathParam("tableName")
String tableName,
- @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: (ALL|TASK|UPSERT)")
+ @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: "
+ + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip,
@ApiParam(value = "Reload the table if the new schema is backward compatible") @DefaultValue("false")
@QueryParam("reload") boolean reload,
@@ -452,7 +455,8 @@ public class TableConfigsRestletResource {
@ApiOperation(value = "Validate the TableConfigs", notes = "Validate the TableConfigs")
@ManualAuthorization // performed after parsing TableConfigs
public String validateConfig(String tableConfigsStr,
- @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: (ALL|TASK|UPSERT)")
+ @ApiParam(value = "comma separated list of validation type(s) to skip. supported types: "
+ + "(ALL|TASK|UPSERT|TENANT|MINION_INSTANCES|ACTIVE_TASKS)")
@QueryParam("validationTypesToSkip") @Nullable String typesToSkip, @Context HttpHeaders httpHeaders,
@Context Request request) {
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
@@ -489,6 +493,11 @@ public class TableConfigsRestletResource {
TableConfigUtils.ensureStorageQuotaConstraints(tableConfig, _controllerConf.getDimTableMaxSize());
}
+
+ /**
+ * Validates the provided TableConfigs. Hybrid table validation is performed only on the provided
+ * configs and does not check for conflicts with existing tables in the cluster.
+ */
private void validateConfig(TableConfigs tableConfigs, String database, @Nullable String typesToSkip) {
String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), database);
TableConfig offlineTableConfig = tableConfigs.getOffline();
@@ -505,6 +514,8 @@ public class TableConfigsRestletResource {
Preconditions.checkState(rawTableName.equals(schemaName),
"'tableName': %s must be equal to 'schemaName' from 'schema': %s", rawTableName, schema.getSchemaName());
SchemaUtils.validate(schema);
+ // Parse validation types to skip using TableConfigUtils method
+ Set<TableConfigUtils.ValidationType> skipTypes = TableConfigUtils.parseTypesToSkipString(typesToSkip);
if (offlineTableConfig != null) {
String offlineRawTableName = DatabaseUtils.translateTableName(
TableNameBuilder.extractRawTableName(offlineTableConfig.getTableName()), database);
@@ -512,6 +523,17 @@ public class TableConfigsRestletResource {
"Name in 'offline' table config: %s must be equal to 'tableName': %s", offlineRawTableName, rawTableName);
TableConfigUtils.validateTableName(offlineTableConfig);
TableConfigUtils.validate(offlineTableConfig, schema, typesToSkip);
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.ALL)) {
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.TENANT)) {
+ _pinotHelixResourceManager.validateTableTenantConfig(offlineTableConfig);
+ }
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.MINION_INSTANCES)) {
+ _pinotHelixResourceManager.validateTableTaskMinionInstanceTagConfig(offlineTableConfig);
+ }
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.ACTIVE_TASKS)) {
+ PinotTableRestletResource.tableTasksValidation(offlineTableConfig, _pinotHelixTaskResourceManager);
+ }
+ }
TaskConfigUtils.validateTaskConfigs(tableConfigs.getOffline(), schema, _pinotTaskManager, typesToSkip);
}
if (realtimeTableConfig != null) {
@@ -521,6 +543,17 @@ public class TableConfigsRestletResource {
"Name in 'realtime' table config: %s must be equal to 'tableName': %s", realtimeRawTableName, rawTableName);
TableConfigUtils.validateTableName(realtimeTableConfig);
TableConfigUtils.validate(realtimeTableConfig, schema, typesToSkip);
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.ALL)) {
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.TENANT)) {
+ _pinotHelixResourceManager.validateTableTenantConfig(realtimeTableConfig);
+ }
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.MINION_INSTANCES)) {
+ _pinotHelixResourceManager.validateTableTaskMinionInstanceTagConfig(realtimeTableConfig);
+ }
+ if (!skipTypes.contains(TableConfigUtils.ValidationType.ACTIVE_TASKS)) {
+ PinotTableRestletResource.tableTasksValidation(realtimeTableConfig, _pinotHelixTaskResourceManager);
+ }
+ }
TaskConfigUtils.validateTaskConfigs(tableConfigs.getRealtime(), schema, _pinotTaskManager, typesToSkip);
}
if (offlineTableConfig != null && realtimeTableConfig != null) {
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
index 7960f120dc3..ea80b910a02 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
@@ -730,6 +730,61 @@ public class TableConfigsRestletResourceTest extends ControllerTest {
sendDeleteRequest(DEFAULT_INSTANCE.getControllerRequestURLBuilder().forSchemaDelete(tableName));
}
+ @Test
+ public void testValidateConfigWithClusterValidationSkipTypes()
+ throws IOException {
+ String validateConfigUrl = DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsValidate();
+ String tableName = "testValidateCluster";
+ TableConfig offlineTableConfig = createOfflineTableConfig(tableName);
+ TableConfig realtimeTableConfig = createRealtimeTableConfig(tableName);
+ Schema schema = createDummySchema(tableName);
+ TableConfigs tableConfigs = new TableConfigs(tableName, schema, offlineTableConfig, realtimeTableConfig);
+
+ // Test validation with TENANT skip type - should pass even without proper tenant setup
+ String responseWithTenantSkip = sendPostRequest(validateConfigUrl + "?validationTypesToSkip=TENANT",
+ tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(responseWithTenantSkip);
+
+ // Test validation with MINION_INSTANCES skip type - should pass even without minion instances
+ String responseWithMinionSkip = sendPostRequest(validateConfigUrl + "?validationTypesToSkip=MINION_INSTANCES",
+ tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(responseWithMinionSkip);
+
+ // Test validation with ACTIVE_TASKS skip type - should pass even with potential task conflicts
+ String responseWithTasksSkip = sendPostRequest(validateConfigUrl + "?validationTypesToSkip=ACTIVE_TASKS",
+ tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(responseWithTasksSkip);
+
+ // Test validation with multiple skip types
+ String responseWithMultipleSkips = sendPostRequest(validateConfigUrl
+ + "?validationTypesToSkip=TENANT,MINION_INSTANCES,ACTIVE_TASKS",
+ tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(responseWithMultipleSkips);
+
+ // Test validation with ALL skip type - should skip all validations
+ String responseWithAllSkip = sendPostRequest(validateConfigUrl + "?validationTypesToSkip=ALL",
+ tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(responseWithAllSkip);
+ }
+
+ @Test
+ public void testValidateConfigClusterValidationsEnabled()
+ throws IOException {
+ String validateConfigUrl = DEFAULT_INSTANCE.getControllerRequestURLBuilder().forTableConfigsValidate();
+ String tableName = "testValidateClusterEnabled";
+ TableConfig offlineTableConfig = createOfflineTableConfig(tableName);
+ Schema schema = createDummySchema(tableName);
+ TableConfigs tableConfigs = new TableConfigs(tableName, schema, offlineTableConfig, null);
+
+ // Test that cluster validations are enabled by default (should pass with default tenant setup)
+ // Note: In test environment, default tenant should exist
+ String response = sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+ Assert.assertNotNull(response);
+
+ // Verify response contains the table config
+ Assert.assertTrue(response.contains(tableName));
+ }
+
@AfterClass
public void tearDown() {
DEFAULT_INSTANCE.cleanup();
diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
index fefbf2efa96..e8ee6c588e3 100644
--- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
+++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java
@@ -239,7 +239,7 @@ public final class TableConfigUtils {
"Instance pool and replica group configurations must be enabled");
}
- private static Set<ValidationType> parseTypesToSkipString(@Nullable String typesToSkip) {
+ public static Set<ValidationType> parseTypesToSkipString(@Nullable String typesToSkip) {
return typesToSkip == null ? Collections.emptySet()
: Arrays.stream(typesToSkip.split(",")).map(s -> ValidationType.valueOf(s.toUpperCase()))
.collect(Collectors.toSet());
@@ -1808,7 +1808,7 @@ public final class TableConfigUtils {
// enum of all the skip-able validation types.
public enum ValidationType {
- ALL, TASK, UPSERT
+ ALL, TASK, UPSERT, TENANT, MINION_INSTANCES, ACTIVE_TASKS
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org