You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2024/09/04 11:51:27 UTC
(pinot) branch master updated: Allow table names in TableConfigs with or without database name when database context is passed (#13934)
This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4d5017444a Allow table names in TableConfigs with or without database name when database context is passed (#13934)
4d5017444a is described below
commit 4d5017444a582a510d0b3bc7d7076bb85b5a28fa
Author: Shounak kulkarni <sh...@gmail.com>
AuthorDate: Wed Sep 4 17:21:21 2024 +0530
Allow table names in TableConfigs with or without database name when database context is passed (#13934)
---
.../api/resources/TableConfigsRestletResource.java | 31 ++++++++------
.../api/TableConfigsRestletResourceTest.java | 50 ++++++++++++++++++++++
2 files changed, 69 insertions(+), 12 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 385875f42f..a1e2f74fb7 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -190,8 +190,9 @@ public class TableConfigsRestletResource {
Response.Status.BAD_REQUEST, e);
}
TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
- validateConfig(tableConfigs, typesToSkip);
- String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), httpHeaders);
+ String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(httpHeaders);
+ validateConfig(tableConfigs, databaseName, typesToSkip);
+ String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName);
tableConfigs.setTableName(rawTableName);
if (_pinotHelixResourceManager.hasOfflineTable(rawTableName) || _pinotHelixResourceManager.hasRealtimeTable(
@@ -318,15 +319,17 @@ public class TableConfigsRestletResource {
@ApiParam(value = "Force update the table schema") @DefaultValue("false") @QueryParam("forceTableSchemaUpdate")
boolean forceTableSchemaUpdate, String tableConfigsStr, @Context HttpHeaders headers)
throws Exception {
- tableName = DatabaseUtils.translateTableName(tableName, headers);
+ String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(headers);
+ tableName = DatabaseUtils.translateTableName(tableName, databaseName);
Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
TableConfigs tableConfigs;
try {
tableConfigsAndUnrecognizedProps =
JsonUtils.stringToObjectAndUnrecognizedProperties(tableConfigsStr, TableConfigs.class);
tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
- validateConfig(tableConfigs, typesToSkip);
- Preconditions.checkState(DatabaseUtils.translateTableName(tableConfigs.getTableName(), headers).equals(tableName),
+ validateConfig(tableConfigs, databaseName, typesToSkip);
+ Preconditions.checkState(
+ DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName).equals(tableName),
"'tableName' in TableConfigs: %s must match provided tableName: %s", tableConfigs.getTableName(), tableName);
tableConfigs.setTableName(tableName);
} catch (Exception e) {
@@ -405,9 +408,10 @@ public class TableConfigsRestletResource {
throw new ControllerApplicationException(LOGGER,
String.format("Invalid TableConfigs json string: %s", tableConfigsStr), Response.Status.BAD_REQUEST, e);
}
+ String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(httpHeaders);
TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
- validateConfig(tableConfigs, typesToSkip);
- String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), httpHeaders);
+ validateConfig(tableConfigs, databaseName, typesToSkip);
+ String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName);
tableConfigs.setTableName(rawTableName);
// validate permission
@@ -429,8 +433,8 @@ public class TableConfigsRestletResource {
TableConfigUtils.ensureStorageQuotaConstraints(tableConfig, _controllerConf.getDimTableMaxSize());
}
- private void validateConfig(TableConfigs tableConfigs, @Nullable String typesToSkip) {
- String rawTableName = tableConfigs.getTableName();
+ private void validateConfig(TableConfigs tableConfigs, String database, @Nullable String typesToSkip) {
+ String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), database);
TableConfig offlineTableConfig = tableConfigs.getOffline();
TableConfig realtimeTableConfig = tableConfigs.getRealtime();
Schema schema = tableConfigs.getSchema();
@@ -439,20 +443,23 @@ public class TableConfigsRestletResource {
"Must provide at least one of 'realtime' or 'offline' table configs for adding TableConfigs: %s",
rawTableName);
Preconditions.checkState(schema != null, "Must provide 'schema' for adding TableConfigs: %s", rawTableName);
+ String schemaName = DatabaseUtils.translateTableName(schema.getSchemaName(), database);
Preconditions.checkState(!rawTableName.isEmpty(), "'tableName' cannot be empty in TableConfigs");
- Preconditions.checkState(rawTableName.equals(schema.getSchemaName()),
+ Preconditions.checkState(rawTableName.equals(schemaName),
"'tableName': %s must be equal to 'schemaName' from 'schema': %s", rawTableName, schema.getSchemaName());
SchemaUtils.validate(schema);
if (offlineTableConfig != null) {
- String offlineRawTableName = TableNameBuilder.extractRawTableName(offlineTableConfig.getTableName());
+ String offlineRawTableName = DatabaseUtils.translateTableName(
+ TableNameBuilder.extractRawTableName(offlineTableConfig.getTableName()), database);
Preconditions.checkState(offlineRawTableName.equals(rawTableName),
"Name in 'offline' table config: %s must be equal to 'tableName': %s", offlineRawTableName, rawTableName);
TableConfigUtils.validateTableName(offlineTableConfig);
TableConfigUtils.validate(offlineTableConfig, schema, typesToSkip);
}
if (realtimeTableConfig != null) {
- String realtimeRawTableName = TableNameBuilder.extractRawTableName(realtimeTableConfig.getTableName());
+ String realtimeRawTableName = DatabaseUtils.translateTableName(
+ TableNameBuilder.extractRawTableName(realtimeTableConfig.getTableName()), database);
Preconditions.checkState(realtimeRawTableName.equals(rawTableName),
"Name in 'realtime' table config: %s must be equal to 'tableName': %s", realtimeRawTableName, rawTableName);
TableConfigUtils.validateTableName(realtimeTableConfig);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
index 05830f3fa2..54cc7b0ac0 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
@@ -23,7 +23,9 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Sets;
import java.io.IOException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.pinot.controller.helix.ControllerTest;
import org.apache.pinot.core.realtime.impl.fakestream.FakeStreamConfigUtils;
import org.apache.pinot.spi.config.TableConfigs;
@@ -36,8 +38,10 @@ import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.Schema;
import org.apache.pinot.spi.stream.StreamConfig;
+import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.JsonUtils;
import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
@@ -242,6 +246,52 @@ public class TableConfigsRestletResourceTest extends ControllerTest {
// expected
}
+ // table name check fails when database context is not passed in header but one of the configs has database prefix
+ Schema dummySchema = createDummySchema(tableName);
+ TableConfig offlineTableConfig1 = createOfflineTableConfig("db1." + tableName);
+ TableConfig realtimeTableConfig1 = createRealtimeTableConfig(tableName);
+ tableConfigs = new TableConfigs(tableName, dummySchema, offlineTableConfig1, realtimeTableConfig1);
+ try {
+ sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+ fail("Creation of an TableConfigs without database context in header but provided in one of the configs should "
+ + "fail");
+ } catch (Exception e) {
+ // expected
+ }
+ // fails with schema as well
+ offlineTableConfig1.setTableName(TableNameBuilder.OFFLINE.tableNameWithType(tableName));
+ dummySchema.setSchemaName("db1." + tableName);
+ try {
+ sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+ fail("Creation of an TableConfigs without database context in header but provided in one of the configs should "
+ + "fail");
+ } catch (Exception e) {
+ // expected
+ }
+
+ // fails even though both configs and schema have the database prefix in the table names but
+ // database context is not passed in header
+ tableConfigs.setTableName("db1." + tableName);
+ try {
+ sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+ fail("Creation of an TableConfigs without database context in header but provided in all of the configs should "
+ + "fail");
+ } catch (Exception e) {
+ // expected
+ }
+
+ // successfully created with all 3 configs when database context is passed in header and configs may or may not
+ // have the database prefix in the table names
+ Map<String, String> headers = new HashMap<>();
+ tableConfigs.setTableName(tableName);
+ headers.put(CommonConstants.DATABASE, "db1");
+ // only schema has the database prefix
+ dummySchema.setSchemaName("db1." + tableName);
+ sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString(), headers);
+ // one of the table config has database prefix
+ offlineTableConfig1.setTableName(TableNameBuilder.OFFLINE.tableNameWithType(tableName));
+ sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString(), headers);
+
// successfully created with all 3 configs
String tableName1 = "testValidate1";
tableConfigs = new TableConfigs(tableName1, createDummySchema(tableName1), createOfflineTableConfig(tableName1),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org