You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by kh...@apache.org on 2024/09/04 11:51:27 UTC

(pinot) branch master updated: Allow table names in TableConfigs with or without database name when database context is passed (#13934)

This is an automated email from the ASF dual-hosted git repository.

kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 4d5017444a Allow table names in TableConfigs with or without database name when database context is passed (#13934)
4d5017444a is described below

commit 4d5017444a582a510d0b3bc7d7076bb85b5a28fa
Author: Shounak kulkarni <sh...@gmail.com>
AuthorDate: Wed Sep 4 17:21:21 2024 +0530

    Allow table names in TableConfigs with or without database name when database context is passed (#13934)
---
 .../api/resources/TableConfigsRestletResource.java | 31 ++++++++------
 .../api/TableConfigsRestletResourceTest.java       | 50 ++++++++++++++++++++++
 2 files changed, 69 insertions(+), 12 deletions(-)

diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
index 385875f42f..a1e2f74fb7 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableConfigsRestletResource.java
@@ -190,8 +190,9 @@ public class TableConfigsRestletResource {
           Response.Status.BAD_REQUEST, e);
     }
     TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
-    validateConfig(tableConfigs, typesToSkip);
-    String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), httpHeaders);
+    String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(httpHeaders);
+    validateConfig(tableConfigs, databaseName, typesToSkip);
+    String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName);
     tableConfigs.setTableName(rawTableName);
 
     if (_pinotHelixResourceManager.hasOfflineTable(rawTableName) || _pinotHelixResourceManager.hasRealtimeTable(
@@ -318,15 +319,17 @@ public class TableConfigsRestletResource {
       @ApiParam(value = "Force update the table schema") @DefaultValue("false") @QueryParam("forceTableSchemaUpdate")
       boolean forceTableSchemaUpdate, String tableConfigsStr, @Context HttpHeaders headers)
       throws Exception {
-    tableName = DatabaseUtils.translateTableName(tableName, headers);
+    String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(headers);
+    tableName = DatabaseUtils.translateTableName(tableName, databaseName);
     Pair<TableConfigs, Map<String, Object>> tableConfigsAndUnrecognizedProps;
     TableConfigs tableConfigs;
     try {
       tableConfigsAndUnrecognizedProps =
           JsonUtils.stringToObjectAndUnrecognizedProperties(tableConfigsStr, TableConfigs.class);
       tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
-      validateConfig(tableConfigs, typesToSkip);
-      Preconditions.checkState(DatabaseUtils.translateTableName(tableConfigs.getTableName(), headers).equals(tableName),
+      validateConfig(tableConfigs, databaseName, typesToSkip);
+      Preconditions.checkState(
+          DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName).equals(tableName),
           "'tableName' in TableConfigs: %s must match provided tableName: %s", tableConfigs.getTableName(), tableName);
       tableConfigs.setTableName(tableName);
     } catch (Exception e) {
@@ -405,9 +408,10 @@ public class TableConfigsRestletResource {
       throw new ControllerApplicationException(LOGGER,
           String.format("Invalid TableConfigs json string: %s", tableConfigsStr), Response.Status.BAD_REQUEST, e);
     }
+    String databaseName = DatabaseUtils.extractDatabaseFromHttpHeaders(httpHeaders);
     TableConfigs tableConfigs = tableConfigsAndUnrecognizedProps.getLeft();
-    validateConfig(tableConfigs, typesToSkip);
-    String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), httpHeaders);
+    validateConfig(tableConfigs, databaseName, typesToSkip);
+    String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), databaseName);
     tableConfigs.setTableName(rawTableName);
 
     // validate permission
@@ -429,8 +433,8 @@ public class TableConfigsRestletResource {
     TableConfigUtils.ensureStorageQuotaConstraints(tableConfig, _controllerConf.getDimTableMaxSize());
   }
 
-  private void validateConfig(TableConfigs tableConfigs, @Nullable String typesToSkip) {
-    String rawTableName = tableConfigs.getTableName();
+  private void validateConfig(TableConfigs tableConfigs, String database, @Nullable String typesToSkip) {
+    String rawTableName = DatabaseUtils.translateTableName(tableConfigs.getTableName(), database);
     TableConfig offlineTableConfig = tableConfigs.getOffline();
     TableConfig realtimeTableConfig = tableConfigs.getRealtime();
     Schema schema = tableConfigs.getSchema();
@@ -439,20 +443,23 @@ public class TableConfigsRestletResource {
           "Must provide at least one of 'realtime' or 'offline' table configs for adding TableConfigs: %s",
           rawTableName);
       Preconditions.checkState(schema != null, "Must provide 'schema' for adding TableConfigs: %s", rawTableName);
+      String schemaName = DatabaseUtils.translateTableName(schema.getSchemaName(), database);
       Preconditions.checkState(!rawTableName.isEmpty(), "'tableName' cannot be empty in TableConfigs");
 
-      Preconditions.checkState(rawTableName.equals(schema.getSchemaName()),
+      Preconditions.checkState(rawTableName.equals(schemaName),
           "'tableName': %s must be equal to 'schemaName' from 'schema': %s", rawTableName, schema.getSchemaName());
       SchemaUtils.validate(schema);
       if (offlineTableConfig != null) {
-        String offlineRawTableName = TableNameBuilder.extractRawTableName(offlineTableConfig.getTableName());
+        String offlineRawTableName = DatabaseUtils.translateTableName(
+            TableNameBuilder.extractRawTableName(offlineTableConfig.getTableName()), database);
         Preconditions.checkState(offlineRawTableName.equals(rawTableName),
             "Name in 'offline' table config: %s must be equal to 'tableName': %s", offlineRawTableName, rawTableName);
         TableConfigUtils.validateTableName(offlineTableConfig);
         TableConfigUtils.validate(offlineTableConfig, schema, typesToSkip);
       }
       if (realtimeTableConfig != null) {
-        String realtimeRawTableName = TableNameBuilder.extractRawTableName(realtimeTableConfig.getTableName());
+        String realtimeRawTableName = DatabaseUtils.translateTableName(
+            TableNameBuilder.extractRawTableName(realtimeTableConfig.getTableName()), database);
         Preconditions.checkState(realtimeRawTableName.equals(rawTableName),
             "Name in 'realtime' table config: %s must be equal to 'tableName': %s", realtimeRawTableName, rawTableName);
         TableConfigUtils.validateTableName(realtimeTableConfig);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
index 05830f3fa2..54cc7b0ac0 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/api/TableConfigsRestletResourceTest.java
@@ -23,7 +23,9 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Sets;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import org.apache.pinot.controller.helix.ControllerTest;
 import org.apache.pinot.core.realtime.impl.fakestream.FakeStreamConfigUtils;
 import org.apache.pinot.spi.config.TableConfigs;
@@ -36,8 +38,10 @@ import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.MetricFieldSpec;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.stream.StreamConfig;
+import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.spi.utils.JsonUtils;
 import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
+import org.apache.pinot.spi.utils.builder.TableNameBuilder;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -242,6 +246,52 @@ public class TableConfigsRestletResourceTest extends ControllerTest {
       // expected
     }
 
+    // table name check fails when database context is not passed in header but one of the configs has database prefix
+    Schema dummySchema = createDummySchema(tableName);
+    TableConfig offlineTableConfig1 = createOfflineTableConfig("db1." + tableName);
+    TableConfig realtimeTableConfig1 = createRealtimeTableConfig(tableName);
+    tableConfigs = new TableConfigs(tableName, dummySchema, offlineTableConfig1, realtimeTableConfig1);
+    try {
+      sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+      fail("Creation of an TableConfigs without database context in header but provided in one of the configs should "
+          + "fail");
+    } catch (Exception e) {
+      // expected
+    }
+    // fails with schema as well
+    offlineTableConfig1.setTableName(TableNameBuilder.OFFLINE.tableNameWithType(tableName));
+    dummySchema.setSchemaName("db1." + tableName);
+    try {
+      sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+      fail("Creation of an TableConfigs without database context in header but provided in one of the configs should "
+          + "fail");
+    } catch (Exception e) {
+      // expected
+    }
+
+    // fails even though both configs and schema have the database prefix in the table names but
+    // database context is not passed in header
+    tableConfigs.setTableName("db1." + tableName);
+    try {
+      sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString());
+      fail("Creation of an TableConfigs without database context in header but provided in all of the configs should "
+          + "fail");
+    } catch (Exception e) {
+      // expected
+    }
+
+    // successfully created with all 3 configs when database context is passed in header and configs may or may not
+    // have the database prefix in the table names
+    Map<String, String> headers = new HashMap<>();
+    tableConfigs.setTableName(tableName);
+    headers.put(CommonConstants.DATABASE, "db1");
+    // only schema has the database prefix
+    dummySchema.setSchemaName("db1." + tableName);
+    sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString(), headers);
+    // one of the table config has database prefix
+    offlineTableConfig1.setTableName(TableNameBuilder.OFFLINE.tableNameWithType(tableName));
+    sendPostRequest(validateConfigUrl, tableConfigs.toPrettyJsonString(), headers);
+
     // successfully created with all 3 configs
     String tableName1 = "testValidate1";
     tableConfigs = new TableConfigs(tableName1, createDummySchema(tableName1), createOfflineTableConfig(tableName1),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org