You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2021/10/20 18:46:27 UTC
[pinot] branch master updated: Make dim tables creation bypass
tenant validation (#7559)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 471d16c Make dim tables creation bypass tenant validation (#7559)
471d16c is described below
commit 471d16cf5d835f41293b09d991e3d279a0b9dc1d
Author: deemoliu <qi...@uber.com>
AuthorDate: Wed Oct 20 11:46:13 2021 -0700
Make dim tables creation bypass tenant validation (#7559)
---
.../pinot/common/utils/config/TagNameUtils.java | 7 +++++++
.../helix/core/PinotHelixResourceManager.java | 13 +++++++++++-
.../PinotHelixResourceManagerStatelessTest.java | 23 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TagNameUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TagNameUtils.java
index 231e7a2..15ed490 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TagNameUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/config/TagNameUtils.java
@@ -125,6 +125,13 @@ public class TagNameUtils {
}
/**
+ * Extracts the REALTIME server tag name from the given tenant config.
+ */
+ public static String extractRealtimeServerTag(TenantConfig tenantConfig) {
+ return getRealtimeTagForTenant(tenantConfig.getServer());
+ }
+
+ /**
* Extracts the REALTIME consuming server tag name from the given tenant config.
*/
public static String extractConsumingServerTag(TenantConfig tenantConfig) {
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 27f0da0..8686671 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1282,7 +1282,18 @@ public class PinotHelixResourceManager {
// Check if tenant exists before creating the table
Set<String> tagsToCheck = new TreeSet<>();
tagsToCheck.add(TagNameUtils.extractBrokerTag(tenantConfig));
- if (tableConfig.getTableType() == TableType.OFFLINE) {
+
+ // The dimension table is an OFFLINE table but it will be deployed to the same tenant as the fact table.
+ // Thus, check if any REALTIME or OFFLINE tagged servers exists before creating dimension table.
+ if (tableConfig.isDimTable()) {
+ String offlineTag = TagNameUtils.extractOfflineServerTag(tenantConfig);
+ String realtimeTag = TagNameUtils.extractRealtimeServerTag(tenantConfig);
+
+ if (getInstancesWithTag(offlineTag).isEmpty() && getInstancesWithTag(realtimeTag).isEmpty()) {
+ throw new InvalidTableConfigException(
+ "Failed to find instances for dimension table: " + tableNameWithType);
+ }
+ } else if (tableConfig.getTableType() == TableType.OFFLINE) {
tagsToCheck.add(TagNameUtils.extractOfflineServerTag(tenantConfig));
} else {
String consumingServerTag = TagNameUtils.extractConsumingServerTag(tenantConfig);
diff --git a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
index 6cd7eb9..3ab0d48 100644
--- a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
+++ b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManagerStatelessTest.java
@@ -76,6 +76,29 @@ public class PinotHelixResourceManagerStatelessTest extends ControllerTest {
}
@Test
+ public void testValidateDimTableTenantConfig() {
+ // Create broker tenant on 3 Brokers
+ Tenant brokerTenant = new Tenant(TenantRole.BROKER, BROKER_TENANT_NAME, 3, 0, 0);
+ _helixResourceManager.createBrokerTenant(brokerTenant);
+
+ String rawTableName = "testTable";
+
+ // Dim table missing broker
+ TableConfig dimTableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(rawTableName).build();
+ dimTableConfig.setTenantConfig(new TenantConfig(null, SERVER_TENANT_NAME, null));
+ try {
+ _helixResourceManager.validateTableTenantConfig(dimTableConfig);
+ Assert.fail("Expected InvalidTableConfigException");
+ } catch (InvalidTableConfigException e) {
+ // expected
+ }
+
+ // Dim table (offline) deployed to realtime tenant
+ dimTableConfig.setTenantConfig(new TenantConfig(BROKER_TENANT_NAME, SERVER_TENANT_NAME, null));
+ _helixResourceManager.validateTableTenantConfig(dimTableConfig);
+ }
+
+ @Test
public void testValidateTenantConfig() {
// Create broker tenant on 3 Brokers
Tenant brokerTenant = new Tenant(TenantRole.BROKER, BROKER_TENANT_NAME, 3, 0, 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org