You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2021/10/08 23:15:04 UTC
[pinot] 01/01: Improve get tenant API to differentiate offline and
realtime tenants
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch improve-get-tenant-api
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit 035767314ac495f844688d91b1e9fcf8b02d1eed
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Fri Oct 8 16:14:24 2021 -0700
Improve get tenant API to differentiate offline and realtime tenants
---
.../pinot/common/utils/helix/HelixHelper.java | 27 ++++++++++++++-----
.../api/resources/PinotTenantRestletResource.java | 30 ++++++++++++++++++----
.../helix/core/PinotHelixResourceManager.java | 5 ++++
3 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
index 36490d4..9952588 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
@@ -46,6 +46,7 @@ import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.builder.HelixConfigScopeBuilder;
import org.apache.pinot.common.utils.config.TagNameUtils;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.utils.CommonConstants;
import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.apache.pinot.spi.utils.retry.RetryPolicy;
@@ -482,12 +483,26 @@ public class HelixHelper {
* TODO: refactor code to use this method if applicable to reuse instance configs in order to reduce ZK accesses
*/
public static Set<String> getServerInstancesForTenant(List<InstanceConfig> instanceConfigs, String tenant) {
- Set<String> serverInstances = new HashSet<>();
- serverInstances
- .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getOfflineTagForTenant(tenant)));
- serverInstances
- .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getRealtimeTagForTenant(tenant)));
- return serverInstances;
+ return getServerInstancesForTenantWithType(instanceConfigs, tenant, null);
+ }
+
+ /**
+ * Returns the server instances in the cluster for the given tenant name and tenant type.
+ *
+ * TODO: refactor code to use this method if applicable to reuse instance configs in order to reduce ZK accesses
+ */
+ public static Set<String> getServerInstancesForTenantWithType(List<InstanceConfig> instanceConfigs, String tenant,
+ TableType tableType) {
+ Set<String> serverInstancesWithType = new HashSet<>();
+ if (tableType == null || tableType == TableType.OFFLINE) {
+ serverInstancesWithType
+ .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getOfflineTagForTenant(tenant)));
+ }
+ if (tableType == null || tableType == TableType.REALTIME) {
+ serverInstancesWithType
+ .addAll(HelixHelper.getInstancesWithTag(instanceConfigs, TagNameUtils.getRealtimeTagForTenant(tenant)));
+ }
+ return serverInstancesWithType;
}
/**
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTenantRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTenantRestletResource.java
index 472945d..c295f39 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTenantRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTenantRestletResource.java
@@ -52,6 +52,7 @@ import org.apache.pinot.controller.api.exception.ControllerApplicationException;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import org.apache.pinot.controller.helix.core.PinotResourceManagerResponse;
import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.config.table.TableType;
import org.apache.pinot.spi.config.tenant.Tenant;
import org.apache.pinot.spi.config.tenant.TenantRole;
import org.apache.pinot.spi.utils.JsonUtils;
@@ -158,6 +159,10 @@ public class PinotTenantRestletResource {
public static class TenantMetadata {
@JsonProperty(value = "ServerInstances")
Set<String> _serverInstances;
+ @JsonProperty(value = "OfflineServerInstances")
+ Set<String> _offlineServerInstances;
+ @JsonProperty(value = "RealtimeServerInstances")
+ Set<String> _realtimeServerInstances;
@JsonProperty(value = "BrokerInstances")
Set<String> _brokerInstances;
@JsonProperty(TENANT_NAME)
@@ -197,10 +202,11 @@ public class PinotTenantRestletResource {
public String listInstanceOrToggleTenantState(
@ApiParam(value = "Tenant name", required = true) @PathParam("tenantName") String tenantName,
@ApiParam(value = "Tenant type (server|broker)") @QueryParam("type") String tenantType,
+ @ApiParam(value = "Table type (offline|realtime)") @QueryParam("tableType") String tableType,
@ApiParam(value = "state") @QueryParam("state") String stateStr)
throws Exception {
if (stateStr == null) {
- return listInstancesForTenant(tenantName, tenantType);
+ return listInstancesForTenant(tenantName, tenantType, tableType);
} else {
return toggleTenantState(tenantName, stateStr, tenantType);
}
@@ -280,7 +286,7 @@ public class PinotTenantRestletResource {
return null;
}
- private String listInstancesForTenant(String tenantName, String tenantType) {
+ private String listInstancesForTenant(String tenantName, String tenantType, String tableTypeString) {
ObjectNode resourceGetRet = JsonUtils.newObjectNode();
List<InstanceConfig> instanceConfigList = _pinotHelixResourceManager.getAllHelixInstanceConfigs();
@@ -299,9 +305,23 @@ public class PinotTenantRestletResource {
resourceGetRet.set("BrokerInstances", JsonUtils.objectToJsonNode(allBrokerInstances));
} else {
if (tenantType.equalsIgnoreCase("server")) {
- Set<String> allServerInstances =
- _pinotHelixResourceManager.getAllInstancesForServerTenant(instanceConfigList, tenantName);
-
+ Set<String> allServerInstances = new HashSet<>();
+ TableType specificTableType = null;
+ if (tableTypeString != null) {
+ specificTableType = TableType.valueOf(tableTypeString.toUpperCase());
+ }
+ if (specificTableType == null || specificTableType == TableType.OFFLINE) {
+ Set<String> offlineServerInstances = _pinotHelixResourceManager
+ .getAllInstancesForServerTenantWithType(instanceConfigList, tenantName, TableType.OFFLINE);
+ resourceGetRet.set("OfflineServerInstances", JsonUtils.objectToJsonNode(offlineServerInstances));
+ allServerInstances.addAll(offlineServerInstances);
+ }
+ if (specificTableType == null || specificTableType == TableType.REALTIME) {
+ Set<String> realtimeServerInstances = _pinotHelixResourceManager
+ .getAllInstancesForServerTenantWithType(instanceConfigList, tenantName, TableType.REALTIME);
+ resourceGetRet.set("RealtimeServerInstances", JsonUtils.objectToJsonNode(realtimeServerInstances));
+ allServerInstances.addAll(realtimeServerInstances);
+ }
if (allServerInstances.isEmpty()) {
throw new ControllerApplicationException(LOGGER,
"Failed to find any instances for server tenant: " + tenantName, Response.Status.NOT_FOUND);
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
index 8677798..27f0da0 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/PinotHelixResourceManager.java
@@ -1020,6 +1020,11 @@ public class PinotHelixResourceManager {
return getAllInstancesForServerTenant(HelixHelper.getInstanceConfigs(_helixZkManager), tenantName);
}
+ public Set<String> getAllInstancesForServerTenantWithType(List<InstanceConfig> instanceConfigs, String tenantName,
+ TableType tableType) {
+ return HelixHelper.getServerInstancesForTenantWithType(instanceConfigs, tenantName, tableType);
+ }
+
/**
* TODO: refactor code to use this method over {@link #getAllInstancesForBrokerTenant(String)} if applicable to reuse
* instance configs in order to reduce ZK accesses
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org