You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2022/10/20 17:45:05 UTC
[pinot] branch master updated: Add segment level debug API (#9609)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7de438e434 Add segment level debug API (#9609)
7de438e434 is described below
commit 7de438e434f84da094554fec8922bd7d8b23faac
Author: Saurabh Dubey <sa...@gmail.com>
AuthorDate: Thu Oct 20 23:14:58 2022 +0530
Add segment level debug API (#9609)
---
...{TableDebugResource.java => DebugResource.java} | 84 ++++++++++++++++++++--
.../pinot/server/api/resources/DebugResource.java | 34 ++++++++-
2 files changed, 112 insertions(+), 6 deletions(-)
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
similarity index 83%
rename from pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java
rename to pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
index d10c29f1e6..badd427bcc 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/TableDebugResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/DebugResource.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Executor;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -89,9 +90,9 @@ import static org.apache.pinot.spi.utils.CommonConstants.SWAGGER_AUTHORIZATION_K
@Api(tags = Constants.CLUSTER_TAG, authorizations = {@Authorization(value = SWAGGER_AUTHORIZATION_KEY)})
@SwaggerDefinition(securityDefinition = @SecurityDefinition(apiKeyAuthDefinitions = @ApiKeyAuthDefinition(name =
HttpHeaders.AUTHORIZATION, in = ApiKeyAuthDefinition.ApiKeyLocation.HEADER, key = SWAGGER_AUTHORIZATION_KEY)))
-@Path("/")
-public class TableDebugResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(TableDebugResource.class);
+@Path("/debug/")
+public class DebugResource {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DebugResource.class);
@Inject
PinotHelixResourceManager _pinotHelixResourceManager;
@@ -112,7 +113,7 @@ public class TableDebugResource {
ControllerConf _controllerConf;
@GET
- @Path("/debug/tables/{tableName}")
+ @Path("tables/{tableName}")
@Produces(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Get debug information for table.", notes = "Debug information for table.")
@ApiResponses(value = {
@@ -142,6 +143,22 @@ public class TableDebugResource {
return JsonUtils.objectToPrettyString(tableDebugInfos);
}
+ @GET
+ @Path("segments/{tableName}/{segmentName}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get debug information for segment.", notes = "Debug information for segment.")
+ @ApiResponses(value = {
+ @ApiResponse(code = 200, message = "Success"), @ApiResponse(code = 404, message = "Segment not found"),
+ @ApiResponse(code = 500, message = "Internal server error")
+ })
+ public TableDebugInfo.SegmentDebugInfo getSegmentDebugInfo(
+ @ApiParam(value = "Name of the table (with type)", required = true) @PathParam("tableName")
+ String tableNameWithType,
+ @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") String segmentName)
+ throws Exception {
+ return debugSegment(tableNameWithType, segmentName);
+ }
+
/**
* Helper method to collect debug information about the table.
*
@@ -217,6 +234,65 @@ public class TableDebugResource {
tableSizeDetails._estimatedSizeInBytes) : new TableDebugInfo.TableSizeSummary(-1, -1);
}
+ private TableDebugInfo.SegmentDebugInfo debugSegment(String tableNameWithType, String segmentName)
+ throws IOException {
+ IdealState idealState = _pinotHelixResourceManager.getTableIdealState(tableNameWithType);
+ if (idealState == null) {
+ return null;
+ }
+
+ ExternalView externalView = _pinotHelixResourceManager.getTableExternalView(tableNameWithType);
+ Map<String, String> evStateMap = (externalView != null) ? externalView.getStateMap(segmentName) : null;
+
+ Map<String, String> isServerToStateMap = idealState.getRecord().getMapFields().get(segmentName);
+ Set<String> serversHostingSegment = _pinotHelixResourceManager.getServers(tableNameWithType, segmentName);
+
+ int serverRequestTimeoutMs = _controllerConf.getServerAdminRequestTimeoutSeconds() * 1000;
+ BiMap<String, String> serverToEndpoints;
+ try {
+ serverToEndpoints = _pinotHelixResourceManager.getDataInstanceAdminEndpoints(serversHostingSegment);
+ } catch (InvalidConfigException e) {
+ throw new WebApplicationException(
+ "Caught exception when getting segment debug info for table: " + tableNameWithType);
+ }
+
+ List<String> serverUrls = new ArrayList<>(serverToEndpoints.size());
+ BiMap<String, String> endpointsToServers = serverToEndpoints.inverse();
+ for (String endpoint : endpointsToServers.keySet()) {
+ String segmentDebugInfoURI = String.format("%s/debug/segments/%s/%s", endpoint, tableNameWithType, segmentName);
+ serverUrls.add(segmentDebugInfoURI);
+ }
+
+ CompletionServiceHelper completionServiceHelper =
+ new CompletionServiceHelper(_executor, _connectionManager, endpointsToServers);
+ CompletionServiceHelper.CompletionServiceResponse serviceResponse =
+ completionServiceHelper.doMultiGetRequest(serverUrls, tableNameWithType, false, serverRequestTimeoutMs);
+
+ Map<String, SegmentServerDebugInfo> serverToSegmentDebugInfo = new HashMap<>();
+ for (Map.Entry<String, String> streamResponse : serviceResponse._httpResponses.entrySet()) {
+ SegmentServerDebugInfo segmentDebugInfo =
+ JsonUtils.stringToObject(streamResponse.getValue(), SegmentServerDebugInfo.class);
+ serverToSegmentDebugInfo.put(streamResponse.getKey(), segmentDebugInfo);
+ }
+
+ Map<String, TableDebugInfo.SegmentState> segmentServerState = new HashMap<>();
+ for (String instanceName : isServerToStateMap.keySet()) {
+ String isState = isServerToStateMap.get(instanceName);
+ String evState = (evStateMap != null) ? evStateMap.get(instanceName) : null;
+ SegmentServerDebugInfo segmentServerDebugInfo = serverToSegmentDebugInfo.get(instanceName);
+
+ if (segmentServerDebugInfo != null) {
+ segmentServerState.put(instanceName,
+ new TableDebugInfo.SegmentState(isState, evState, segmentServerDebugInfo.getSegmentSize(),
+ segmentServerDebugInfo.getConsumerInfo(), segmentServerDebugInfo.getErrorInfo()));
+ } else {
+ segmentServerState.put(instanceName, new TableDebugInfo.SegmentState(isState, evState, null, null, null));
+ }
+ }
+
+ return new TableDebugInfo.SegmentDebugInfo(segmentName, segmentServerState);
+ }
+
/**
* Helper method to debug segments. Computes differences between ideal state and external view for each segment.
*
diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
index 9d0a1976fc..9132f28556 100644
--- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
+++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/DebugResource.java
@@ -75,12 +75,42 @@ public class DebugResource {
@ApiOperation(value = "Get segments debug info for this table",
notes = "This is a debug endpoint, and won't maintain backward compatibility")
public List<SegmentServerDebugInfo> getSegmentsDebugInfo(
- @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableNameWithType) {
-
+ @ApiParam(value = "Name of the table (with type)", required = true) @PathParam("tableName")
+ String tableNameWithType) {
TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
return getSegmentServerDebugInfo(tableNameWithType, tableType);
}
+ @GET
+ @Path("segments/{tableName}/{segmentName}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation(value = "Get segment debug info",
+ notes = "This is a debug endpoint, and won't maintain backward compatibility")
+ public SegmentServerDebugInfo getSegmentDebugInfo(
+ @ApiParam(value = "Name of the table (with type)", required = true) @PathParam("tableName")
+ String tableNameWithType,
+ @ApiParam(value = "Name of the segment", required = true) @PathParam("segmentName") String segmentName) {
+ TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
+ TableDataManager tableDataManager =
+ ServerResourceUtils.checkGetTableDataManager(_serverInstance, tableNameWithType);
+ Map<String, SegmentErrorInfo> segmentErrorsMap = tableDataManager.getSegmentErrors();
+ SegmentDataManager segmentDataManager = tableDataManager.acquireSegment(segmentName);
+ try {
+ SegmentConsumerInfo segmentConsumerInfo = getSegmentConsumerInfo(segmentDataManager, tableType);
+ long segmentSize = getSegmentSize(segmentDataManager);
+ SegmentErrorInfo segmentErrorInfo = segmentErrorsMap.get(segmentName);
+ return new SegmentServerDebugInfo(segmentName, FileUtils.byteCountToDisplaySize(segmentSize), segmentConsumerInfo,
+ segmentErrorInfo);
+ } catch (Exception e) {
+ throw new WebApplicationException(
+ "Caught exception when getting consumer info for table: " + tableNameWithType + " segment: " + segmentName);
+ } finally {
+ if (segmentDataManager != null) {
+ tableDataManager.releaseSegment(segmentDataManager);
+ }
+ }
+ }
+
private List<SegmentServerDebugInfo> getSegmentServerDebugInfo(String tableNameWithType, TableType tableType) {
List<SegmentServerDebugInfo> segmentServerDebugInfos = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org