You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ja...@apache.org on 2024/11/04 22:39:19 UTC
(pinot) branch master updated: Improve MSQ explain and stageStats when dealing with empty tables (#14374)
This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c0529240ba Improve MSQ explain and stageStats when dealing with empty tables (#14374)
c0529240ba is described below
commit c0529240bafc9c09a35ac7b1904a98ab72e4f873
Author: Gonzalo Ortiz Jaureguizar <go...@users.noreply.github.com>
AuthorDate: Mon Nov 4 23:39:12 2024 +0100
Improve MSQ explain and stageStats when dealing with empty tables (#14374)
---
.../MultiStageBrokerRequestHandler.java | 7 ++----
.../org/apache/pinot/query/catalog/PinotTable.java | 9 ++++---
.../explain/AskingServerStageExplainer.java | 16 +++++++++----
.../MultiStageExplainAskingServersUtils.java | 2 +-
.../query/runtime/MultiStageStatsTreeBuilder.java | 28 +++++++++++++++++++---
.../query/service/dispatch/QueryDispatcher.java | 4 ++++
.../runtime/queries/ResourceBasedQueriesTest.java | 8 +------
7 files changed, 51 insertions(+), 23 deletions(-)
diff --git a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
index 8aef51ebd1..c199665425 100644
--- a/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
+++ b/pinot-broker/src/main/java/org/apache/pinot/broker/requesthandler/MultiStageBrokerRequestHandler.java
@@ -297,11 +297,8 @@ public class MultiStageBrokerRequestHandler extends BaseBrokerRequestHandler {
List<MultiStageQueryStats.StageStats.Closed> queryStats, DispatchableSubPlan dispatchableSubPlan) {
try {
List<DispatchablePlanFragment> stagePlans = dispatchableSubPlan.getQueryStageList();
- List<PlanNode> planNodes = new ArrayList<>(stagePlans.size());
- for (DispatchablePlanFragment stagePlan : stagePlans) {
- planNodes.add(stagePlan.getPlanFragment().getFragmentRoot());
- }
- MultiStageStatsTreeBuilder treeBuilder = new MultiStageStatsTreeBuilder(planNodes, queryStats);
+
+ MultiStageStatsTreeBuilder treeBuilder = new MultiStageStatsTreeBuilder(stagePlans, queryStats);
brokerResponse.setStageStats(treeBuilder.jsonStatsByStage(0));
for (MultiStageQueryStats.StageStats.Closed stageStats : queryStats) {
if (stageStats != null) { // for example pipeline breaker may not have stats
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/catalog/PinotTable.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/catalog/PinotTable.java
index 36a5f3730c..aee8ea8573 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/catalog/PinotTable.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/catalog/PinotTable.java
@@ -18,7 +18,6 @@
*/
package org.apache.pinot.query.catalog;
-import com.google.common.base.Preconditions;
import org.apache.calcite.DataContext;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.rel.type.RelDataType;
@@ -44,8 +43,12 @@ public class PinotTable extends AbstractTable implements ScannableTable {
@Override
public RelDataType getRowType(RelDataTypeFactory relDataTypeFactory) {
- Preconditions.checkState(relDataTypeFactory instanceof TypeFactory);
- TypeFactory typeFactory = (TypeFactory) relDataTypeFactory;
+ TypeFactory typeFactory;
+ if (relDataTypeFactory instanceof TypeFactory) {
+ typeFactory = (TypeFactory) relDataTypeFactory;
+ } else { // this can happen when using Frameworks.withPrepare, which wraps our factory in a JavaTypeFactoryImpl
+ typeFactory = TypeFactory.INSTANCE;
+ }
return typeFactory.createRelDataTypeFromSchema(_schema);
}
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/AskingServerStageExplainer.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/AskingServerStageExplainer.java
index fa68e3f1fb..a84f6f1f18 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/AskingServerStageExplainer.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/AskingServerStageExplainer.java
@@ -55,9 +55,20 @@ public class AskingServerStageExplainer {
public RelNode explainFragment(DispatchablePlanFragment fragment) {
_relBuilder.clear();
- Collection<PlanNode> planNodes = _onServerExplainer.explainOnServers(fragment);
+
+ PlanNode fragmentRoot = fragment.getPlanFragment().getFragmentRoot();
+ int stageId = fragmentRoot.getStageId();
+ DataSchema schema = fragmentRoot.getDataSchema();
HashMap<PlanNode, Integer> planNodesMap = new HashMap<>();
+ if (fragment.getWorkerIdToSegmentsMap().isEmpty()) {
+ PlanNode mergedNode = new ExplainedNode(stageId, schema, null, Collections.emptyList(), "PlanWithNoSegments",
+ new ExplainAttributeBuilder()
+ .putString("table", fragment.getTableName())
+ .build());
+ return PlanNodeToRelConverter.convert(_relBuilder, mergedNode);
+ }
+ Collection<PlanNode> planNodes = _onServerExplainer.explainOnServers(fragment);
for (PlanNode planNode : planNodes) {
PlanNode simplifiedPlan = !_verbose ? ExplainNodeSimplifier.simplifyNode(planNode) : planNode;
PlanNode sortedPlan = PlanNodeSorter.sort(simplifiedPlan);
@@ -66,9 +77,6 @@ public class AskingServerStageExplainer {
}
PlanNode mergedNode;
- PlanNode fragmentRoot = fragment.getPlanFragment().getFragmentRoot();
- int stageId = fragmentRoot.getStageId();
- DataSchema schema = fragmentRoot.getDataSchema();
switch (planNodesMap.size()) {
case 0: {
mergedNode = new ExplainedNode(stageId, schema, null, Collections.emptyList(), "NoPlanInformation",
diff --git a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/MultiStageExplainAskingServersUtils.java b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/MultiStageExplainAskingServersUtils.java
index 4e287be646..9e19261da2 100644
--- a/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/MultiStageExplainAskingServersUtils.java
+++ b/pinot-query-planner/src/main/java/org/apache/pinot/query/planner/explain/MultiStageExplainAskingServersUtils.java
@@ -52,7 +52,7 @@ public class MultiStageExplainAskingServersUtils {
TransformationTracker<PlanNode, RelNode> tracker, AskingServerStageExplainer serversExplainer) {
// extract a key node operator
Map<DispatchablePlanFragment, PlanNode> leafNodes = queryStages.stream()
- .filter(fragment -> !fragment.getWorkerIdToSegmentsMap().isEmpty()) // ignore root and intermediate stages
+ .filter(fragment -> fragment.getTableName() != null) // ignore root and intermediate stages
.collect(Collectors.toMap(Function.identity(), fragment -> fragment.getPlanFragment().getFragmentRoot()));
// creates a map where each leaf node is converted into another RelNode that may contain physical information
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/MultiStageStatsTreeBuilder.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/MultiStageStatsTreeBuilder.java
index 74c81ed820..cd48518e2f 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/MultiStageStatsTreeBuilder.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/runtime/MultiStageStatsTreeBuilder.java
@@ -19,24 +19,46 @@
package org.apache.pinot.query.runtime;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.pinot.query.planner.physical.DispatchablePlanFragment;
import org.apache.pinot.query.planner.plannode.PlanNode;
import org.apache.pinot.query.runtime.plan.MultiStageQueryStats;
+import org.apache.pinot.spi.utils.JsonUtils;
public class MultiStageStatsTreeBuilder {
private final List<PlanNode> _planNodes;
private final List<? extends MultiStageQueryStats.StageStats> _queryStats;
+ private final List<DispatchablePlanFragment> _planFragments;
- public MultiStageStatsTreeBuilder(List<PlanNode> planNodes,
+ public MultiStageStatsTreeBuilder(List<DispatchablePlanFragment> planFragments,
List<? extends MultiStageQueryStats.StageStats> queryStats) {
- _planNodes = planNodes;
+ _planFragments = planFragments;
+ _planNodes = new ArrayList<>(planFragments.size());
+ for (DispatchablePlanFragment stagePlan : planFragments) {
+ _planNodes.add(stagePlan.getPlanFragment().getFragmentRoot());
+ }
_queryStats = queryStats;
}
public ObjectNode jsonStatsByStage(int stage) {
- MultiStageQueryStats.StageStats stageStats = _queryStats.get(stage);
PlanNode planNode = _planNodes.get(stage);
+
+ MultiStageQueryStats.StageStats stageStats = stage < _queryStats.size() ? _queryStats.get(stage) : null;
+ if (stageStats == null) {
+ // We don't have stats for this stage. This can happen when the stage is not executed. For example, when there
+ // are no segments for a table.
+ ObjectNode jsonNodes = JsonUtils.newObjectNode();
+ jsonNodes.put("type", "EMPTY_MAILBOX_SEND");
+ jsonNodes.put("stage", stage);
+ jsonNodes.put("description", "No stats available for this stage. It may have been pruned.");
+ String tableName = _planFragments.get(stage).getTableName();
+ if (tableName != null) {
+ jsonNodes.put("table", tableName);
+ }
+ return jsonNodes;
+ }
InStageStatsTreeBuilder treeBuilder = new InStageStatsTreeBuilder(stageStats, this::jsonStatsByStage);
return planNode.visit(treeBuilder, null);
}
diff --git a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
index 4a2c72ad10..c6d21b7c1c 100644
--- a/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
+++ b/pinot-query-runtime/src/main/java/org/apache/pinot/query/service/dispatch/QueryDispatcher.java
@@ -222,6 +222,10 @@ public class QueryDispatcher {
List<StageInfo> stageInfos = serializePlanFragments(stagePlans, serverInstances, deadline);
+ if (serverInstances.isEmpty()) {
+ throw new RuntimeException("No server instances to dispatch query to");
+ }
+
Map<String, String> requestMetadata = prepareRequestMetadata(requestId, queryOptions, deadline);
ByteString protoRequestMetadata = QueryPlanSerDeUtils.toProtoProperties(requestMetadata);
diff --git a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
index 9ffce6810d..c09c3cd8ec 100644
--- a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
+++ b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/queries/ResourceBasedQueriesTest.java
@@ -37,16 +37,13 @@ import java.util.Set;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.common.response.broker.BrokerResponseNativeV2;
import org.apache.pinot.query.QueryEnvironmentTestBase;
import org.apache.pinot.query.QueryServerEnclosure;
import org.apache.pinot.query.mailbox.MailboxService;
-import org.apache.pinot.query.planner.PlanFragment;
import org.apache.pinot.query.planner.physical.DispatchablePlanFragment;
-import org.apache.pinot.query.planner.plannode.PlanNode;
import org.apache.pinot.query.routing.QueryServerInstance;
import org.apache.pinot.query.runtime.MultiStageStatsTreeBuilder;
import org.apache.pinot.query.runtime.operator.LeafStageTransferableBlockOperator;
@@ -279,10 +276,7 @@ public class ResourceBasedQueriesTest extends QueryRunnerTestBase {
private Map<String, JsonNode> tableToStats(String sql, QueryDispatcher.QueryResult queryResult) {
- List<PlanNode> planNodes = planQuery(sql).getQueryPlan().getQueryStageList().stream()
- .map(DispatchablePlanFragment::getPlanFragment)
- .map(PlanFragment::getFragmentRoot)
- .collect(Collectors.toList());
+ List<DispatchablePlanFragment> planNodes = planQuery(sql).getQueryPlan().getQueryStageList();
MultiStageStatsTreeBuilder multiStageStatsTreeBuilder =
new MultiStageStatsTreeBuilder(planNodes, queryResult.getQueryStats());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org