You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/11/07 03:02:04 UTC
[iotdb] 02/03: add metrics (Parser/Analyzer/Planner)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6fb98918bcdc921dbf6fa46d3b7d80c5d8e30571
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Nov 7 10:57:58 2022 +0800
add metrics (Parser/Analyzer/Planner)
---
.../apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 7 +++++++
.../org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 9 ++++++++-
.../apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java | 6 ++++--
.../plan/planner/distribution/DistributionPlanner.java | 16 ++++++++++++----
.../db/service/thrift/impl/ClientRPCServiceImpl.java | 8 +++++++-
5 files changed, 38 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index cf6acaa2f9..392e0ae6d9 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -105,6 +105,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.ShowSchemaTempla
import org.apache.iotdb.db.mpp.plan.statement.sys.ExplainStatement;
import org.apache.iotdb.db.mpp.plan.statement.sys.ShowVersionStatement;
import org.apache.iotdb.db.mpp.plan.statement.sys.sync.ShowPipeSinkTypeStatement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.SessionManager;
import org.apache.iotdb.db.utils.FileLoaderUtils;
import org.apache.iotdb.db.utils.TimePartitionUtils;
@@ -196,11 +197,15 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
// request schema fetch API
logger.debug("[StartFetchSchema]");
ISchemaTree schemaTree;
+
+ long t1 = System.nanoTime();
if (queryStatement.isGroupByTag()) {
schemaTree = schemaFetcher.fetchSchemaWithTags(patternTree);
} else {
schemaTree = schemaFetcher.fetchSchema(patternTree);
}
+ QueryStatistics.getInstance().addCost("SchemaFetcher", System.nanoTime() - t1);
+
logger.debug("[EndFetchSchema]");
// If there is no leaf node in the schema tree, the query should be completed immediately
if (schemaTree.isEmpty()) {
@@ -278,7 +283,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analyzeOutput(analysis, queryStatement, outputExpressions);
// fetch partition information
+ long t2 = System.nanoTime();
analyzeDataPartition(analysis, queryStatement, schemaTree);
+ QueryStatistics.getInstance().addCost("PartitionFetcher", System.nanoTime() - t2);
} catch (StatementAnalyzeException e) {
logger.error("Meet error when analyzing the query statement: ", e);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java
index 010b491e55..6fed101bc3 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.mpp.plan.analyze;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.mpp.common.MPPQueryContext;
import org.apache.iotdb.db.mpp.plan.statement.Statement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import static org.apache.iotdb.db.mpp.common.QueryId.mockQueryId;
@@ -40,7 +41,13 @@ public class Analyzer {
}
public Analysis analyze(Statement statement) {
- return new AnalyzeVisitor(partitionFetcher, schemaFetcher).process(statement, context);
+ long startTime = System.nanoTime();
+ Analysis analysis =
+ new AnalyzeVisitor(partitionFetcher, schemaFetcher).process(statement, context);
+ if (statement.isQuery()) {
+ QueryStatistics.getInstance().addCost("Analyzer", System.nanoTime() - startTime);
+ }
+ return analysis;
}
public static void validate(Statement statement) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
index 308887f53a..a894c8ddf0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.db.mpp.plan.analyze.Analysis;
import org.apache.iotdb.db.mpp.plan.optimization.PlanOptimizer;
import org.apache.iotdb.db.mpp.plan.planner.plan.LogicalQueryPlan;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import java.util.List;
@@ -39,10 +39,12 @@ public class LogicalPlanner {
}
public LogicalQueryPlan plan(Analysis analysis) {
+ long startTime = System.nanoTime();
PlanNode rootNode = new LogicalPlanVisitor(analysis).process(analysis.getStatement(), context);
// optimize the query logical plan
- if (analysis.getStatement() instanceof QueryStatement) {
+ if (analysis.getStatement().isQuery()) {
+ QueryStatistics.getInstance().addCost("LogicalPlanner", System.nanoTime() - startTime);
for (PlanOptimizer optimizer : optimizers) {
rootNode = optimizer.optimize(rootNode, context);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/DistributionPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/DistributionPlanner.java
index 60882f5cdf..2b2a69188c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/DistributionPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/distribution/DistributionPlanner.java
@@ -32,7 +32,7 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.process.ExchangeNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.sink.FragmentSinkNode;
-import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import java.util.List;
@@ -65,9 +65,11 @@ public class DistributionPlanner {
}
public DistributedQueryPlan planFragments() {
+ long startTime = System.nanoTime();
+
PlanNode rootAfterRewrite = rewriteSource();
PlanNode rootWithExchange = addExchangeNode(rootAfterRewrite);
- if (analysis.getStatement() instanceof QueryStatement) {
+ if (analysis.getStatement().isQuery()) {
analysis
.getRespDatasetHeader()
.setColumnToTsBlockIndexMap(rootWithExchange.getOutputColumnNames());
@@ -80,8 +82,14 @@ public class DistributionPlanner {
if (context.getQueryType() == QueryType.READ) {
SetSinkForRootInstance(subPlan, fragmentInstances);
}
- return new DistributedQueryPlan(
- logicalPlan.getContext(), subPlan, subPlan.getPlanFragmentList(), fragmentInstances);
+ DistributedQueryPlan distributedQueryPlan =
+ new DistributedQueryPlan(
+ logicalPlan.getContext(), subPlan, subPlan.getPlanFragmentList(), fragmentInstances);
+
+ if (analysis.getStatement().isQuery()) {
+ QueryStatistics.getInstance().addCost("DistributionPlanner", System.nanoTime() - startTime);
+ }
+ return distributedQueryPlan;
}
// Convert fragment to detailed instance
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
index e002d89ad5..4ab5592612 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/ClientRPCServiceImpl.java
@@ -59,6 +59,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.template.CreateSchemaTemp
import org.apache.iotdb.db.mpp.plan.statement.metadata.template.DropSchemaTemplateStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.template.SetSchemaTemplateStatement;
import org.apache.iotdb.db.mpp.plan.statement.metadata.template.UnsetSchemaTemplateStatement;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.SessionManager;
import org.apache.iotdb.db.query.control.clientsession.IClientSession;
import org.apache.iotdb.db.service.basic.BasicOpenSessionResp;
@@ -190,16 +191,21 @@ public class ClientRPCServiceImpl implements IClientRPCServiceWithHandler {
}
long startTime = System.currentTimeMillis();
+ long startTimeInNano = System.nanoTime();
try {
Statement s =
StatementGenerator.createStatement(
statement, SESSION_MANAGER.getCurrSession().getZoneId());
-
if (s == null) {
return RpcUtils.getTSExecuteStatementResp(
RpcUtils.getStatus(
TSStatusCode.SQL_PARSE_ERROR, "This operation type is not supported"));
}
+
+ if (s.isQuery()) {
+ QueryStatistics.getInstance().addCost("Parser", System.nanoTime() - startTimeInNano);
+ }
+
// permission check
TSStatus status = AuthorityChecker.checkAuthority(s, SESSION_MANAGER.getCurrSession());
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {