You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/03/08 02:03:49 UTC
(iotdb) branch master updated: [IOTDB-6304] Support EXPLAN ANALYZE in query
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 279f47e372e [IOTDB-6304] Support EXPLAN ANALYZE in query
279f47e372e is described below
commit 279f47e372e85c6596e29ff11c4089e77cc7dc70
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Fri Mar 8 10:03:43 2024 +0800
[IOTDB-6304] Support EXPLAN ANALYZE in query
---
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +
.../assembly/resources/conf/logback-datanode.xml | 18 +
.../mpp/FragmentInstanceFetchException.java | 35 ++
.../impl/DataNodeInternalRPCServiceImpl.java | 20 ++
.../db/queryengine/common/MPPQueryContext.java | 103 ++++++
.../common/header/ColumnHeaderConstant.java | 1 +
.../fragment/FragmentInstanceContext.java | 76 ++++
.../fragment/FragmentInstanceExecution.java | 178 ++++++++-
.../fragment/FragmentInstanceManager.java | 28 ++
.../execution/fragment/QueryStatistics.java | 38 ++
.../execution/operator/ExplainAnalyzeOperator.java | 214 +++++++++++
.../queryengine/execution/operator/Operator.java | 15 +-
.../execution/operator/OperatorContext.java | 48 +++
.../operator/process/AbstractIntoOperator.java | 2 +-
.../operator/process/ColumnInjectOperator.java | 2 +-
.../operator/process/DeviceViewOperator.java | 2 +-
.../operator/process/FilterAndProjectOperator.java | 4 +
.../execution/operator/process/LimitOperator.java | 2 +-
.../operator/process/MergeSortOperator.java | 3 +-
.../execution/operator/process/OffsetOperator.java | 2 +-
.../operator/process/SingleDeviceViewOperator.java | 2 +-
.../execution/operator/process/SortOperator.java | 37 +-
.../execution/operator/process/TopKOperator.java | 2 +-
.../process/join/FullOuterTimeJoinOperator.java | 3 +-
.../process/join/HorizontallyConcatOperator.java | 3 +-
.../process/join/InnerTimeJoinOperator.java | 3 +-
.../process/join/LeftOuterTimeJoinOperator.java | 3 +-
.../operator/sink/IdentitySinkOperator.java | 6 +-
.../operator/sink/ShuffleHelperOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 2 +-
.../operator/source/SeriesScanOperator.java | 2 +-
.../execution/schedule/DriverScheduler.java | 19 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 1 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 41 ++-
.../queryengine/plan/execution/QueryExecution.java | 17 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 10 +-
.../plan/planner/LocalExecutionPlanner.java | 2 +-
.../plan/planner/LogicalPlanVisitor.java | 17 +
.../queryengine/plan/planner/LogicalPlanner.java | 7 +-
.../plan/planner/OperatorTreeGenerator.java | 17 +
.../planner/distribution/ExchangeNodeAdder.java | 18 +
.../SimpleFragmentParallelPlanner.java | 3 +
.../plan/planner/plan/FragmentInstance.java | 12 +
.../plan/planner/plan/PlanFragment.java | 4 +
.../plan/planner/plan/node/ExplainAnalyzeNode.java | 98 +++++
.../plan/planner/plan/node/PlanNodeType.java | 6 +-
.../plan/planner/plan/node/PlanVisitor.java | 4 +
.../scheduler/FragmentInstanceDispatcherImpl.java | 8 +-
.../plan/statement/StatementVisitor.java | 5 +
.../statement/sys/ExplainAnalyzeStatement.java | 60 ++++
.../FragmentInstanceStatisticsDrawer.java | 398 +++++++++++++++++++++
.../statistics/QueryPlanStatistics.java | 77 ++++
.../statistics/QueryStatisticsFetcher.java | 97 +++++
.../statistics/SpecifiedInfoMergerFactory.java | 54 +++
.../db/queryengine/statistics/StatisticLine.java | 38 ++
.../statistics/StatisticsMergeUtil.java | 94 +++++
.../apache/iotdb/db/utils/sort/DiskSpiller.java | 4 +
.../fragment/FragmentInstanceExecutionTest.java | 1 +
.../execution/operator/OperatorMemoryTest.java | 8 +-
.../schedule/DefaultDriverSchedulerTest.java | 23 ++
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../src/main/thrift/datanode.thrift | 85 +++++
65 files changed, 2034 insertions(+), 63 deletions(-)
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 19a2a282764..00ee303d4aa 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -112,6 +112,7 @@ public enum TSStatusCode {
INTERNAL_REQUEST_RETRY_ERROR(713),
NO_SUCH_QUERY(714),
QUERY_WAS_KILLED(715),
+ EXPLAIN_ANALYZE_FETCH_ERROR(716),
// Authentication
INIT_AUTH_ERROR(800),
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
index 2b3a3ebca0f..3108f6be07f 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
@@ -39,6 +39,7 @@ keyWords
| ALIGNED
| ALL
| ALTER
+ | ANALYZE
| AND
| ANY
| APPEND
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 47e798b80c2..2b74137554c 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -961,7 +961,7 @@ stopRepairData
// Explain
explain
- : EXPLAIN selectStatement
+ : EXPLAIN (ANALYZE VERBOSE?)? selectStatement?
;
// Set System To readonly/running/error
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index 94906afb3fc..5490806caf2 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -73,6 +73,10 @@ ALTER
: A L T E R
;
+ANALYZE
+ : A N A L Y Z E
+ ;
+
AND
: A N D
;
@@ -870,6 +874,10 @@ VARIATION
: V A R I A T I O N
;
+VERBOSE
+ : V E R B O S E
+ ;
+
VERIFY
: V E R I F Y
;
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
index 5e14a9fc6ea..4836791625f 100644
--- a/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
+++ b/iotdb-core/datanode/src/assembly/resources/conf/logback-datanode.xml
@@ -222,6 +222,21 @@
<level>INFO</level>
</filter>
</appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="EXPLAIN_ANALYZE">
+ <file>${IOTDB_HOME}/logs/log_explain_analyze.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_HOME}/logs/log-datanode-explain-%d{yyyyMMdd}.log.gz</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
<root level="info">
<appender-ref ref="FILETRACE"/>
<appender-ref ref="FILEDEBUG"/>
@@ -261,4 +276,7 @@
<logger level="info" name="org.apache.iotdb.commons.pipe">
<appender-ref ref="PIPE"/>
</logger>
+ <logger level="info" name="EXPLAIN_ANALYZE">
+ <appender-ref ref="EXPLAIN_ANALYZE"/>
+ </logger>
</configuration>
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/mpp/FragmentInstanceFetchException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/mpp/FragmentInstanceFetchException.java
new file mode 100644
index 00000000000..e84fbd73083
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/mpp/FragmentInstanceFetchException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.exception.mpp;
+
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+
+public class FragmentInstanceFetchException extends Exception {
+ private final TSStatus failureStatus;
+
+ public FragmentInstanceFetchException(TSStatus failureStatus) {
+ super(failureStatus.getMessage());
+ this.failureStatus = failureStatus;
+ }
+
+ public TSStatus getFailureStatus() {
+ return failureStatus;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index a0dab7b8e9a..4a027202321 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -166,6 +166,8 @@ import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TExecuteCQ;
import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceInfoReq;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsReq;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListReq;
import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListResp;
import org.apache.iotdb.mpp.rpc.thrift.TFireTriggerReq;
@@ -1124,6 +1126,24 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
return throttleQuotaManager.setThrottleQuota(req);
}
+ @Override
+ public TFetchFragmentInstanceStatisticsResp fetchFragmentInstanceStatistics(
+ TFetchFragmentInstanceStatisticsReq req) throws TException {
+ FragmentInstanceManager fragmentInstanceManager = FragmentInstanceManager.getInstance();
+ TFetchFragmentInstanceStatisticsResp resp;
+ try {
+ resp =
+ fragmentInstanceManager.getFragmentInstanceStatistics(
+ FragmentInstanceId.fromThrift(req.getFragmentInstanceId()));
+ resp.setStatus(RpcUtils.SUCCESS_STATUS);
+ } catch (Exception e) {
+ resp = new TFetchFragmentInstanceStatisticsResp();
+ resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXPLAIN_ANALYZE_FETCH_ERROR, e.getMessage()));
+ LOGGER.error(e.getMessage());
+ }
+ return resp;
+ }
+
private PathPatternTree filterPathPatternTree(PathPatternTree patternTree, String storageGroup) {
PathPatternTree filteredPatternTree = new PathPatternTree();
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
index 56e0fb0628d..4d7e5a4512f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider;
+import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import java.time.ZoneId;
@@ -38,6 +39,10 @@ import java.util.List;
public class MPPQueryContext {
private String sql;
private QueryId queryId;
+
+ // LocalQueryId is kept to adapt to the old client, it's unique in current datanode.
+ // Now it's only be used by EXPLAIN ANALYZE to get queryExecution.
+ private long localQueryId;
private SessionInfo session;
private QueryType queryType = QueryType.READ;
private long timeOut;
@@ -62,6 +67,10 @@ public class MPPQueryContext {
private int acquiredLockNum;
+ private boolean isExplainAnalyze = false;
+
+ QueryPlanStatistics queryPlanStatistics = null;
+
public MPPQueryContext(QueryId queryId) {
this.queryId = queryId;
this.endPointBlackList = new LinkedList<>();
@@ -82,6 +91,22 @@ public class MPPQueryContext {
this.initResultNodeContext();
}
+ public MPPQueryContext(
+ String sql,
+ QueryId queryId,
+ long localQueryId,
+ SessionInfo session,
+ TEndPoint localDataBlockEndpoint,
+ TEndPoint localInternalEndpoint) {
+ this(queryId);
+ this.sql = sql;
+ this.session = session;
+ this.localQueryId = localQueryId;
+ this.localDataBlockEndpoint = localDataBlockEndpoint;
+ this.localInternalEndpoint = localInternalEndpoint;
+ this.initResultNodeContext();
+ }
+
public void prepareForRetry() {
this.initResultNodeContext();
}
@@ -94,6 +119,10 @@ public class MPPQueryContext {
return queryId;
}
+ public long getLocalQueryId() {
+ return localQueryId;
+ }
+
public QueryType getQueryType() {
return queryType;
}
@@ -179,4 +208,78 @@ public class MPPQueryContext {
public ZoneId getZoneId() {
return session.getZoneId();
}
+
+ public void setExplainAnalyze(boolean explainAnalyze) {
+ isExplainAnalyze = explainAnalyze;
+ }
+
+ public boolean isExplainAnalyze() {
+ return isExplainAnalyze;
+ }
+
+ public long getAnalyzeCost() {
+ return queryPlanStatistics.getAnalyzeCost();
+ }
+
+ public long getDistributionPlanCost() {
+ return queryPlanStatistics.getDistributionPlanCost();
+ }
+
+ public long getFetchPartitionCost() {
+ return queryPlanStatistics.getFetchPartitionCost();
+ }
+
+ public long getFetchSchemaCost() {
+ return queryPlanStatistics.getFetchSchemaCost();
+ }
+
+ public long getLogicalPlanCost() {
+ return queryPlanStatistics.getLogicalPlanCost();
+ }
+
+ public long getLogicalOptimizationCost() {
+ return queryPlanStatistics.getLogicalOptimizationCost();
+ }
+
+ public void setAnalyzeCost(long analyzeCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setAnalyzeCost(analyzeCost);
+ }
+
+ public void setDistributionPlanCost(long distributionPlanCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setDistributionPlanCost(distributionPlanCost);
+ }
+
+ public void setFetchPartitionCost(long fetchPartitionCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setFetchPartitionCost(fetchPartitionCost);
+ }
+
+ public void setFetchSchemaCost(long fetchSchemaCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setFetchSchemaCost(fetchSchemaCost);
+ }
+
+ public void setLogicalPlanCost(long logicalPlanCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setLogicalPlanCost(logicalPlanCost);
+ }
+
+ public void setLogicalOptimizationCost(long logicalOptimizeCost) {
+ if (queryPlanStatistics == null) {
+ queryPlanStatistics = new QueryPlanStatistics();
+ }
+ queryPlanStatistics.setLogicalOptimizationCost(logicalOptimizeCost);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
index 4192821860b..1d1c3aa3e97 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
@@ -36,6 +36,7 @@ public class ColumnHeaderConstant {
public static final String ENDTIME = "__endTime";
public static final String VALUE = "Value";
public static final String DEVICE = "Device";
+ public static final String EXPLAIN_ANALYZE = "Explain Analyze";
// column names for schema statement
public static final String DATABASE = "Database";
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index cc92266242b..8c74170cef3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.slf4j.Logger;
@@ -92,6 +93,18 @@ public class FragmentInstanceContext extends QueryContext {
private final Map<QueryId, DataNodeQueryContext> dataNodeQueryContextMap;
private DataNodeQueryContext dataNodeQueryContext;
+ // Used for EXPLAIN ANALYZE to cache statistics result when the FI is finished,
+ // it will not be released until it's fetched.
+ private TFetchFragmentInstanceStatisticsResp fragmentInstanceStatistics = null;
+
+ private long initQueryDataSourceCost = 0;
+ private final AtomicLong readyQueueTime = new AtomicLong(0);
+ private final AtomicLong blockQueueTime = new AtomicLong(0);
+ private long unclosedSeqFileNum = 0;
+ private long unclosedUnseqFileNum = 0;
+ private long closedSeqFileNum = 0;
+ private long closedUnseqFileNum = 0;
+
public static FragmentInstanceContext createFragmentInstanceContext(
FragmentInstanceId id, FragmentInstanceStateMachine stateMachine, SessionInfo sessionInfo) {
FragmentInstanceContext instanceContext =
@@ -282,6 +295,10 @@ public class FragmentInstanceContext extends QueryContext {
return executionEndTime.get();
}
+ public boolean isEndTimeUpdate() {
+ return executionEndTime.get() != END_TIME_INITIAL_VALUE;
+ }
+
@Override
public long getStartTime() {
return executionStartTime.get();
@@ -325,6 +342,7 @@ public class FragmentInstanceContext extends QueryContext {
}
public void initQueryDataSource(List<PartialPath> sourcePaths) throws QueryProcessException {
+ long startTime = System.nanoTime();
if (sourcePaths == null) {
return;
}
@@ -356,6 +374,7 @@ public class FragmentInstanceContext extends QueryContext {
addUsedFilesForQuery(sharedQueryDataSource);
}
} finally {
+ setInitQueryDataSourceCost(System.nanoTime() - startTime);
dataRegion.readUnlock();
}
}
@@ -375,8 +394,16 @@ public class FragmentInstanceContext extends QueryContext {
// sequence data
addUsedFilesForQuery(dataSource.getSeqResources());
+ // Record statistics of seqFiles
+ unclosedSeqFileNum = unClosedFilePaths.size();
+ closedSeqFileNum = closedFilePaths.size();
+
// unsequence data
addUsedFilesForQuery(dataSource.getUnseqResources());
+
+ // Record statistics of files of unseqFiles
+ unclosedUnseqFileNum = unClosedFilePaths.size() - unclosedSeqFileNum;
+ closedUnseqFileNum = closedFilePaths.size() - closedSeqFileNum;
}
private void addUsedFilesForQuery(List<TsFileResource> resources) {
@@ -545,4 +572,53 @@ public class FragmentInstanceContext extends QueryContext {
public void setTimePartitions(List<Long> timePartitions) {
this.timePartitions = timePartitions;
}
+
+ // Only used in EXPLAIN ANALYZE
+ public void setFragmentInstanceStatistics(TFetchFragmentInstanceStatisticsResp statistics) {
+ this.fragmentInstanceStatistics = statistics;
+ }
+
+ public TFetchFragmentInstanceStatisticsResp getFragmentInstanceStatistics() {
+ return fragmentInstanceStatistics;
+ }
+
+ public void setInitQueryDataSourceCost(long initQueryDataSourceCost) {
+ this.initQueryDataSourceCost = initQueryDataSourceCost;
+ }
+
+ public long getInitQueryDataSourceCost() {
+ return initQueryDataSourceCost;
+ }
+
+ public void addReadyQueuedTime(long time) {
+ readyQueueTime.addAndGet(time);
+ }
+
+ public void addBlockQueuedTime(long time) {
+ blockQueueTime.addAndGet(time);
+ }
+
+ public long getReadyQueueTime() {
+ return readyQueueTime.get();
+ }
+
+ public long getBlockQueueTime() {
+ return blockQueueTime.get();
+ }
+
+ public long getClosedSeqFileNum() {
+ return closedSeqFileNum;
+ }
+
+ public long getUnclosedUnseqFileNum() {
+ return unclosedUnseqFileNum;
+ }
+
+ public long getClosedUnseqFileNum() {
+ return closedUnseqFileNum;
+ }
+
+ public long getUnclosedSeqFileNum() {
+ return unclosedSeqFileNum;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecution.java
index 890d1098353..9f95813f72f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecution.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecution.java
@@ -27,18 +27,31 @@ import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException;
import org.apache.iotdb.db.queryengine.execution.driver.IDriver;
import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager;
import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink;
+import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import org.apache.iotdb.db.queryengine.execution.schedule.IDriverScheduler;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery;
+import org.apache.iotdb.db.storageengine.dataregion.VirtualDataRegion;
import org.apache.iotdb.db.utils.SetThreadName;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
+import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics;
import io.airlift.stats.CounterStat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import static java.util.Objects.requireNonNull;
import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceState.FAILED;
+import static org.apache.iotdb.db.queryengine.statistics.StatisticsMergeUtil.merge;
+import static org.apache.iotdb.db.queryengine.statistics.StatisticsMergeUtil.mergeAllOperatorStatistics;
+import static org.apache.iotdb.db.queryengine.statistics.StatisticsMergeUtil.mergeOperatorStatisticsIfDuplicate;
public class FragmentInstanceExecution {
@@ -58,6 +71,13 @@ public class FragmentInstanceExecution {
private final MPPDataExchangeManager exchangeManager;
+ // This lock is used to guarantee the atomicity of buildStatistics() and the set of
+ // statisticsRemoved,
+ // so that fetchStatistics() in FragmentInstanceManager won't get the statistics which was just
+ // cleared.
+ private final ReadWriteLock statisticsLock = new ReentrantReadWriteLock();
+ private boolean staticsRemoved = false;
+
@SuppressWarnings("squid:S107")
public static FragmentInstanceExecution createFragmentInstanceExecution(
IDriverScheduler scheduler,
@@ -68,12 +88,13 @@ public class FragmentInstanceExecution {
FragmentInstanceStateMachine stateMachine,
CounterStat failedInstances,
long timeOut,
+ boolean isExplainAnalyze,
MPPDataExchangeManager exchangeManager)
throws CpuNotEnoughException, MemoryNotEnoughException {
FragmentInstanceExecution execution =
new FragmentInstanceExecution(
instanceId, context, drivers, sinkHandle, stateMachine, timeOut, exchangeManager);
- execution.initialize(failedInstances, scheduler);
+ execution.initialize(failedInstances, scheduler, isExplainAnalyze);
scheduler.submitDrivers(instanceId.getQueryId(), drivers, timeOut, context.getSessionInfo());
return execution;
}
@@ -115,13 +136,146 @@ public class FragmentInstanceExecution {
return timeoutInMs;
}
+ public FragmentInstanceContext getFragmentInstanceContext() {
+ return context;
+ }
+
+ public List<IDriver> getDrivers() {
+ return drivers;
+ }
+
public FragmentInstanceStateMachine getStateMachine() {
return stateMachine;
}
+ // Fill Fragment-Level info for statistics
+ private boolean fillFragmentInstanceStatistics(
+ FragmentInstanceContext context, TFetchFragmentInstanceStatisticsResp statistics) {
+ statistics.setFragmentInstanceId(context.getId().toThrift());
+ statistics.setQueryStatistics(context.getQueryStatistics().toThrift());
+
+ IDataRegionForQuery dataRegionForQuery = context.getDataRegion();
+ if (dataRegionForQuery instanceof VirtualDataRegion) {
+ // We don't need to output the region having ExplainAnalyzeOperator only.
+ return false;
+ }
+ statistics.setDataRegion(((DataRegion) context.getDataRegion()).getDataRegionId());
+ statistics.setIp(IoTDBDescriptor.getInstance().getConfig().getAddressAndPort().ip);
+ statistics.setStartTimeInMS(context.getStartTime());
+ statistics.setEndTimeInMS(
+ context.isEndTimeUpdate() ? context.getEndTime() : System.currentTimeMillis());
+
+ statistics.setBlockQueuedTime(context.getBlockQueueTime());
+ statistics.setReadyQueuedTime(context.getReadyQueueTime());
+
+ statistics.setInitDataQuerySourceCost(context.getInitQueryDataSourceCost());
+
+ statistics.setSeqClosednNum(context.getClosedSeqFileNum());
+ statistics.setSeqUnclosedNum(context.getUnclosedSeqFileNum());
+ statistics.setUnseqClosedNum(context.getClosedUnseqFileNum());
+ statistics.setUnseqUnclosedNum(context.getUnclosedUnseqFileNum());
+ return true;
+ }
+
+ // Fill Operator-Level info for statistics
+ // Return needMerge to indicate if operatorStatistics in current fragmentInstance is merged.
+ private boolean fillFragmentInstanceStatistics(
+ List<OperatorContext> contexts,
+ Map<String, TOperatorStatistics> operatorStatisticsMap,
+ Map<String, Integer> operatorCoutMap,
+ Map<String, String> leadOverloadOperators,
+ boolean needMerge) {
+ for (OperatorContext operatorContext : contexts) {
+ TOperatorStatistics operatorStatistics = new TOperatorStatistics();
+ // some exchange operators don't have planNodeId
+ if (operatorContext.getPlanNodeId() == null) continue;
+
+ String operatorType = operatorContext.getOperatorType();
+ // If the operatorType is already overloaded, then merge all operatorStatistics with the
+ // leadOverloadOperator
+ if (needMerge) {
+ setOperatorStatistics(operatorStatistics, operatorContext);
+ if (leadOverloadOperators.containsKey(operatorType)) {
+ merge(
+ operatorStatisticsMap.get(leadOverloadOperators.get(operatorType)),
+ operatorStatistics);
+ } else {
+ String planNodeId = operatorContext.getPlanNodeId().toString();
+ operatorStatistics.setCount(1);
+ operatorStatistics.getSpecifiedInfo().clear();
+ leadOverloadOperators.put(operatorType, planNodeId);
+ operatorStatisticsMap.put(planNodeId, operatorStatistics);
+ }
+ } else {
+ setOperatorStatistics(operatorStatistics, operatorContext);
+ operatorStatisticsMap.put(operatorContext.getPlanNodeId().toString(), operatorStatistics);
+ operatorCoutMap.put(operatorType, operatorCoutMap.getOrDefault(operatorType, 0) + 1);
+ if (operatorCoutMap.get(operatorType) >= 10) {
+ needMerge = true;
+ // merge all the operatorStatistics with the overload type and remain only one in
+ // operatorStatisticsMap
+ mergeAllOperatorStatistics(operatorStatisticsMap, leadOverloadOperators);
+ }
+ }
+ }
+ return needMerge;
+ }
+
+ private void setOperatorStatistics(
+ TOperatorStatistics operatorStatistics, OperatorContext operatorContext) {
+ operatorStatistics.setPlanNodeId(operatorContext.getPlanNodeId().toString());
+ operatorStatistics.setOperatorType(operatorContext.getOperatorType());
+ operatorStatistics.setTotalExecutionTimeInNanos(operatorContext.getTotalExecutionTimeInNanos());
+ operatorStatistics.setNextCalledCount(operatorContext.getNextCalledCount());
+ operatorStatistics.setHasNextCalledCount(operatorContext.getHasNextCalledCount());
+ operatorStatistics.setInputRows(operatorContext.getInputRows());
+ operatorStatistics.setSpecifiedInfo(operatorContext.getSpecifiedInfo());
+ operatorStatistics.setMemoryUsage(operatorContext.getEstimatedMemorySize());
+ }
+
+ // Directly build statistics from FragmentInstanceExecution, which is still running.
+ public TFetchFragmentInstanceStatisticsResp buildStatistics() {
+ TFetchFragmentInstanceStatisticsResp statistics = new TFetchFragmentInstanceStatisticsResp();
+
+ boolean res = fillFragmentInstanceStatistics(context, statistics);
+ if (!res) {
+ return statistics;
+ }
+
+ Map<String, TOperatorStatistics> operatorStatisticsMap = new HashMap<>();
+ Map<String, Integer> operatorCountMap = new HashMap<>();
+ Map<String, String> leadOverloadOperators = new HashMap<>();
+ boolean merge = false;
+ // Currently, they should be the drivers for each pipeline
+ for (IDriver driver : drivers) {
+ merge =
+ fillFragmentInstanceStatistics(
+ driver.getDriverContext().getOperatorContexts(),
+ operatorStatisticsMap,
+ operatorCountMap,
+ leadOverloadOperators,
+ merge);
+ }
+
+ if (merge) {
+ Map<String, TOperatorStatistics> newOperatorStatisticsMap = new HashMap<>();
+ for (Map.Entry<String, String> entry : leadOverloadOperators.entrySet()) {
+ newOperatorStatisticsMap.put(entry.getValue(), operatorStatisticsMap.get(entry.getValue()));
+ }
+ statistics.setOperatorStatisticsMap(newOperatorStatisticsMap);
+ } else {
+ statistics.setOperatorStatisticsMap(operatorStatisticsMap);
+ }
+
+ mergeOperatorStatisticsIfDuplicate(statistics.getOperatorStatisticsMap());
+
+ return statistics;
+ }
+
// this is a separate method to ensure that the `this` reference is not leaked during construction
@SuppressWarnings("squid:S1181")
- private void initialize(CounterStat failedInstances, IDriverScheduler scheduler) {
+ private void initialize(
+ CounterStat failedInstances, IDriverScheduler scheduler, boolean isExplainAnalyze) {
requireNonNull(failedInstances, "failedInstances is null");
stateMachine.addStateChangeListener(
newState -> {
@@ -139,6 +293,14 @@ public class FragmentInstanceExecution {
failedInstances.update(1);
}
+ statisticsLock.writeLock().lock();
+ // Store statistics in context for EXPLAIN ANALYZE to avoid being released.
+ if (isExplainAnalyze) {
+ context.setFragmentInstanceStatistics(buildStatistics());
+ }
+ staticsRemoved = true;
+ statisticsLock.writeLock().unlock();
+
clearShuffleSinkHandle(newState);
// delete tmp file if exists
@@ -195,4 +357,16 @@ public class FragmentInstanceExecution {
}
}
}
+
+ public boolean isStaticsRemoved() {
+ return staticsRemoved;
+ }
+
+ public void lockStatistics() {
+ statisticsLock.readLock().lock();
+ }
+
+ public void unlockStatistics() {
+ statisticsLock.readLock().unlock();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
index f17d6d3926d..59b256c1802 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery;
import org.apache.iotdb.db.utils.SetThreadName;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
import io.airlift.stats.CounterStat;
import io.airlift.units.Duration;
@@ -178,6 +179,7 @@ public class FragmentInstanceManager {
stateMachine,
failedInstances,
instance.getTimeOut(),
+ instance.isExplainAnalyze(),
exchangeManager);
} catch (Throwable t) {
logger.warn("error when create FragmentInstanceExecution.", t);
@@ -250,6 +252,7 @@ public class FragmentInstanceManager {
stateMachine,
failedInstances,
instance.getTimeOut(),
+ false,
exchangeManager);
} catch (Throwable t) {
logger.warn("Execute error caused by ", t);
@@ -316,6 +319,31 @@ public class FragmentInstanceManager {
return context.getInstanceInfo();
}
+ public TFetchFragmentInstanceStatisticsResp getFragmentInstanceStatistics(
+ FragmentInstanceId instanceId) {
+ requireNonNull(instanceId, "instanceId is null");
+ // If the instance is still running, we directly get the statistics from instanceExecution
+ if (instanceExecution.containsKey(instanceId)) {
+ FragmentInstanceExecution fragmentInstanceExecution = instanceExecution.get(instanceId);
+ try {
+ fragmentInstanceExecution.lockStatistics();
+ if (!fragmentInstanceExecution.isStaticsRemoved()) {
+ return fragmentInstanceExecution.buildStatistics();
+ }
+ } finally {
+ fragmentInstanceExecution.unlockStatistics();
+ }
+ }
+ // If the instance has finished, we get the statistics which was cached in the instanceContext
+ // when instanceExecution was removed.
+ FragmentInstanceContext context = instanceContext.get(instanceId);
+ TFetchFragmentInstanceStatisticsResp statistics = context.getFragmentInstanceStatistics();
+ if (statistics == null) {
+ return null;
+ }
+ return statistics;
+ }
+
public CounterStat getFailedInstances() {
return failedInstances;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
index 06086b3a1c8..60cd84b111e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.queryengine.execution.fragment;
+import org.apache.iotdb.mpp.rpc.thrift.TQueryStatistics;
+
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -66,4 +68,40 @@ public class QueryStatistics {
public AtomicLong pageReadersDecodeNonAlignedDiskTime = new AtomicLong(0);
public AtomicLong pageReadersDecodeNonAlignedMemCount = new AtomicLong(0);
public AtomicLong pageReadersDecodeNonAlignedMemTime = new AtomicLong(0);
+
+ public TQueryStatistics toThrift() {
+ return new TQueryStatistics(
+ loadTimeSeriesMetadataDiskSeqCount.get(),
+ loadTimeSeriesMetadataDiskUnSeqCount.get(),
+ loadTimeSeriesMetadataMemSeqCount.get(),
+ loadTimeSeriesMetadataMemUnSeqCount.get(),
+ loadTimeSeriesMetadataAlignedDiskSeqCount.get(),
+ loadTimeSeriesMetadataAlignedDiskUnSeqCount.get(),
+ loadTimeSeriesMetadataAlignedMemSeqCount.get(),
+ loadTimeSeriesMetadataAlignedMemUnSeqCount.get(),
+ loadTimeSeriesMetadataDiskSeqTime.get(),
+ loadTimeSeriesMetadataDiskUnSeqTime.get(),
+ loadTimeSeriesMetadataMemSeqTime.get(),
+ loadTimeSeriesMetadataMemUnSeqTime.get(),
+ loadTimeSeriesMetadataAlignedDiskSeqTime.get(),
+ loadTimeSeriesMetadataAlignedDiskUnSeqTime.get(),
+ loadTimeSeriesMetadataAlignedMemSeqTime.get(),
+ loadTimeSeriesMetadataAlignedMemUnSeqTime.get(),
+ constructNonAlignedChunkReadersDiskCount.get(),
+ constructNonAlignedChunkReadersMemCount.get(),
+ constructAlignedChunkReadersDiskCount.get(),
+ constructAlignedChunkReadersMemCount.get(),
+ constructNonAlignedChunkReadersDiskTime.get(),
+ constructNonAlignedChunkReadersMemTime.get(),
+ constructAlignedChunkReadersDiskTime.get(),
+ constructAlignedChunkReadersMemTime.get(),
+ pageReadersDecodeAlignedDiskCount.get(),
+ pageReadersDecodeAlignedDiskTime.get(),
+ pageReadersDecodeAlignedMemCount.get(),
+ pageReadersDecodeAlignedMemTime.get(),
+ pageReadersDecodeNonAlignedDiskCount.get(),
+ pageReadersDecodeNonAlignedDiskTime.get(),
+ pageReadersDecodeNonAlignedMemCount.get(),
+ pageReadersDecodeNonAlignedMemTime.get());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java
new file mode 100644
index 00000000000..ae8fb9ccdcc
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/ExplainAnalyzeOperator.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.operator;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.exception.mpp.FragmentInstanceFetchException;
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator;
+import org.apache.iotdb.db.queryengine.plan.Coordinator;
+import org.apache.iotdb.db.queryengine.plan.execution.QueryExecution;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
+import org.apache.iotdb.db.queryengine.statistics.FragmentInstanceStatisticsDrawer;
+import org.apache.iotdb.db.queryengine.statistics.QueryStatisticsFetcher;
+import org.apache.iotdb.db.queryengine.statistics.StatisticLine;
+import org.apache.iotdb.db.utils.SetThreadName;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
+import org.apache.iotdb.tsfile.utils.Binary;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+public class ExplainAnalyzeOperator implements ProcessOperator {
+ private final OperatorContext operatorContext;
+ private final Operator child;
+ private final boolean verbose;
+ private boolean outputResult = false;
+ private final List<FragmentInstance> instances;
+ private static final long LOG_INTERNAL_IN_MS = 10000;
+ private static final Logger logger =
+ LoggerFactory.getLogger(IoTDBConstant.EXPLAIN_ANALYZE_LOGGER_NAME);
+ private final FragmentInstanceStatisticsDrawer fragmentInstanceStatisticsDrawer =
+ new FragmentInstanceStatisticsDrawer();
+ private static final String LOG_TITLE =
+ "---------------------Intermediate result of EXPLAIN ANALYZE---------------------:";
+ private final ScheduledFuture<?> logRecordTask;
+ private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> clientManager;
+
+ public ExplainAnalyzeOperator(
+ OperatorContext operatorContext, Operator child, long queryId, boolean verbose) {
+ this.operatorContext = operatorContext;
+ this.child = child;
+ this.verbose = verbose;
+ Coordinator coordinator = Coordinator.getInstance();
+
+ this.clientManager = coordinator.getInternalServiceClientManager();
+
+ QueryExecution queryExecution = (QueryExecution) coordinator.getQueryExecution(queryId);
+ this.instances = queryExecution.getDistributedPlan().getInstances();
+ fragmentInstanceStatisticsDrawer.renderPlanStatistics(queryExecution.getContext());
+ this.logRecordTask =
+ ScheduledExecutorUtil.safelyScheduleAtFixedRate(
+ queryExecution.getScheduledExecutor(),
+ this::logIntermediateResultIfTimeout,
+ LOG_INTERNAL_IN_MS,
+ LOG_INTERNAL_IN_MS,
+ TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public OperatorContext getOperatorContext() {
+ return operatorContext;
+ }
+
+ @Override
+ public TsBlock next() throws Exception {
+ if (child.hasNextWithTimer()) {
+ child.nextWithTimer();
+ return null;
+ }
+
+ // fetch statics from all fragment instances
+ TsBlock result = buildResult();
+ outputResult = true;
+ return result;
+ }
+
+ private List<String> buildFragmentInstanceStatistics(
+ List<FragmentInstance> instances, boolean verbose) throws FragmentInstanceFetchException {
+
+ Map<FragmentInstanceId, TFetchFragmentInstanceStatisticsResp> allStatistics =
+ QueryStatisticsFetcher.fetchAllStatistics(instances, clientManager);
+ List<StatisticLine> statisticLines =
+ fragmentInstanceStatisticsDrawer.renderFragmentInstances(instances, allStatistics, verbose);
+
+ List<String> analyzeResult = new ArrayList<>();
+ for (StatisticLine line : statisticLines) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(line.getValue());
+ for (int i = 0;
+ i < fragmentInstanceStatisticsDrawer.getMaxLineLength() - line.getValue().length();
+ i++) {
+ sb.append(" ");
+ }
+ analyzeResult.add(sb.toString());
+ }
+ return analyzeResult;
+ }
+
+ // We will log the intermediate result of analyze if timeout
+ // It can be used to analyze deadlock problem.
+ private void logIntermediateResultIfTimeout() {
+ try (SetThreadName ignored =
+ new SetThreadName(
+ String.format(
+ "%s-Explain-Analyze-Logger",
+ operatorContext.getInstanceContext().getId().getQueryId()))) {
+ List<String> analyzeResult = buildFragmentInstanceStatistics(instances, verbose);
+
+ StringBuilder logContent = new StringBuilder();
+ logContent.append("\n").append(LOG_TITLE).append("\n");
+ for (String line : analyzeResult) {
+ logContent.append(line).append("\n");
+ }
+ String res = logContent.toString();
+ logger.info(res);
+ } catch (Exception e) {
+ logger.error("Error occurred when logging intermediate result of analyze.", e);
+ }
+ }
+
+ private TsBlock buildResult() throws FragmentInstanceFetchException {
+
+ List<String> analyzeResult = buildFragmentInstanceStatistics(instances, verbose);
+
+ TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(TSDataType.TEXT));
+ TimeColumnBuilder timeColumnBuilder = builder.getTimeColumnBuilder();
+ ColumnBuilder columnBuilder = builder.getColumnBuilder(0);
+
+ for (String line : analyzeResult) {
+ timeColumnBuilder.writeLong(0);
+ columnBuilder.writeBinary(new Binary(line.getBytes()));
+ builder.declarePosition();
+ }
+ return builder.build();
+ }
+
+ @Override
+ public boolean hasNext() throws Exception {
+ return child.hasNext() || !outputResult;
+ }
+
+ @Override
+ public ListenableFuture<?> isBlocked() {
+ return child.isBlocked();
+ }
+
+ @Override
+ public void close() throws Exception {
+ child.close();
+
+ if (logRecordTask != null) {
+ boolean cancelResult = logRecordTask.cancel(true);
+ if (!cancelResult) {
+ logger.debug("cancel state tracking task failed. {}", logRecordTask.isCancelled());
+ }
+ } else {
+ logger.debug("trackTask not started");
+ }
+ }
+
+ @Override
+ public boolean isFinished() throws Exception {
+ return !child.hasNext() && outputResult;
+ }
+
+ @Override
+ public long calculateMaxPeekMemory() {
+ return 0;
+ }
+
+ @Override
+ public long calculateMaxReturnSize() {
+ return 0;
+ }
+
+ @Override
+ public long calculateRetainedSizeAfterCallingNext() {
+ return 0;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/Operator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/Operator.java
index e7a2d3d08ee..26665559721 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/Operator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/Operator.java
@@ -43,13 +43,17 @@ public interface Operator extends AutoCloseable {
default TsBlock nextWithTimer() throws Exception {
OperatorContext context = getOperatorContext();
long startTime = System.nanoTime();
-
+ TsBlock input = null;
try {
- return next();
+ input = next();
} finally {
context.recordExecutionTime(System.nanoTime() - startTime);
+ if (input != null) {
+ context.addInputRows(input.getPositionCount());
+ }
context.recordNextCalled();
}
+ return input;
}
/**
@@ -68,9 +72,16 @@ public interface Operator extends AutoCloseable {
return hasNext();
} finally {
context.recordExecutionTime(System.nanoTime() - startTime);
+ context.recordHasNextCalled();
}
}
+ default long calculateMaxPeekMemoryWithCounter() {
+ long maxPeekMemory = calculateMaxPeekMemory();
+ getOperatorContext().setEstimatedMemorySize(maxPeekMemory);
+ return maxPeekMemory;
+ }
+
/**
* return true if the operator has more data, otherwise false.
*
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorContext.java
index 709e34fb0eb..64bb52259e9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorContext.java
@@ -28,6 +28,8 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import io.airlift.units.Duration;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
@@ -51,6 +53,13 @@ public class OperatorContext {
private long totalExecutionTimeInNanos = 0L;
private long nextCalledCount = 0L;
+ private long hasNextCalledCount = 0L;
+
+ // SpecifiedInfo is used to record some custom information for the operator,
+ // which will be shown in the result of EXPLAIN ANALYZE to analyze the query.
+ private Map<String, String> specifiedInfo = null;
+ private long inputRows = 0;
+ private long estimatedMemorySize;
public OperatorContext(
int operatorId, PlanNodeId planNodeId, String operatorType, DriverContext driverContext) {
@@ -104,6 +113,10 @@ public class OperatorContext {
return getInstanceContext().getSessionInfo();
}
+ public PlanNodeId getPlanNodeId() {
+ return planNodeId;
+ }
+
public void recordExecutionTime(long executionTimeInNanos) {
this.totalExecutionTimeInNanos += executionTimeInNanos;
}
@@ -112,6 +125,10 @@ public class OperatorContext {
this.nextCalledCount++;
}
+ public void recordHasNextCalled() {
+ this.hasNextCalledCount++;
+ }
+
public long getTotalExecutionTimeInNanos() {
return totalExecutionTimeInNanos;
}
@@ -120,6 +137,37 @@ public class OperatorContext {
return nextCalledCount;
}
+ public long getHasNextCalledCount() {
+ return hasNextCalledCount;
+ }
+
+ public void setEstimatedMemorySize(long estimatedMemorySize) {
+ this.estimatedMemorySize = estimatedMemorySize;
+ }
+
+ public long getEstimatedMemorySize() {
+ return estimatedMemorySize;
+ }
+
+ public void addInputRows(long inputRows) {
+ this.inputRows += inputRows;
+ }
+
+ public long getInputRows() {
+ return inputRows;
+ }
+
+ public void recordSpecifiedInfo(String key, String value) {
+ if (specifiedInfo == null) {
+ specifiedInfo = new HashMap<>();
+ }
+ specifiedInfo.put(key, value);
+ }
+
+ public Map<String, String> getSpecifiedInfo() {
+ return specifiedInfo == null ? new HashMap<>() : specifiedInfo;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractIntoOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractIntoOperator.java
index e60bcbf2e64..17e1122bdc1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractIntoOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AbstractIntoOperator.java
@@ -324,7 +324,7 @@ public abstract class AbstractIntoOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
- return maxReturnSize + maxRetainedSize + child.calculateMaxPeekMemory();
+ return maxReturnSize + maxRetainedSize + child.calculateMaxPeekMemoryWithCounter();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ColumnInjectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ColumnInjectOperator.java
index 1b3be180a66..8f57bd5414b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ColumnInjectOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ColumnInjectOperator.java
@@ -100,7 +100,7 @@ public class ColumnInjectOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
- return child.calculateMaxPeekMemory() + maxExtraColumnSize;
+ return child.calculateMaxPeekMemoryWithCounter() + maxExtraColumnSize;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/DeviceViewOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/DeviceViewOperator.java
index 0dd6329ac40..e05076368b0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/DeviceViewOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/DeviceViewOperator.java
@@ -168,7 +168,7 @@ public class DeviceViewOperator implements ProcessOperator {
public long calculateMaxPeekMemory() {
long maxPeekMemory = calculateMaxReturnSize() + calculateRetainedSizeAfterCallingNext();
for (Operator child : deviceOperators) {
- maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter());
}
return maxPeekMemory;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java
index fc281fe843a..f62658c68de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/FilterAndProjectOperator.java
@@ -107,7 +107,11 @@ public class FilterAndProjectOperator implements ProcessOperator {
return getTransformedTsBlock(input);
}
+ long inputRowCount = input.getPositionCount();
TsBlock filterResult = getFilterTsBlock(input);
+ long filteredRowCount =
+ filterResult == null ? inputRowCount : inputRowCount - filterResult.getPositionCount();
+ operatorContext.recordSpecifiedInfo("Filtered Rows", Long.toString(filteredRowCount));
// contains non-mappable udf, we leave calculation for TransformOperator
if (hasNonMappableUDF) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/LimitOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/LimitOperator.java
index a961de485df..8393f798835 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/LimitOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/LimitOperator.java
@@ -84,7 +84,7 @@ public class LimitOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
- return child.calculateMaxPeekMemory();
+ return child.calculateMaxPeekMemoryWithCounter();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/MergeSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/MergeSortOperator.java
index cd3ec0711b7..e20a13d025f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/MergeSortOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/MergeSortOperator.java
@@ -61,6 +61,7 @@ public class MergeSortOperator extends AbstractConsumeAllOperator {
this.comparator = comparator;
this.noMoreTsBlocks = new boolean[inputOperatorsCount];
this.tsBlockBuilder = new TsBlockBuilder(dataTypes);
+ operatorContext.recordSpecifiedInfo("Merge sort branches", String.valueOf(inputOperatorsCount));
}
@Override
@@ -191,7 +192,7 @@ public class MergeSortOperator extends AbstractConsumeAllOperator {
maxPeekMemory += operator.calculateRetainedSizeAfterCallingNext();
}
for (Operator operator : children) {
- maxPeekMemory = Math.max(maxPeekMemory, operator.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, operator.calculateMaxPeekMemoryWithCounter());
}
return Math.max(maxPeekMemory, calculateMaxReturnSize());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/OffsetOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/OffsetOperator.java
index 6b97e82978f..575369ca8fb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/OffsetOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/OffsetOperator.java
@@ -85,7 +85,7 @@ public class OffsetOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
- return child.calculateMaxPeekMemory();
+ return child.calculateMaxPeekMemoryWithCounter();
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SingleDeviceViewOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SingleDeviceViewOperator.java
index 214c33a4322..cf53e007e7c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SingleDeviceViewOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SingleDeviceViewOperator.java
@@ -123,7 +123,7 @@ public class SingleDeviceViewOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
long maxPeekMemory = calculateMaxReturnSize() + calculateRetainedSizeAfterCallingNext();
- maxPeekMemory = Math.max(maxPeekMemory, deviceOperator.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, deviceOperator.calculateMaxPeekMemoryWithCounter());
return maxPeekMemory;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
index 247f4861c1f..13a27bbd51f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/SortOperator.java
@@ -75,6 +75,10 @@ public class SortOperator implements ProcessOperator {
private final int maxReturnSize =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();
+ private long prepareUntilReadyCost = 0;
+ private long dataSize = 0;
+ private long sortCost = 0;
+
public SortOperator(
OperatorContext operatorContext,
Operator inputOperator,
@@ -104,7 +108,6 @@ public class SortOperator implements ProcessOperator {
@Override
public TsBlock next() throws Exception {
-
if (!inputOperator.hasNextWithTimer()) {
if (diskSpiller.hasSpilledData()) {
try {
@@ -116,33 +119,47 @@ public class SortOperator implements ProcessOperator {
}
} else {
if (curRow == -1) {
+ long startTime = System.nanoTime();
cachedData.sort(comparator);
+ sortCost += System.nanoTime() - startTime;
curRow = 0;
}
return buildTsBlockInMemory();
}
}
-
- TsBlock tsBlock = inputOperator.nextWithTimer();
- if (tsBlock == null) {
- return null;
- }
-
+ long startTime = System.nanoTime();
try {
+ TsBlock tsBlock = inputOperator.nextWithTimer();
+ if (tsBlock == null) {
+ return null;
+ }
+ dataSize += tsBlock.getRetainedSizeInBytes();
cacheTsBlock(tsBlock);
} catch (IoTDBException e) {
clear();
throw e;
+ } finally {
+ prepareUntilReadyCost += System.nanoTime() - startTime;
}
return null;
}
+ private void recordMetrics() {
+ operatorContext.recordSpecifiedInfo("prepareCost/ns", Long.toString(prepareUntilReadyCost));
+ operatorContext.recordSpecifiedInfo("sortedDataSize", Long.toString(dataSize));
+ operatorContext.recordSpecifiedInfo("sortCost/ns", Long.toString(sortCost));
+ int spilledFileSize = diskSpiller.getFileSize();
+ if (spilledFileSize > 0) {
+ operatorContext.recordSpecifiedInfo(
+ "merge sort branch", Integer.toString(diskSpiller.getFileSize() + 1));
+ }
+ }
+
private void prepareSortReaders() throws IoTDBException {
if (sortReaders != null) {
return;
}
-
sortReaders = new ArrayList<>();
if (cachedBytes != 0) {
cachedData.sort(comparator);
@@ -252,6 +269,7 @@ public class SortOperator implements ProcessOperator {
break;
}
}
+ sortCost += System.nanoTime() - startTime;
return tsBlockBuilder.build();
}
@@ -318,6 +336,7 @@ public class SortOperator implements ProcessOperator {
@Override
public void close() throws Exception {
+ recordMetrics();
cachedData = null;
clear();
inputOperator.close();
@@ -330,7 +349,7 @@ public class SortOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
- return inputOperator.calculateMaxPeekMemory()
+ return inputOperator.calculateMaxPeekMemoryWithCounter()
+ inputOperator.calculateRetainedSizeAfterCallingNext()
+ SORT_BUFFER_SIZE;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
index 507e5ae3456..e1ae2c206cd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java
@@ -237,7 +237,7 @@ public class TopKOperator implements ProcessOperator {
// so no need to accumulate the returnSize and retainedSize of each child
long maxPeekMemory = calculateMaxReturnSize();
for (Operator operator : childrenOperators) {
- maxPeekMemory = Math.max(maxPeekMemory, operator.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, operator.calculateMaxPeekMemoryWithCounter());
}
return Math.max(maxPeekMemory, topValue * getMemoryUsageOfOneMergeSortKey() * 2);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/FullOuterTimeJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/FullOuterTimeJoinOperator.java
index 2617e8dec97..a829225bed8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/FullOuterTimeJoinOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/FullOuterTimeJoinOperator.java
@@ -245,7 +245,8 @@ public class FullOuterTimeJoinOperator extends AbstractConsumeAllOperator {
long childrenMaxPeekMemory = 0;
for (Operator child : children) {
childrenMaxPeekMemory =
- Math.max(childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemory());
+ Math.max(
+ childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemoryWithCounter());
maxPeekMemory +=
(child.calculateMaxReturnSize() + child.calculateRetainedSizeAfterCallingNext());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/HorizontallyConcatOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/HorizontallyConcatOperator.java
index 89082d9b5d2..818263a5e6b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/HorizontallyConcatOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/HorizontallyConcatOperator.java
@@ -130,7 +130,8 @@ public class HorizontallyConcatOperator extends AbstractConsumeAllOperator {
long childrenMaxPeekMemory = 0;
for (Operator child : children) {
childrenMaxPeekMemory =
- Math.max(childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemory());
+ Math.max(
+ childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemoryWithCounter());
maxPeekMemory +=
(child.calculateMaxReturnSize() + child.calculateRetainedSizeAfterCallingNext());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/InnerTimeJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/InnerTimeJoinOperator.java
index 64a75493b18..77cdedd6858 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/InnerTimeJoinOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/InnerTimeJoinOperator.java
@@ -369,7 +369,8 @@ public class InnerTimeJoinOperator implements ProcessOperator {
long childrenMaxPeekMemory = 0;
for (Operator child : children) {
childrenMaxPeekMemory =
- Math.max(childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemory());
+ Math.max(
+ childrenMaxPeekMemory, maxPeekMemory + child.calculateMaxPeekMemoryWithCounter());
maxPeekMemory +=
(child.calculateMaxReturnSize() + child.calculateRetainedSizeAfterCallingNext());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/LeftOuterTimeJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/LeftOuterTimeJoinOperator.java
index f9c40038672..9afef746f7b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/LeftOuterTimeJoinOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/LeftOuterTimeJoinOperator.java
@@ -320,7 +320,8 @@ public class LeftOuterTimeJoinOperator implements ProcessOperator {
@Override
public long calculateMaxPeekMemory() {
return Math.max(
- Math.max(left.calculateMaxPeekMemory(), right.calculateMaxPeekMemory()),
+ Math.max(
+ left.calculateMaxPeekMemoryWithCounter(), right.calculateMaxPeekMemoryWithCounter()),
calculateRetainedSizeAfterCallingNext() + calculateMaxReturnSize());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/IdentitySinkOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/IdentitySinkOperator.java
index 82d4550512f..b605d2aa039 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/IdentitySinkOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/IdentitySinkOperator.java
@@ -58,7 +58,7 @@ public class IdentitySinkOperator implements Operator {
public boolean hasNext() throws Exception {
int currentIndex = downStreamChannelIndex.getCurrentIndex();
boolean currentChannelClosed = sinkHandle.isChannelClosed(currentIndex);
- if (!currentChannelClosed && children.get(currentIndex).hasNext()) {
+ if (!currentChannelClosed && children.get(currentIndex).hasNextWithTimer()) {
return true;
} else if (currentChannelClosed) {
// we close the child directly. The child could be an ExchangeOperator which is the downstream
@@ -97,7 +97,7 @@ public class IdentitySinkOperator implements Operator {
needToReturnNull = false;
return null;
}
- return children.get(downStreamChannelIndex.getCurrentIndex()).next();
+ return children.get(downStreamChannelIndex.getCurrentIndex()).nextWithTimer();
}
@Override
@@ -129,7 +129,7 @@ public class IdentitySinkOperator implements Operator {
public long calculateMaxPeekMemory() {
long maxPeekMemory = 0;
for (Operator child : children) {
- maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter());
}
return maxPeekMemory;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/ShuffleHelperOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/ShuffleHelperOperator.java
index c18289c78c3..1ca32d32c84 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/ShuffleHelperOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/sink/ShuffleHelperOperator.java
@@ -136,7 +136,7 @@ public class ShuffleHelperOperator implements Operator {
public long calculateMaxPeekMemory() {
long maxPeekMemory = 0;
for (Operator child : children) {
- maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemory());
+ maxPeekMemory = Math.max(maxPeekMemory, child.calculateMaxPeekMemoryWithCounter());
}
return maxPeekMemory;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java
index 840f682b559..29dbcbd9d56 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanOperator.java
@@ -138,7 +138,7 @@ public class AlignedSeriesScanOperator extends AbstractDataSourceOperator {
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return calculateMaxPeekMemory() - calculateMaxReturnSize();
+ return calculateMaxPeekMemoryWithCounter() - calculateMaxReturnSize();
}
private boolean readFileData() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java
index a5d66c1d2d3..0e17309f00f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanOperator.java
@@ -114,7 +114,7 @@ public class SeriesScanOperator extends AbstractDataSourceOperator {
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return calculateMaxPeekMemory() - calculateMaxReturnSize();
+ return calculateMaxPeekMemoryWithCounter() - calculateMaxReturnSize();
}
private boolean readFileData() throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java
index e8ddb8161d5..09dab3d7819 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/schedule/DriverScheduler.java
@@ -474,9 +474,14 @@ public class DriverScheduler implements IDriverScheduler, IService {
}
task.setStatus(DriverTaskStatus.READY);
- DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(
- BLOCK_QUEUED_TIME, System.nanoTime() - task.getLastEnterBlockQueueTime());
- task.setLastEnterReadyQueueTime(System.nanoTime());
+ long currentTime = System.nanoTime();
+ long blockQueuedTime = currentTime - task.getLastEnterBlockQueueTime();
+ task.getDriver()
+ .getDriverContext()
+ .getFragmentInstanceContext()
+ .addBlockQueuedTime(blockQueuedTime);
+ DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(BLOCK_QUEUED_TIME, blockQueuedTime);
+ task.setLastEnterReadyQueueTime(currentTime);
task.resetLevelScheduledTime();
readyQueue.repush(task);
blockedTasks.remove(task);
@@ -494,8 +499,12 @@ public class DriverScheduler implements IDriverScheduler, IService {
}
task.setStatus(DriverTaskStatus.RUNNING);
- DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(
- READY_QUEUED_TIME, System.nanoTime() - task.getLastEnterReadyQueueTime());
+ long readyQueuedTime = System.nanoTime() - task.getLastEnterReadyQueueTime();
+ task.getDriver()
+ .getDriverContext()
+ .getFragmentInstanceContext()
+ .addReadyQueuedTime(readyQueuedTime);
+ DRIVER_SCHEDULER_METRIC_SET.recordTaskQueueTime(READY_QUEUED_TIME, readyQueuedTime);
} finally {
task.unlock();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
index 6c8c2d15bf9..327008e9c2a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
@@ -144,6 +144,7 @@ public class Coordinator {
new MPPQueryContext(
sql,
globalQueryId,
+ queryId,
session,
DataNodeEndPoints.LOCAL_HOST_DATA_BLOCK_ENDPOINT,
DataNodeEndPoints.LOCAL_HOST_INTERNAL_ENDPOINT);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 6ccdcedc7bc..d6b16b15087 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -132,6 +132,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ShowSche
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.CreateLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.ShowLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement;
@@ -233,6 +234,20 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return analysis;
}
+ @Override
+ public Analysis visitExplainAnalyze(
+ ExplainAnalyzeStatement explainAnalyzeStatement, MPPQueryContext context) {
+ Analysis analysis = visitQuery(explainAnalyzeStatement.getQueryStatement(), context);
+ context.setExplainAnalyze(true);
+ analysis.setStatement(explainAnalyzeStatement);
+ analysis.setRespDatasetHeader(
+ new DatasetHeader(
+ Collections.singletonList(
+ new ColumnHeader(ColumnHeaderConstant.EXPLAIN_ANALYZE, TSDataType.TEXT, null)),
+ true));
+ return analysis;
+ }
+
@Override
public Analysis visitQuery(QueryStatement queryStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
@@ -358,7 +373,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analyzeOutput(analysis, queryStatement, outputExpressions);
// fetch partition information
- analyzeDataPartition(analysis, queryStatement, schemaTree, context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, queryStatement, schemaTree, context);
} catch (StatementAnalyzeException e) {
throw new StatementAnalyzeException(
@@ -402,8 +417,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
updateSchemaTreeByViews(analysis, schemaTree, context);
} finally {
logger.debug("[EndFetchSchema]");
- QueryPlanCostMetricSet.getInstance()
- .recordPlanCost(SCHEMA_FETCHER, System.nanoTime() - startTime);
+ long schemaFetchCost = System.nanoTime() - startTime;
+ context.setFetchSchemaCost(schemaFetchCost);
+ QueryPlanCostMetricSet.getInstance().recordPlanCost(SCHEMA_FETCHER, schemaFetchCost);
}
analysis.setSchemaTree(schemaTree);
@@ -478,7 +494,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
analysis.setRespDatasetHeader(DatasetHeaderFactory.getLastQueryHeader());
// fetch partition information
- analyzeDataPartition(analysis, queryStatement, schemaTree, context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, queryStatement, schemaTree, context);
return analysis;
}
@@ -1892,7 +1908,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Analysis analysis,
QueryStatement queryStatement,
ISchemaTree schemaTree,
- Filter globalTimeFilter) {
+ MPPQueryContext context) {
Set<String> deviceSet = new HashSet<>();
if (queryStatement.isAlignByDevice()) {
deviceSet = new HashSet<>(analysis.getOutputDeviceToQueriedDevicesMap().values());
@@ -1901,17 +1917,16 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
deviceSet.add(ExpressionAnalyzer.getDeviceNameInSourceExpression(expression));
}
}
- DataPartition dataPartition =
- fetchDataPartitionByDevices(deviceSet, schemaTree, globalTimeFilter);
+ DataPartition dataPartition = fetchDataPartitionByDevices(deviceSet, schemaTree, context);
analysis.setDataPartitionInfo(dataPartition);
}
private DataPartition fetchDataPartitionByDevices(
- Set<String> deviceSet, ISchemaTree schemaTree, Filter globalTimeFilter) {
+ Set<String> deviceSet, ISchemaTree schemaTree, MPPQueryContext context) {
long startTime = System.nanoTime();
try {
Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
- getTimePartitionSlotList(globalTimeFilter);
+ getTimePartitionSlotList(context.getGlobalTimeFilter());
// there is no satisfied time range
if (res.left.isEmpty() && Boolean.FALSE.equals(res.right.left)) {
return new DataPartition(
@@ -1934,8 +1949,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return partitionFetcher.getDataPartition(sgNameToQueryParamsMap);
}
} finally {
- QueryPlanCostMetricSet.getInstance()
- .recordPlanCost(PARTITION_FETCHER, System.nanoTime() - startTime);
+ long partitionFetchCost = System.nanoTime() - startTime;
+ QueryPlanCostMetricSet.getInstance().recordPlanCost(PARTITION_FETCHER, partitionFetchCost);
+ context.setFetchPartitionCost(partitionFetchCost);
}
}
@@ -2768,8 +2784,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
Collections.singletonList(
new TimeSeriesOperand(showTimeSeriesStatement.getPathPattern())),
schemaTree);
- analyzeDataPartition(
- analysis, new QueryStatement(), schemaTree, context.getGlobalTimeFilter());
+ analyzeDataPartition(analysis, new QueryStatement(), schemaTree, context);
}
analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
index aa3bb22973b..99eb73e3425 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
@@ -310,7 +310,9 @@ public class QueryExecution implements IQueryExecution {
try {
result = new Analyzer(context, partitionFetcher, schemaFetcher).analyze(statement);
} finally {
- PERFORMANCE_OVERVIEW_METRICS.recordAnalyzeCost(System.nanoTime() - startTime);
+ long analyzeCost = System.nanoTime() - startTime;
+ context.setAnalyzeCost(analyzeCost);
+ PERFORMANCE_OVERVIEW_METRICS.recordAnalyzeCost(analyzeCost);
}
return result;
}
@@ -369,8 +371,9 @@ public class QueryExecution implements IQueryExecution {
this.distributedPlan = planner.planFragments();
if (rawStatement.isQuery()) {
- QUERY_PLAN_COST_METRIC_SET.recordPlanCost(
- DISTRIBUTION_PLANNER, System.nanoTime() - startTime);
+ long distributionPlanCost = System.nanoTime() - startTime;
+ context.setDistributionPlanCost(distributionPlanCost);
+ QUERY_PLAN_COST_METRIC_SET.recordPlanCost(DISTRIBUTION_PLANNER, distributionPlanCost);
}
// if is this Statement is ShowQueryStatement, set its instances to the highest priority, so
@@ -775,7 +778,15 @@ public class QueryExecution implements IQueryExecution {
return analysis.getStatement();
}
+ public MPPQueryContext getContext() {
+ return context;
+ }
+
public String toString() {
return String.format("QueryExecution[%s]", context.getQueryId());
}
+
+ public ScheduledExecutorService getScheduledExecutor() {
+ return scheduledExecutor;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index bae44158f2b..46d2690390e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -185,6 +185,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.DeleteLogica
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.ShowLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ClearCacheStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement;
@@ -2556,7 +2557,14 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
@Override
public Statement visitExplain(IoTDBSqlParser.ExplainContext ctx) {
QueryStatement queryStatement = (QueryStatement) visitSelectStatement(ctx.selectStatement());
- return new ExplainStatement(queryStatement);
+ if (ctx.ANALYZE() == null) {
+ return new ExplainStatement(queryStatement);
+ }
+ ExplainAnalyzeStatement explainAnalyzeStatement = new ExplainAnalyzeStatement(queryStatement);
+ if (ctx.VERBOSE() != null) {
+ explainAnalyzeStatement.setVerbose(true);
+ }
+ return explainAnalyzeStatement;
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index eff85a4b197..2e509a57999 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -125,7 +125,7 @@ public class LocalExecutionPlanner {
return 0;
}
- long estimatedMemorySize = root.calculateMaxPeekMemory();
+ long estimatedMemorySize = root.calculateMaxPeekMemoryWithCounter();
synchronized (this) {
if (estimatedMemorySize > freeMemoryForOperators) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
index aa6189c38fe..795dc85c4ce 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
@@ -24,12 +24,14 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
import org.apache.iotdb.db.queryengine.plan.expression.visitor.TransformToViewExpressionVisitor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.ExplainAnalyzeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode;
@@ -87,6 +89,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.ShowPath
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.CreateLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.ShowLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.db.schemaengine.template.Template;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -122,6 +125,20 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
"Unsupported statement type: " + node.getClass().getName());
}
+ @Override
+ public PlanNode visitExplainAnalyze(
+ ExplainAnalyzeStatement explainAnalyzeStatement, MPPQueryContext context) {
+ PlanNode root = visitQuery(explainAnalyzeStatement.getQueryStatement(), context);
+ root =
+ new ExplainAnalyzeNode(
+ context.getQueryId().genPlanNodeId(),
+ root,
+ explainAnalyzeStatement.isVerbose(),
+ context.getLocalQueryId());
+ context.getTypeProvider().setType(ColumnHeaderConstant.EXPLAIN_ANALYZE, TSDataType.TEXT);
+ return root;
+ }
+
@Override
public PlanNode visitQuery(QueryStatement queryStatement, MPPQueryContext context) {
if (analysis.isAllDevicesInOneTemplate()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
index 4c279b2b57d..5c41e242baa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanner.java
@@ -47,12 +47,15 @@ public class LogicalPlanner {
// optimize the query logical plan
if (analysis.getStatement().isQuery()) {
- QueryPlanCostMetricSet.getInstance()
- .recordPlanCost(LOGICAL_PLANNER, System.nanoTime() - startTime);
+ long planFinishTime = System.nanoTime();
+ QueryPlanCostMetricSet.getInstance()
+ .recordPlanCost(LOGICAL_PLANNER, System.nanoTime() - planFinishTime);
+ context.setLogicalPlanCost(planFinishTime - startTime);
for (PlanOptimizer optimizer : optimizers) {
rootNode = optimizer.optimize(rootNode, analysis, context);
}
+ context.setLogicalOptimizationCost(System.nanoTime() - planFinishTime);
}
return new LogicalQueryPlan(context, rootNode);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
index a7f9c512811..22ea2e1a311 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
@@ -43,6 +43,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle
import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle;
import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager;
import org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil;
+import org.apache.iotdb.db.queryengine.execution.operator.ExplainAnalyzeOperator;
import org.apache.iotdb.db.queryengine.execution.operator.Operator;
import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import org.apache.iotdb.db.queryengine.execution.operator.process.AggregationOperator;
@@ -151,6 +152,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand;
import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand;
import org.apache.iotdb.db.queryengine.plan.expression.visitor.ColumnTransformerVisitor;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.ExplainAnalyzeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
@@ -342,6 +344,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
context.getNextOperatorId(),
node.getPlanNodeId(),
SeriesScanOperator.class.getSimpleName());
+ operatorContext.recordSpecifiedInfo("SeriesPath", seriesPath.getFullPath());
SeriesScanOperator seriesScanOperator =
new SeriesScanOperator(
operatorContext,
@@ -3176,4 +3179,18 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
throw new UnsupportedOperationException("Unsupported column generator type: " + type);
}
}
+
+ @Override
+ public Operator visitExplainAnalyze(ExplainAnalyzeNode node, LocalExecutionPlanContext context) {
+ Operator operator = node.getChild().accept(this, context);
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ ExplainAnalyzeOperator.class.getSimpleName());
+ return new ExplainAnalyzeOperator(
+ operatorContext, operator, node.getQueryId(), node.isVerbose());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java
index 9c9d2d3e8eb..3ae7a3f126f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.distribution;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.commons.partition.DataPartition;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.ExplainAnalyzeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
@@ -380,6 +381,23 @@ public class ExchangeNodeAdder extends PlanVisitor<PlanNode, NodeGroupContext> {
return processOneChildNode(node, context);
}
+ @Override
+ public PlanNode visitExplainAnalyze(ExplainAnalyzeNode node, NodeGroupContext context) {
+ ExplainAnalyzeNode newNode = (ExplainAnalyzeNode) node.clone();
+
+ PlanNode child = newNode.getChild();
+ child = visit(child, context);
+
+ // todo judge if the child is in current region, if so, there is no need to add exchange node
+ ExchangeNode exchangeNode = new ExchangeNode(context.queryContext.getQueryId().genPlanNodeId());
+ exchangeNode.setChild(child);
+ exchangeNode.setOutputColumnNames(node.getChild().getOutputColumnNames());
+ newNode.setChild(exchangeNode);
+
+ context.hasExchangeNode = true;
+ return newNode;
+ }
+
private PlanNode processMultiChildNode(MultiChildProcessNode node, NodeGroupContext context) {
if (analysis.isVirtualSource()) {
return processMultiChildNodeByLocation(node, context);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
index 5d7bb04fdc3..711ff6c25cc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/SimpleFragmentParallelPlanner.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.MultiChildren
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.LastSeriesSourceNode;
import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -144,6 +145,7 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
queryContext.getQueryType(),
queryContext.getTimeOut(),
queryContext.getSession(),
+ queryContext.isExplainAnalyze(),
fragment.isRoot());
// Get the target region for origin PlanFragment, then its instance will be distributed one
@@ -184,6 +186,7 @@ public class SimpleFragmentParallelPlanner implements IFragmentParallelPlaner {
});
if (analysis.getStatement() instanceof QueryStatement
+ || analysis.getStatement() instanceof ExplainAnalyzeStatement
|| analysis.getStatement() instanceof ShowQueriesStatement
|| (analysis.getStatement() instanceof ShowTimeSeriesStatement
&& ((ShowTimeSeriesStatement) analysis.getStatement()).isOrderByHeat())) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
index ae83fd6623a..4c8204a2a22 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
@@ -76,6 +76,10 @@ public class FragmentInstance implements IConsensusRequest {
// indicate which index we are retrying
private transient int nextRetryIndex = 0;
+ // If this query is an EXPLAIN ANALYZE query
+ // We need to cache and calculate the statistics of this FragmentInstance if it is.
+ private boolean isExplainAnalyze = false;
+
// We can add some more params for a specific FragmentInstance
// So that we can make different FragmentInstance owns different data range.
@@ -102,9 +106,11 @@ public class FragmentInstance implements IConsensusRequest {
QueryType type,
long timeOut,
SessionInfo sessionInfo,
+ boolean isExplainAnalyze,
boolean isRoot) {
this(fragment, id, globalTimePredicate, type, timeOut, sessionInfo);
this.isRoot = isRoot;
+ this.isExplainAnalyze = isExplainAnalyze;
}
public FragmentInstance(
@@ -217,6 +223,7 @@ public class FragmentInstance implements IConsensusRequest {
boolean hasHostDataNode = ReadWriteIOUtils.readBool(buffer);
fragmentInstance.hostDataNode =
hasHostDataNode ? ThriftCommonsSerDeUtils.deserializeTDataNodeLocation(buffer) : null;
+ fragmentInstance.isExplainAnalyze = ReadWriteIOUtils.readBool(buffer);
return fragmentInstance;
}
@@ -240,6 +247,7 @@ public class FragmentInstance implements IConsensusRequest {
if (hostDataNode != null) {
ThriftCommonsSerDeUtils.serializeTDataNodeLocation(hostDataNode, outputStream);
}
+ ReadWriteIOUtils.write(isExplainAnalyze, outputStream);
return ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size());
} catch (IOException e) {
logger.error("Unexpected error occurs when serializing this FragmentInstance.", e);
@@ -284,4 +292,8 @@ public class FragmentInstance implements IConsensusRequest {
public SessionInfo getSessionInfo() {
return sessionInfo;
}
+
+ public boolean isExplainAnalyze() {
+ return isExplainAnalyze;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
index e063225b641..7583b624b64 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
@@ -201,6 +201,10 @@ public class PlanFragment {
typeProvider = null;
}
+ public void clearTypeProvider() {
+ typeProvider = null;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/ExplainAnalyzeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/ExplainAnalyzeNode.java
new file mode 100644
index 00000000000..14bb402df75
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/ExplainAnalyzeNode.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node;
+
+import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collections;
+import java.util.List;
+
+public class ExplainAnalyzeNode extends SingleChildProcessNode {
+ private final boolean verbose;
+
+ private final long queryId;
+
+ public ExplainAnalyzeNode(PlanNodeId id, PlanNode child, boolean verbose, long queryId) {
+ super(id, child);
+ this.verbose = verbose;
+ this.queryId = queryId;
+ }
+
+ @Override
+ public PlanNode clone() {
+ return new ExplainAnalyzeNode(getPlanNodeId(), child, verbose, queryId);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitExplainAnalyze(this, context);
+ }
+
+ @Override
+ public List<String> getOutputColumnNames() {
+ return Collections.singletonList(ColumnHeaderConstant.EXPLAIN_ANALYZE);
+ }
+
+ @Override
+ protected void serializeAttributes(ByteBuffer byteBuffer) {
+ PlanNodeType.EXPLAIN_ANALYZE.serialize(byteBuffer);
+ ReadWriteIOUtils.write(verbose, byteBuffer);
+ }
+
+ @Override
+ protected void serializeAttributes(DataOutputStream stream) throws IOException {
+ PlanNodeType.EXPLAIN_ANALYZE.serialize(stream);
+ ReadWriteIOUtils.write(verbose, stream);
+ ReadWriteIOUtils.write(queryId, stream);
+ }
+
+ public static ExplainAnalyzeNode deserialize(ByteBuffer byteBuffer) {
+ boolean verbose = ReadWriteIOUtils.readBool(byteBuffer);
+ long queryId = ReadWriteIOUtils.readLong(byteBuffer);
+ PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
+ return new ExplainAnalyzeNode(planNodeId, null, verbose, queryId);
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ public long getQueryId() {
+ return queryId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof ExplainAnalyzeNode)) return false;
+ ExplainAnalyzeNode that = (ExplainAnalyzeNode) o;
+ return verbose == that.verbose;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() + Boolean.hashCode(verbose);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
index 25551126818..ef66aa32ade 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
@@ -203,7 +203,9 @@ public enum PlanNodeType {
INNER_TIME_JOIN((short) 87),
LEFT_OUTER_TIME_JOIN((short) 88),
- AGG_MERGE_SORT((short) 89);
+ AGG_MERGE_SORT((short) 89),
+
+ EXPLAIN_ANALYZE((short) 90);
public static final int BYTES = Short.BYTES;
@@ -430,6 +432,8 @@ public enum PlanNodeType {
return LeftOuterTimeJoinNode.deserialize(buffer);
case 89:
return AggregationMergeSortNode.deserialize(buffer);
+ case 90:
+ return ExplainAnalyzeNode.deserialize(buffer);
default:
throw new IllegalArgumentException("Invalid node type: " + nodeType);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index 9ba1b3a09a7..d056131c0ad 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -498,4 +498,8 @@ public abstract class PlanVisitor<R, C> {
public R visitPipeEnrichedConfigSchema(PipeEnrichedConfigSchemaNode node, C context) {
return visitPlan(node, context);
}
+
+ public R visitExplainAnalyze(ExplainAnalyzeNode node, C context) {
+ return visitPlan(node, context);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
index 231118cc1f3..f671a87a1f9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
@@ -134,7 +134,13 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher {
} finally {
// friendly for gc, clear the plan node tree, for some queries select all devices, it will
// release lots of memory
- instance.getFragment().clearUselessField();
+ if (!queryContext.isExplainAnalyze()) {
+ // EXPLAIN ANALYZE will use these instances, so we can't clear them
+ instance.getFragment().clearUselessField();
+ } else {
+ // TypeProvider is not used in EXPLAIN ANALYZE, so we can clear it
+ instance.getFragment().clearTypeProvider();
+ }
QUERY_EXECUTION_METRIC_SET.recordExecutionCost(
DISPATCH_READ, System.nanoTime() - startTime);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
index 37c3ef391fc..ca4df4fa884 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
@@ -102,6 +102,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.ShowLogicalV
import org.apache.iotdb.db.queryengine.plan.statement.pipe.PipeEnrichedStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ClearCacheStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainAnalyzeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.ExplainStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement;
@@ -366,6 +367,10 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(explainStatement, context);
}
+ public R visitExplainAnalyze(ExplainAnalyzeStatement explainAnalyzeStatement, C context) {
+ return visitStatement(explainAnalyzeStatement, context);
+ }
+
public R visitDeleteData(DeleteDataStatement deleteDataStatement, C context) {
return visitStatement(deleteDataStatement, context);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ExplainAnalyzeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ExplainAnalyzeStatement.java
new file mode 100644
index 00000000000..0e87e5b5b4d
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/ExplainAnalyzeStatement.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.statement.sys;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
+import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
+
+import java.util.List;
+
+public class ExplainAnalyzeStatement extends Statement {
+
+ private final QueryStatement queryStatement;
+
+ private boolean verbose = false;
+
+ public ExplainAnalyzeStatement(QueryStatement queryStatement) {
+ this.queryStatement = queryStatement;
+ }
+
+ public QueryStatement getQueryStatement() {
+ return queryStatement;
+ }
+
+ public void setVerbose(boolean verbose) {
+ this.verbose = verbose;
+ }
+
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+ @Override
+ public List<PartialPath> getPaths() {
+ return queryStatement.getPaths();
+ }
+
+ @Override
+ public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
+ return visitor.visitExplainAnalyze(this, context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java
new file mode 100644
index 00000000000..55e085b6cc1
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
+import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics;
+import org.apache.iotdb.mpp.rpc.thrift.TQueryStatistics;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class FragmentInstanceStatisticsDrawer {
+ private int maxLineLength = 0;
+ private static final double EPSILON = 1e-10;
+ private final List<StatisticLine> planHeader = new ArrayList<>();
+ private static final double NS_TO_MS_FACTOR = 1.0 / 1000000;
+
+ public void renderPlanStatistics(MPPQueryContext context) {
+ addLine(
+ planHeader,
+ 0,
+ String.format("Analyze Cost: %.3f ms", context.getAnalyzeCost() * NS_TO_MS_FACTOR));
+ addLine(
+ planHeader,
+ 0,
+ String.format(
+ "Fetch Partition Cost: %.3f ms", context.getFetchPartitionCost() * NS_TO_MS_FACTOR));
+ addLine(
+ planHeader,
+ 0,
+ String.format(
+ "Fetch Schema Cost: %.3f ms", context.getFetchSchemaCost() * NS_TO_MS_FACTOR));
+ addLine(
+ planHeader,
+ 0,
+ String.format(
+ "Logical Plan Cost: %.3f ms", context.getLogicalPlanCost() * NS_TO_MS_FACTOR));
+ addLine(
+ planHeader,
+ 0,
+ String.format(
+ "Logical Optimization Cost: %.3f ms",
+ context.getLogicalOptimizationCost() * NS_TO_MS_FACTOR));
+ addLine(
+ planHeader,
+ 0,
+ String.format(
+ "Distribution Plan Cost: %.3f ms",
+ context.getDistributionPlanCost() * NS_TO_MS_FACTOR));
+ }
+
+ public List<StatisticLine> renderFragmentInstances(
+ List<FragmentInstance> instancesToBeRendered,
+ Map<FragmentInstanceId, TFetchFragmentInstanceStatisticsResp> allStatistics,
+ boolean verbose) {
+ List<StatisticLine> table = new ArrayList<>(planHeader);
+ addLine(
+ table, 0, String.format("Fragment Instances Count: %s", instancesToBeRendered.size() - 1));
+ for (FragmentInstance instance : instancesToBeRendered) {
+ List<StatisticLine> singleFragmentInstanceArea = new ArrayList<>();
+ TFetchFragmentInstanceStatisticsResp statistics = allStatistics.get(instance.getId());
+ if (statistics == null || statistics.getDataRegion() == null) {
+ continue;
+ }
+ addBlankLine(singleFragmentInstanceArea);
+ addLine(
+ singleFragmentInstanceArea,
+ 0,
+ String.format(
+ "FRAGMENT-INSTANCE[Id: %s][IP: %s][DataRegion: %s]",
+ instance.getId().toString(), statistics.getIp(), statistics.getDataRegion()));
+ addLine(
+ singleFragmentInstanceArea,
+ 1,
+ String.format(
+ "Total Wall Time: %s ms",
+ (statistics.getEndTimeInMS() - statistics.getStartTimeInMS())));
+ addLine(
+ singleFragmentInstanceArea,
+ 1,
+ String.format(
+ "Cost of initDataQuerySource: %.3f ms",
+ statistics.getInitDataQuerySourceCost() * NS_TO_MS_FACTOR));
+ addLine(
+ singleFragmentInstanceArea,
+ 1,
+ String.format(
+ "Seq File(unclosed): %s, Seq File(closed): %s",
+ statistics.getSeqUnclosedNum(), statistics.getSeqClosednNum()));
+ addLine(
+ singleFragmentInstanceArea,
+ 1,
+ String.format(
+ "UnSeq File(unclosed): %s, UnSeq File(closed): %s",
+ statistics.getUnseqUnclosedNum(), statistics.getUnseqClosedNum()));
+ addLine(
+ singleFragmentInstanceArea,
+ 1,
+ String.format(
+ "ready queued time: %.3f ms, blocked queued time: %.3f ms",
+ statistics.getReadyQueuedTime() * NS_TO_MS_FACTOR,
+ statistics.getBlockQueuedTime() * NS_TO_MS_FACTOR));
+ if (verbose) {
+ renderQueryStatistics(statistics.getQueryStatistics(), singleFragmentInstanceArea);
+ }
+ // render operator
+ PlanNode planNodeTree = instance.getFragment().getPlanNodeTree();
+ renderOperator(
+ planNodeTree, statistics.getOperatorStatisticsMap(), singleFragmentInstanceArea, 2);
+ table.addAll(singleFragmentInstanceArea);
+ }
+
+ return table;
+ }
+
+ private void addLineWithValueCheck(
+ List<StatisticLine> singleFragmentInstanceArea, int level, String valueName, long value) {
+ if (value != 0) {
+ addLine(singleFragmentInstanceArea, level, valueName + String.format(": %s", value));
+ }
+ }
+
+ private void addLineWithValueCheck(
+ List<StatisticLine> singleFragmentInstanceArea, int level, String valueName, double value) {
+ if (Math.abs(value) > EPSILON) {
+ addLine(singleFragmentInstanceArea, level, valueName + String.format(": %.3f", value));
+ }
+ }
+
+ private void addBlankLine(List<StatisticLine> singleFragmentInstanceArea) {
+ addLine(singleFragmentInstanceArea, 0, " ");
+ }
+
+ private void renderQueryStatistics(
+ TQueryStatistics queryStatistics, List<StatisticLine> singleFragmentInstanceArea) {
+ addLine(singleFragmentInstanceArea, 1, "Query Statistics:");
+
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataDiskSeqCount",
+ queryStatistics.loadTimeSeriesMetadataDiskSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataDiskUnSeqCount",
+ queryStatistics.loadTimeSeriesMetadataDiskUnSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataMemSeqCount",
+ queryStatistics.loadTimeSeriesMetadataMemSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataMemUnSeqCount",
+ queryStatistics.loadTimeSeriesMetadataMemUnSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedDiskSeqCount",
+ queryStatistics.loadTimeSeriesMetadataAlignedDiskSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedDiskUnSeqCount",
+ queryStatistics.loadTimeSeriesMetadataAlignedDiskUnSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedMemSeqCount",
+ queryStatistics.loadTimeSeriesMetadataAlignedMemSeqCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedMemUnSeqCount",
+ queryStatistics.loadTimeSeriesMetadataAlignedMemUnSeqCount);
+
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataDiskSeqTime",
+ queryStatistics.loadTimeSeriesMetadataDiskSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataDiskUnSeqTime",
+ queryStatistics.loadTimeSeriesMetadataDiskUnSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataMemSeqTime",
+ queryStatistics.loadTimeSeriesMetadataMemSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataMemUnSeqTime",
+ queryStatistics.loadTimeSeriesMetadataMemUnSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedDiskSeqTime",
+ queryStatistics.loadTimeSeriesMetadataAlignedDiskSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedDiskUnSeqTime",
+ queryStatistics.loadTimeSeriesMetadataAlignedDiskUnSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedMemSeqTime",
+ queryStatistics.loadTimeSeriesMetadataAlignedMemSeqTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "loadTimeSeriesMetadataAlignedMemUnSeqTime",
+ queryStatistics.loadTimeSeriesMetadataAlignedMemUnSeqTime * NS_TO_MS_FACTOR);
+
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructNonAlignedChunkReadersDiskCount",
+ queryStatistics.constructNonAlignedChunkReadersDiskCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructNonAlignedChunkReadersMemCount",
+ queryStatistics.constructNonAlignedChunkReadersMemCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructAlignedChunkReadersDiskCount",
+ queryStatistics.constructAlignedChunkReadersDiskCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructAlignedChunkReadersMemCount",
+ queryStatistics.constructAlignedChunkReadersMemCount);
+
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructNonAlignedChunkReadersDiskTime",
+ queryStatistics.constructNonAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructNonAlignedChunkReadersMemTime",
+ queryStatistics.constructNonAlignedChunkReadersMemTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructAlignedChunkReadersDiskTime",
+ queryStatistics.constructAlignedChunkReadersDiskTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "constructAlignedChunkReadersMemTime",
+ queryStatistics.constructAlignedChunkReadersMemTime * NS_TO_MS_FACTOR);
+
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeAlignedDiskCount",
+ queryStatistics.pageReadersDecodeAlignedDiskCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeAlignedDiskTime",
+ queryStatistics.pageReadersDecodeAlignedDiskTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeAlignedMemCount",
+ queryStatistics.pageReadersDecodeAlignedMemCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeAlignedMemTime",
+ queryStatistics.pageReadersDecodeAlignedMemTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeNonAlignedDiskCount",
+ queryStatistics.pageReadersDecodeNonAlignedDiskCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeNonAlignedDiskTime",
+ queryStatistics.pageReadersDecodeNonAlignedDiskTime * NS_TO_MS_FACTOR);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeNonAlignedMemCount",
+ queryStatistics.pageReadersDecodeNonAlignedMemCount);
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ 2,
+ "pageReadersDecodeNonAlignedMemTime",
+ queryStatistics.pageReadersDecodeNonAlignedMemTime * NS_TO_MS_FACTOR);
+ }
+
+ private void addLine(List<StatisticLine> resultForSingleInstance, int level, String value) {
+ maxLineLength = Math.max(maxLineLength, value.length());
+
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < level; i++) {
+ sb.append(" ");
+ }
+ sb.append(value);
+ maxLineLength = Math.max(maxLineLength, sb.length());
+ resultForSingleInstance.add(new StatisticLine(sb.toString(), level));
+ }
+
+ private void renderOperator(
+ PlanNode planNodeTree,
+ Map<String, TOperatorStatistics> operatorStatistics,
+ List<StatisticLine> singleFragmentInstanceArea,
+ int indentNum) {
+ if (planNodeTree == null) return;
+ TOperatorStatistics operatorStatistic =
+ operatorStatistics.get(planNodeTree.getPlanNodeId().toString());
+ if (operatorStatistic != null) {
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum,
+ String.format(
+ "[PlanNodeId %s]: %s(%s) %s",
+ planNodeTree.getPlanNodeId().toString(),
+ planNodeTree.getClass().getSimpleName(),
+ operatorStatistic.getOperatorType(),
+ operatorStatistic.isSetCount() ? "Count: * " + operatorStatistic.getCount() : ""));
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ String.format(
+ "CPU Time: %.3f ms",
+ operatorStatistic.getTotalExecutionTimeInNanos() * NS_TO_MS_FACTOR));
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ String.format("input: %s rows", operatorStatistic.getInputRows()));
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ String.format("HasNext() Called Count: %s", operatorStatistic.hasNextCalledCount));
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ String.format("Next() Called Count: %s", operatorStatistic.nextCalledCount));
+ addLineWithValueCheck(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ "Estimated Memory Size: ",
+ operatorStatistic.getMemoryUsage());
+
+ if (operatorStatistic.getSpecifiedInfoSize() != 0) {
+ for (Map.Entry<String, String> entry : operatorStatistic.getSpecifiedInfo().entrySet()) {
+ addLine(
+ singleFragmentInstanceArea,
+ indentNum + 2,
+ String.format("%s: %s", entry.getKey(), entry.getValue()));
+ }
+ }
+ }
+
+ for (PlanNode child : planNodeTree.getChildren()) {
+ renderOperator(child, operatorStatistics, singleFragmentInstanceArea, indentNum + 1);
+ }
+ }
+
+ public int getMaxLineLength() {
+ return maxLineLength;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java
new file mode 100644
index 00000000000..cb8a4a6a8c6
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryPlanStatistics.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+public class QueryPlanStatistics {
+ private long analyzeCost;
+ private long fetchPartitionCost;
+ private long fetchSchemaCost;
+ private long logicalPlanCost;
+ private long logicalOptimizationCost;
+ private long distributionPlanCost;
+
+ public void setAnalyzeCost(long analyzeCost) {
+ this.analyzeCost = analyzeCost;
+ }
+
+ public void setFetchPartitionCost(long fetchPartitionCost) {
+ this.fetchPartitionCost = fetchPartitionCost;
+ }
+
+ public void setFetchSchemaCost(long fetchSchemaCost) {
+ this.fetchSchemaCost = fetchSchemaCost;
+ }
+
+ public void setLogicalPlanCost(long logicalPlanCost) {
+ this.logicalPlanCost = logicalPlanCost;
+ }
+
+ public void setDistributionPlanCost(long distributionPlanCost) {
+ this.distributionPlanCost = distributionPlanCost;
+ }
+
+ public void setLogicalOptimizationCost(long logicalOptimizationCost) {
+ this.logicalOptimizationCost = logicalOptimizationCost;
+ }
+
+ public long getAnalyzeCost() {
+ return analyzeCost;
+ }
+
+ public long getFetchPartitionCost() {
+ return fetchPartitionCost;
+ }
+
+ public long getFetchSchemaCost() {
+ return fetchSchemaCost;
+ }
+
+ public long getLogicalPlanCost() {
+ return logicalPlanCost;
+ }
+
+ public long getDistributionPlanCost() {
+ return distributionPlanCost;
+ }
+
+ public long getLogicalOptimizationCost() {
+ return logicalOptimizationCost;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryStatisticsFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryStatisticsFetcher.java
new file mode 100644
index 00000000000..7636caa5628
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/QueryStatisticsFetcher.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.mpp.FragmentInstanceFetchException;
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsReq;
+import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class QueryStatisticsFetcher {
+
+ private static final String LOCAL_HOST_IP =
+ IoTDBDescriptor.getInstance().getConfig().getInternalAddress();
+
+ private static final int LOCAL_HOST_PORT =
+ IoTDBDescriptor.getInstance().getConfig().getInternalPort();
+
+ private QueryStatisticsFetcher() {
+ // allowed to do nothing
+ }
+
+ public static Map<FragmentInstanceId, TFetchFragmentInstanceStatisticsResp> fetchAllStatistics(
+ List<FragmentInstance> instances,
+ IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> clientManager)
+ throws FragmentInstanceFetchException {
+ Map<FragmentInstanceId, TFetchFragmentInstanceStatisticsResp> fragmentInstanceStatistics =
+ new HashMap<>();
+ for (FragmentInstance instance : instances) {
+ fragmentInstanceStatistics.put(instance.getId(), fetchStatistics(instance, clientManager));
+ }
+ return fragmentInstanceStatistics;
+ }
+
+ private static TFetchFragmentInstanceStatisticsResp fetchStatistics(
+ FragmentInstance instance,
+ IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> clientManager)
+ throws FragmentInstanceFetchException {
+ TEndPoint endPoint = instance.getHostDataNode().internalEndPoint;
+
+ TFetchFragmentInstanceStatisticsResp fragmentInstanceStatistics;
+ // If instance is running on the local host, we can directly fetch the statistics from the local
+ // host
+ if (LOCAL_HOST_IP.equals(endPoint.getIp()) && LOCAL_HOST_PORT == endPoint.getPort()) {
+ // Get statistics for current instance
+ fragmentInstanceStatistics =
+ FragmentInstanceManager.getInstance().getFragmentInstanceStatistics(instance.getId());
+ } else {
+ try (SyncDataNodeInternalServiceClient client = clientManager.borrowClient(endPoint)) {
+ fragmentInstanceStatistics =
+ client.fetchFragmentInstanceStatistics(
+ new TFetchFragmentInstanceStatisticsReq(instance.getId().toThrift()));
+ if (fragmentInstanceStatistics.status.getCode()
+ != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new FragmentInstanceFetchException(fragmentInstanceStatistics.status);
+ }
+ } catch (TException | ClientManagerException e) {
+ TSStatus status = new TSStatus();
+ status.setCode(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ status.setMessage(e.getMessage());
+ throw new FragmentInstanceFetchException(status);
+ }
+ }
+ return fragmentInstanceStatistics;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/SpecifiedInfoMergerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/SpecifiedInfoMergerFactory.java
new file mode 100644
index 00000000000..08e0ffd6b12
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/SpecifiedInfoMergerFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+import java.util.Collections;
+import java.util.Map;
+
+public class SpecifiedInfoMergerFactory {
+
+ @FunctionalInterface
+ public interface SpecifiedInfoMerger {
+ Map<String, String> merge(Map<String, String> first, Map<String, String> second);
+ }
+
+ private static final SpecifiedInfoMerger DEFAULT_MERGER =
+ (first, second) -> {
+ return Collections.emptyMap();
+ };
+
+ private static final SpecifiedInfoMerger LONG_MERGER =
+ (first, second) -> {
+ first.replaceAll(
+ (k, v) -> Long.toString(Long.parseLong(v) + Long.parseLong(second.get(k))));
+ return first;
+ };
+
+ public static SpecifiedInfoMerger getMerger(String operatorType) {
+ switch (operatorType) {
+ case "SortOperator":
+ case "MergeSortOperator":
+ case "FilterAndProjectOperator":
+ return LONG_MERGER;
+ default:
+ return DEFAULT_MERGER;
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticLine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticLine.java
new file mode 100644
index 00000000000..ec10f68c020
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticLine.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+public class StatisticLine {
+ private final String value;
+ private final int level;
+
+ public StatisticLine(String value, int level) {
+ this.value = value;
+ this.level = level;
+ }
+
+ public int getLevel() {
+ return level;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticsMergeUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticsMergeUtil.java
new file mode 100644
index 00000000000..88679fe8905
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/StatisticsMergeUtil.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.statistics;
+
+import org.apache.iotdb.mpp.rpc.thrift.TOperatorStatistics;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class StatisticsMergeUtil {
+
+ private StatisticsMergeUtil() {
+ // hidden constructor
+ }
+
+ public static void mergeOperatorStatisticsIfDuplicate(
+ Map<String, TOperatorStatistics> operatorStatisticsMap) {
+ List<String> keysToRemove = new ArrayList<>();
+ Map<String, TOperatorStatistics> entriesToAdd = new HashMap<>();
+
+ for (Map.Entry<String, TOperatorStatistics> entry : operatorStatisticsMap.entrySet()) {
+ String key = entry.getKey();
+ if (key.contains("-")) {
+ String[] keys = key.split("-");
+ String planNodeId = keys[0];
+ if (entriesToAdd.containsKey(planNodeId)) {
+ // merge the two operatorStatistics and put in entriesToAdd
+ merge(entriesToAdd.get(planNodeId), entry.getValue());
+ keysToRemove.add(key);
+ } else {
+ entriesToAdd.put(planNodeId, entry.getValue());
+ keysToRemove.add(key);
+ }
+ }
+ }
+ for (String key : keysToRemove) {
+ operatorStatisticsMap.remove(key);
+ }
+ operatorStatisticsMap.putAll(entriesToAdd);
+ }
+
+ public static void mergeAllOperatorStatistics(
+ Map<String, TOperatorStatistics> operatorStatisticsMap,
+ Map<String, String> leadOverloadOperators) {
+ for (Map.Entry<String, TOperatorStatistics> entry : operatorStatisticsMap.entrySet()) {
+ if (leadOverloadOperators.containsKey(entry.getValue().getOperatorType())) {
+ merge(
+ operatorStatisticsMap.get(
+ leadOverloadOperators.get(entry.getValue().getOperatorType())),
+ entry.getValue());
+ } else {
+ TOperatorStatistics operatorStatistics = entry.getValue();
+ operatorStatistics.setCount(1);
+ // Can't merge specifiedInfo of String-type, so just clear it
+ operatorStatistics.getSpecifiedInfo().clear();
+ // keep the first one in operatorStatisticsMap as the only-one leadOverloadOperator
+ leadOverloadOperators.put(
+ operatorStatistics.getOperatorType(), operatorStatistics.getPlanNodeId());
+ }
+ }
+ }
+
+ public static void merge(TOperatorStatistics first, TOperatorStatistics second) {
+ first.setTotalExecutionTimeInNanos(
+ first.getTotalExecutionTimeInNanos() + second.getTotalExecutionTimeInNanos());
+ first.setNextCalledCount(first.getNextCalledCount() + second.getNextCalledCount());
+ first.setHasNextCalledCount(first.getHasNextCalledCount() + second.getHasNextCalledCount());
+ first.setInputRows(first.getInputRows() + second.getInputRows());
+ first.setMemoryUsage(first.getMemoryUsage() + second.getMemoryUsage());
+ first.setCount(first.getCount() + 1);
+ first.setSpecifiedInfo(
+ SpecifiedInfoMergerFactory.getMerger(first.getOperatorType())
+ .merge(first.getSpecifiedInfo(), second.getSpecifiedInfo()));
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/sort/DiskSpiller.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/sort/DiskSpiller.java
index f7a63ce6ded..1922b6b87ae 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/sort/DiskSpiller.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/sort/DiskSpiller.java
@@ -163,4 +163,8 @@ public class DiskSpiller {
}
return sortReaders;
}
+
+ public int getFileSize() {
+ return fileIndex;
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
index d54a1fc851c..6f3ceab9cdc 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java
@@ -77,6 +77,7 @@ public class FragmentInstanceExecutionTest {
stateMachine,
failedInstances,
timeOut,
+ false,
exchangeManager);
assertEquals(FragmentInstanceState.RUNNING, execution.getInstanceState());
FragmentInstanceInfo instanceInfo = execution.getInstanceInfo();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
index 7f720ae58f0..228f7d99b76 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java
@@ -407,7 +407,7 @@ public class OperatorMemoryTest {
@Test
public void limitOperatorTest() {
Operator child = Mockito.mock(Operator.class);
- Mockito.when(child.calculateMaxPeekMemory()).thenReturn(2 * 1024L);
+ Mockito.when(child.calculateMaxPeekMemoryWithCounter()).thenReturn(2 * 1024L);
Mockito.when(child.calculateMaxReturnSize()).thenReturn(1024L);
Mockito.when(child.calculateRetainedSizeAfterCallingNext()).thenReturn(512L);
@@ -422,7 +422,7 @@ public class OperatorMemoryTest {
@Test
public void offsetOperatorTest() {
Operator child = Mockito.mock(Operator.class);
- Mockito.when(child.calculateMaxPeekMemory()).thenReturn(2 * 1024L);
+ Mockito.when(child.calculateMaxPeekMemoryWithCounter()).thenReturn(2 * 1024L);
Mockito.when(child.calculateMaxReturnSize()).thenReturn(1024L);
Mockito.when(child.calculateRetainedSizeAfterCallingNext()).thenReturn(512L);
@@ -506,7 +506,7 @@ public class OperatorMemoryTest {
@Test
public void sortOperatorTest() {
Operator child = Mockito.mock(Operator.class);
- Mockito.when(child.calculateMaxPeekMemory()).thenReturn(2048L);
+ Mockito.when(child.calculateMaxPeekMemoryWithCounter()).thenReturn(2048L);
Mockito.when(child.calculateMaxReturnSize()).thenReturn(1024L);
Mockito.when(child.calculateRetainedSizeAfterCallingNext()).thenReturn(512L);
@@ -1449,7 +1449,7 @@ public class OperatorMemoryTest {
@Test
public void intoOperatorTest() {
Operator child = Mockito.mock(Operator.class);
- Mockito.when(child.calculateMaxPeekMemory())
+ Mockito.when(child.calculateMaxPeekMemoryWithCounter())
.thenReturn((long) DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES);
Mockito.when(child.calculateMaxReturnSize())
.thenReturn((long) DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java
index 3a7be5f4dcf..d59175a16f9 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/schedule/DefaultDriverSchedulerTest.java
@@ -18,11 +18,15 @@
*/
package org.apache.iotdb.db.queryengine.execution.schedule;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.execution.driver.DriverContext;
import org.apache.iotdb.db.queryengine.execution.driver.IDriver;
import org.apache.iotdb.db.queryengine.execution.exchange.IMPPDataExchangeManager;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
+import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
import org.apache.iotdb.db.queryengine.execution.schedule.queue.multilevelqueue.DriverTaskHandle;
import org.apache.iotdb.db.queryengine.execution.schedule.queue.multilevelqueue.MultilevelPriorityQueue;
import org.apache.iotdb.db.queryengine.execution.schedule.task.DriverTask;
@@ -43,11 +47,16 @@ import java.util.Map;
import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+import static org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext.createFragmentInstanceContext;
+
public class DefaultDriverSchedulerTest {
private final DriverScheduler manager = DriverScheduler.getInstance();
+ private ExecutorService instanceNotificationExecutor =
+ IoTDBThreadPoolFactory.newFixedThreadPool(1, "test");;
@After
public void tearDown() throws IOException {
@@ -97,6 +106,13 @@ public class DefaultDriverSchedulerTest {
}
DriverTask testTask =
new DriverTask(mockDriver, 100L, DriverTaskStatus.BLOCKED, driverTaskHandle, 0, false);
+
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor);
+ FragmentInstanceContext fragmentInstanceContext =
+ createFragmentInstanceContext(instanceId, stateMachine);
+ DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0);
+ Mockito.when(testTask.getDriver().getDriverContext()).thenReturn(driverContext);
manager.getBlockedTasks().add(testTask);
Set<DriverTask> taskSet = new HashSet<>();
taskSet.add(testTask);
@@ -163,6 +179,13 @@ public class DefaultDriverSchedulerTest {
fragmentRelatedTask.put(instanceId, taskSet);
manager.getQueryMap().put(queryId, fragmentRelatedTask);
manager.getTimeoutQueue().push(testTask);
+
+ FragmentInstanceStateMachine stateMachine =
+ new FragmentInstanceStateMachine(instanceId, instanceNotificationExecutor);
+ FragmentInstanceContext fragmentInstanceContext =
+ createFragmentInstanceContext(instanceId, stateMachine);
+ DriverContext driverContext = new DriverContext(fragmentInstanceContext, 0);
+ Mockito.when(testTask.getDriver().getDriverContext()).thenReturn(driverContext);
defaultScheduler.readyToRunning(testTask);
Assert.assertEquals(DriverTaskStatus.RUNNING, testTask.getStatus());
Assert.assertFalse(manager.getBlockedTasks().contains(testTask));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 23bc7884345..801825e29ce 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -102,6 +102,7 @@ public class IoTDBConstant {
public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER";
public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
+ public static final String EXPLAIN_ANALYZE_LOGGER_NAME = "EXPLAIN_ANALYZE";
public static final String IOTDB_JMX_LOCAL = "iotdb.jmx.local";
public static final String IOTDB_JMX_PORT = "com.sun.management.jmxremote.port";
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index 2a35b399593..0ea83f8b9a0 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -456,6 +456,86 @@ struct TExecuteCQ {
7: required string username
}
+/**
+* BEGIN: Used for EXPLAIN ANALYZE
+**/
+struct TOperatorStatistics{
+ 1: required string planNodeId
+ 2: required string operatorType
+ 3: required i64 totalExecutionTimeInNanos
+ 4: required i64 nextCalledCount
+ 5: required i64 hasNextCalledCount
+ 6: required map<string,string> specifiedInfo
+ 7: required i64 inputRows
+ 8: required i64 memoryUsage
+ 9: optional i64 count
+}
+
+struct TQueryStatistics {
+ 1: i64 loadTimeSeriesMetadataDiskSeqCount,
+ 2: i64 loadTimeSeriesMetadataDiskUnSeqCount,
+ 3: i64 loadTimeSeriesMetadataMemSeqCount,
+ 4: i64 loadTimeSeriesMetadataMemUnSeqCount,
+ 5: i64 loadTimeSeriesMetadataAlignedDiskSeqCount,
+ 6: i64 loadTimeSeriesMetadataAlignedDiskUnSeqCount,
+ 7: i64 loadTimeSeriesMetadataAlignedMemSeqCount,
+ 8: i64 loadTimeSeriesMetadataAlignedMemUnSeqCount,
+
+ 9: i64 loadTimeSeriesMetadataDiskSeqTime,
+ 10: i64 loadTimeSeriesMetadataDiskUnSeqTime,
+ 11: i64 loadTimeSeriesMetadataMemSeqTime,
+ 12: i64 loadTimeSeriesMetadataMemUnSeqTime,
+ 13: i64 loadTimeSeriesMetadataAlignedDiskSeqTime,
+ 14: i64 loadTimeSeriesMetadataAlignedDiskUnSeqTime,
+ 15: i64 loadTimeSeriesMetadataAlignedMemSeqTime,
+ 16: i64 loadTimeSeriesMetadataAlignedMemUnSeqTime,
+
+ 17: i64 constructNonAlignedChunkReadersDiskCount,
+ 18: i64 constructNonAlignedChunkReadersMemCount,
+ 19: i64 constructAlignedChunkReadersDiskCount,
+ 20: i64 constructAlignedChunkReadersMemCount,
+
+ 21: i64 constructNonAlignedChunkReadersDiskTime,
+ 22: i64 constructNonAlignedChunkReadersMemTime,
+ 23: i64 constructAlignedChunkReadersDiskTime,
+ 24: i64 constructAlignedChunkReadersMemTime,
+
+ 25: i64 pageReadersDecodeAlignedDiskCount,
+ 26: i64 pageReadersDecodeAlignedDiskTime,
+ 27: i64 pageReadersDecodeAlignedMemCount,
+ 28: i64 pageReadersDecodeAlignedMemTime,
+ 29: i64 pageReadersDecodeNonAlignedDiskCount,
+ 30: i64 pageReadersDecodeNonAlignedDiskTime,
+ 31: i64 pageReadersDecodeNonAlignedMemCount,
+ 32: i64 pageReadersDecodeNonAlignedMemTime
+}
+
+
+struct TFetchFragmentInstanceStatisticsReq {
+ 1: required TFragmentInstanceId fragmentInstanceId
+}
+
+struct TFetchFragmentInstanceStatisticsResp {
+ 1: required common.TSStatus status
+ 2: optional TFragmentInstanceId fragmentInstanceId
+ 3: optional string dataRegion
+ 4: optional i64 startTimeInMS
+ 5: optional i64 endTimeInMS
+ 6: optional TQueryStatistics queryStatistics
+ 7: optional map<string, TOperatorStatistics> operatorStatisticsMap
+ 8: optional i64 initDataQuerySourceCost
+ 9: optional i64 seqUnclosedNum
+ 10: optional i64 seqClosednNum
+ 11: optional i64 unseqClosedNum
+ 12: optional i64 unseqUnclosedNum
+ 13: optional i64 readyQueuedTime
+ 14: optional i64 blockQueuedTime
+ 15: optional string ip
+}
+/**
+* END: Used for EXPLAIN ANALYZE
+**/
+
service IDataNodeRPCService {
// -----------------------------------For Data Node-----------------------------------------------
@@ -789,6 +869,11 @@ service IDataNodeRPCService {
* Set throttle quota
**/
common.TSStatus setThrottleQuota(common.TSetThrottleQuotaReq req)
+
+ /**
+ * Fetch fragment instance statistics for EXPLAIN ANALYZE
+ */
+ TFetchFragmentInstanceStatisticsResp fetchFragmentInstanceStatistics(TFetchFragmentInstanceStatisticsReq req)
}
service MPPDataExchangeService {