You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/12/21 03:48:15 UTC
[iotdb] branch QueryMetrics updated (b95dc6e7d1 -> d7c442b516)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from b95dc6e7d1 Add more
new 73b7ec3207 Add more metrics
new 4d12580807 format print
new 81724c093f format print
new d7c442b516 add format code
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../iotdb/db/engine/storagegroup/DataRegion.java | 5 +
.../iotdb/db/mpp/aggregation/Aggregator.java | 3 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 33 +++-
.../db/mpp/execution/exchange/LocalSinkHandle.java | 4 +-
.../AbstractSeriesAggregationScanOperator.java | 20 +-
.../execution/operator/source/SeriesScanUtil.java | 4 +-
.../iotdb/db/mpp/statistics/QueryStatistics.java | 218 ++++++++++++++++++++-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 18 +-
8 files changed, 260 insertions(+), 45 deletions(-)
[iotdb] 04/04: add format code
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d7c442b51630ce52a8afe7671ab73f13fe32ee8e
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Dec 21 11:48:08 2022 +0800
add format code
---
.../java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 9488fce01c..be908bae34 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory;
import javax.annotation.concurrent.ThreadSafe;
+import java.text.DecimalFormat;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
@@ -42,6 +43,8 @@ public class QueryStatistics {
private static final Logger QUERY_STATISTICS_LOGGER =
LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME);
+ private static final DecimalFormat format = new DecimalFormat("#,###");
+
private final AtomicBoolean tracing = new AtomicBoolean(false);
private final Map<String, OperationStatistic> operationStatistics = new ConcurrentHashMap<>();
@@ -406,12 +409,12 @@ public class QueryStatistics {
long count = totalCount.get();
return "{"
+ "totalTime="
- + time
+ + format.format(time)
+ "us"
+ ", totalCount="
- + count
+ + format.format(count)
+ ", avgOpTime="
- + (time / count)
+ + format.format(time / count)
+ "us"
+ '}';
}
[iotdb] 03/04: format print
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 81724c093f92f15ad29f0f99cca4559343103104
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Dec 20 20:29:35 2022 +0800
format print
---
.../iotdb/db/mpp/statistics/QueryStatistics.java | 339 ++++++++++++---------
1 file changed, 194 insertions(+), 145 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 80436327de..9488fce01c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -165,151 +165,200 @@ public class QueryStatistics {
private void printQueryStatistics() {
if (tracing.get()) {
- String res =
- System.lineSeparator()
- + "Client Connection Thread:"
- + System.lineSeparator()
- + System.lineSeparator()
- + "ServerRpcRT "
- + operationStatistics.get(SERVER_RPC_RT)
- + System.lineSeparator()
- + "|___CreateQueryExec "
- + operationStatistics.get(CREATE_QUERY_EXEC)
- + System.lineSeparator()
- + "| |___Parser "
- + operationStatistics.get(PARSER)
- + System.lineSeparator()
- + "| |___Analyzer "
- + operationStatistics.get(ANALYZER)
- + System.lineSeparator()
- + "| | |___PartitionFetcher "
- + operationStatistics.get(PARTITION_FETCHER)
- + System.lineSeparator()
- + "| | |___SchemaFetcher "
- + operationStatistics.get(SCHEMA_FETCHER)
- + System.lineSeparator()
- + "| |___LogicalPlanner "
- + operationStatistics.get(LOGICAL_PLANNER)
- + System.lineSeparator()
- + "| |___DistributionPlanner "
- + operationStatistics.get(DISTRIBUTION_PLANNER)
- + System.lineSeparator()
- + "| |___Dispatcher "
- + operationStatistics.get(DISPATCHER)
- + System.lineSeparator()
- + "| |___DispatchRead "
- + operationStatistics.get(DISPATCH_READ)
- + System.lineSeparator()
- + "| |___LocalExecPlanner "
- + operationStatistics.get(LOCAL_EXECUTION_PLANNER)
- + System.lineSeparator()
- + "| |___FIContext "
- + operationStatistics.get(CREATE_FI_CONTEXT)
- + System.lineSeparator()
- + "| |___ToOpTree "
- + operationStatistics.get(NODE_TO_OPERATOR)
- + System.lineSeparator()
- + "| |___CheckMem "
- + operationStatistics.get(CHECK_MEMORY)
- + System.lineSeparator()
- + "| |___AllocExcgMem "
- + operationStatistics.get(ALLOC_EX_MEMORY)
- + System.lineSeparator()
- + "| |___FIExec "
- + operationStatistics.get(CREATE_FI_EXEC)
- + System.lineSeparator()
- + "|___SerTsBlock "
- + operationStatistics.get(SERIALIZE_TSBLOCK)
- + System.lineSeparator()
- + " |___WaitForResult "
- + operationStatistics.get(WAIT_FOR_RESULT)
- + System.lineSeparator()
- + " |___GetTsBlock "
- + operationStatistics.get(LOCAL_SOURCE_HANDLE_GET_TSBLOCK)
- + System.lineSeparator()
- + " |___FreeMem "
- + operationStatistics.get(FREE_MEM)
- + System.lineSeparator()
- + "Query Execution Thread:"
- + System.lineSeparator()
- + System.lineSeparator()
- + "|___QueryResourceInit "
- + operationStatistics.get(QUERY_RESOURCE_INIT)
- + System.lineSeparator()
- + "| |___TsFileList "
- + operationStatistics.get(QUERY_RESOURCE_LIST)
- + System.lineSeparator()
- + "| |___AddRef "
- + operationStatistics.get(ADD_REFERENCE)
- + System.lineSeparator()
- + "| |___InitSourceOp "
- + operationStatistics.get(INIT_SOURCE_OP)
- + System.lineSeparator()
- + "|___DriverInternalProcess "
- + operationStatistics.get(DRIVER_INTERNAL_PROCESS)
- + System.lineSeparator()
- + "| |___AggScanOperator "
- + operationStatistics.get(AGG_SCAN_OPERATOR)
- + System.lineSeparator()
- + "| | |___CalcNextAggRes "
- + operationStatistics.get(CAL_NEXT_AGG_RES)
- + System.lineSeparator()
- + "| | |___loadTSMeta "
- + operationStatistics.get(LOAD_TIME_SERIES_METADATA_ALIGNED)
- + System.lineSeparator()
- + "| | |___AggFromFile "
- + operationStatistics.get(CAL_AGG_FROM_FILE)
- + System.lineSeparator()
- + "| | | |___loadChunkMeta "
- + operationStatistics.get(LOAD_CHUNK_METADATA_LIST)
- + System.lineSeparator()
- + "| | | |___AggFromChunk "
- + operationStatistics.get(CAL_AGG_FROM_CHUNK)
- + System.lineSeparator()
- + "| | | |___loadChunk "
- + operationStatistics.get(LOAD_PAGE_READER_LIST)
- + System.lineSeparator()
- + "| | | |___AggFromPage "
- + operationStatistics.get(CAL_AGG_FROM_PAGE)
- + System.lineSeparator()
- + "| | | |___loadPage "
- + operationStatistics.get(PAGE_READER)
- + System.lineSeparator()
- + "| | | |___AggFromRawData "
- + operationStatistics.get(CAL_AGG_FROM_RAW_DATA)
- + System.lineSeparator()
- + "| | | |___AggProcTsBlock "
- + operationStatistics.get(AGGREGATOR_PROCESS_TSBLOCK)
- + System.lineSeparator()
- + "| | |___BuildAggRes "
- + operationStatistics.get(BUILD_AGG_RES)
- + System.lineSeparator()
- + "| |___SendTsBlock "
- + operationStatistics.get(SEND_TSBLOCK)
- + System.lineSeparator()
- + "| |___ReserveMem "
- + operationStatistics.get(RESERVE_MEMORY)
- + System.lineSeparator()
- + "| |___NotifyNewTsBlock "
- + operationStatistics.get(NOTIFY_NEW_TSBLOCK)
- + System.lineSeparator()
- + "|___SetNoMoreTsBlock "
- + operationStatistics.get(SET_NO_MORE_TSBLOCK)
- + System.lineSeparator()
- + " |___NotifyEnd "
- + operationStatistics.get(NOTIFY_END)
- + System.lineSeparator()
- + " |___EndListener "
- + operationStatistics.get(SINK_HANDLE_END_LISTENER)
- + System.lineSeparator()
- + " |___CkAndInvOnFinished "
- + operationStatistics.get(CHECK_AND_INVOKE_ON_FINISHED)
- + System.lineSeparator()
- + " |___FinishListener "
- + operationStatistics.get(SINK_HANDLE_FINISH_LISTENER)
- + System.lineSeparator();
-
- QUERY_STATISTICS_LOGGER.info(res);
+ StringBuilder builder = new StringBuilder(System.lineSeparator());
+ builder
+ .append("Client Connection Thread:")
+ .append(System.lineSeparator())
+ .append(System.lineSeparator());
+
+ builder
+ .append("ServerRpcRT ")
+ .append(operationStatistics.get(SERVER_RPC_RT))
+ .append(System.lineSeparator());
+ builder
+ .append("|___CreateQueryExec ")
+ .append(operationStatistics.get(CREATE_QUERY_EXEC))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___Parser ")
+ .append(operationStatistics.get(PARSER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___Analyzer ")
+ .append(operationStatistics.get(ANALYZER))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___PartitionFetcher ")
+ .append(operationStatistics.get(PARTITION_FETCHER))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___SchemaFetcher ")
+ .append(operationStatistics.get(SCHEMA_FETCHER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___LogicalPlanner ")
+ .append(operationStatistics.get(LOGICAL_PLANNER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___DistributionPlanner ")
+ .append(operationStatistics.get(DISTRIBUTION_PLANNER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___Dispatcher ")
+ .append(operationStatistics.get(DISPATCHER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___DispatchRead ")
+ .append(operationStatistics.get(DISPATCH_READ))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___LocalExecPlanner ")
+ .append(operationStatistics.get(LOCAL_EXECUTION_PLANNER))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___FIContext ")
+ .append(operationStatistics.get(CREATE_FI_CONTEXT))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___ToOpTree ")
+ .append(operationStatistics.get(NODE_TO_OPERATOR))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___CheckMem ")
+ .append(operationStatistics.get(CHECK_MEMORY))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___AllocExcgMem ")
+ .append(operationStatistics.get(ALLOC_EX_MEMORY))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___FIExec ")
+ .append(operationStatistics.get(CREATE_FI_EXEC))
+ .append(System.lineSeparator());
+ builder
+ .append("|___SerTsBlock ")
+ .append(operationStatistics.get(SERIALIZE_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___WaitForResult ")
+ .append(operationStatistics.get(WAIT_FOR_RESULT))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___GetTsBlock ")
+ .append(operationStatistics.get(LOCAL_SOURCE_HANDLE_GET_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___FreeMem ")
+ .append(operationStatistics.get(FREE_MEM))
+ .append(System.lineSeparator());
+
+ builder
+ .append("Query Execution Thread:")
+ .append(System.lineSeparator())
+ .append(System.lineSeparator());
+
+ builder
+ .append("|___QueryResourceInit ")
+ .append(operationStatistics.get(QUERY_RESOURCE_INIT))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___TsFileList ")
+ .append(operationStatistics.get(QUERY_RESOURCE_LIST))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___AddRef ")
+ .append(operationStatistics.get(ADD_REFERENCE))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___InitSourceOp ")
+ .append(operationStatistics.get(INIT_SOURCE_OP))
+ .append(System.lineSeparator());
+ builder
+ .append("|___DriverInternalProcess ")
+ .append(operationStatistics.get(DRIVER_INTERNAL_PROCESS))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___AggScanOperator ")
+ .append(operationStatistics.get(AGG_SCAN_OPERATOR))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___CalcNextAggRes ")
+ .append(operationStatistics.get(CAL_NEXT_AGG_RES))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___loadTSMeta ")
+ .append(operationStatistics.get(LOAD_TIME_SERIES_METADATA_ALIGNED))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___AggFromFile ")
+ .append(operationStatistics.get(CAL_AGG_FROM_FILE))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___loadChunkMeta ")
+ .append(operationStatistics.get(LOAD_CHUNK_METADATA_LIST))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromChunk ")
+ .append(operationStatistics.get(CAL_AGG_FROM_CHUNK))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___loadChunk ")
+ .append(operationStatistics.get(LOAD_PAGE_READER_LIST))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromPage ")
+ .append(operationStatistics.get(CAL_AGG_FROM_PAGE))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___loadPage ")
+ .append(operationStatistics.get(PAGE_READER))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___AggFromRawData ")
+ .append(operationStatistics.get(CAL_AGG_FROM_RAW_DATA))
+ .append(System.lineSeparator());
+ builder
+ .append("| | | |___AggProcTsBlock ")
+ .append(operationStatistics.get(AGGREGATOR_PROCESS_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append("| | |___BuildAggRes ")
+ .append(operationStatistics.get(BUILD_AGG_RES))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___SendTsBlock ")
+ .append(operationStatistics.get(SEND_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___ReserveMem ")
+ .append(operationStatistics.get(RESERVE_MEMORY))
+ .append(System.lineSeparator());
+ builder
+ .append("| |___NotifyNewTsBlock ")
+ .append(operationStatistics.get(NOTIFY_NEW_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append("|___SetNoMoreTsBlock ")
+ .append(operationStatistics.get(SET_NO_MORE_TSBLOCK))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___NotifyEnd ")
+ .append(operationStatistics.get(NOTIFY_END))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___EndListener ")
+ .append(operationStatistics.get(SINK_HANDLE_END_LISTENER))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___CkAndInvOnFinished ")
+ .append(operationStatistics.get(CHECK_AND_INVOKE_ON_FINISHED))
+ .append(System.lineSeparator());
+ builder
+ .append(" |___FinishListener ")
+ .append(operationStatistics.get(SINK_HANDLE_FINISH_LISTENER))
+ .append(System.lineSeparator());
+
+ QUERY_STATISTICS_LOGGER.info(builder.toString());
QUERY_STATISTICS_LOGGER.info("");
}
}
[iotdb] 02/04: format print
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4d12580807e59d6d28d91999e5f3984dfba04b27
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Dec 20 20:27:42 2022 +0800
format print
---
.../db/mpp/execution/exchange/LocalSinkHandle.java | 4 +-
.../iotdb/db/mpp/statistics/QueryStatistics.java | 151 ++++++++++++++++++++-
2 files changed, 148 insertions(+), 7 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
index e43052c7e6..505d5dad32 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
@@ -98,7 +98,6 @@ public class LocalSinkHandle implements ISinkHandle {
}
public void checkAndInvokeOnFinished() {
- long startTime = System.nanoTime();
synchronized (queue) {
if (isFinished()) {
synchronized (this) {
@@ -108,7 +107,6 @@ public class LocalSinkHandle implements ISinkHandle {
}
}
}
- QUERY_STATISTICS.addCost(CHECK_AND_INVOKE_ON_FINISHED, System.nanoTime() - startTime);
}
@Override
@@ -152,7 +150,9 @@ public class LocalSinkHandle implements ISinkHandle {
}
}
+ long startTime = System.nanoTime();
checkAndInvokeOnFinished();
+ QUERY_STATISTICS.addCost(CHECK_AND_INVOKE_ON_FINISHED, System.nanoTime() - startTime);
logger.debug("[EndSetNoMoreTsBlocksOnLocal]");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 043dde1a72..80436327de 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -164,11 +164,152 @@ public class QueryStatistics {
private void printQueryStatistics() {
if (tracing.get()) {
- operationStatistics.forEach(
- (k, v) -> {
- QUERY_STATISTICS_LOGGER.info("Operation: {}, Statistics: {}", k, v);
- });
- // line breaker
+
+ String res =
+ System.lineSeparator()
+ + "Client Connection Thread:"
+ + System.lineSeparator()
+ + System.lineSeparator()
+ + "ServerRpcRT "
+ + operationStatistics.get(SERVER_RPC_RT)
+ + System.lineSeparator()
+ + "|___CreateQueryExec "
+ + operationStatistics.get(CREATE_QUERY_EXEC)
+ + System.lineSeparator()
+ + "| |___Parser "
+ + operationStatistics.get(PARSER)
+ + System.lineSeparator()
+ + "| |___Analyzer "
+ + operationStatistics.get(ANALYZER)
+ + System.lineSeparator()
+ + "| | |___PartitionFetcher "
+ + operationStatistics.get(PARTITION_FETCHER)
+ + System.lineSeparator()
+ + "| | |___SchemaFetcher "
+ + operationStatistics.get(SCHEMA_FETCHER)
+ + System.lineSeparator()
+ + "| |___LogicalPlanner "
+ + operationStatistics.get(LOGICAL_PLANNER)
+ + System.lineSeparator()
+ + "| |___DistributionPlanner "
+ + operationStatistics.get(DISTRIBUTION_PLANNER)
+ + System.lineSeparator()
+ + "| |___Dispatcher "
+ + operationStatistics.get(DISPATCHER)
+ + System.lineSeparator()
+ + "| |___DispatchRead "
+ + operationStatistics.get(DISPATCH_READ)
+ + System.lineSeparator()
+ + "| |___LocalExecPlanner "
+ + operationStatistics.get(LOCAL_EXECUTION_PLANNER)
+ + System.lineSeparator()
+ + "| |___FIContext "
+ + operationStatistics.get(CREATE_FI_CONTEXT)
+ + System.lineSeparator()
+ + "| |___ToOpTree "
+ + operationStatistics.get(NODE_TO_OPERATOR)
+ + System.lineSeparator()
+ + "| |___CheckMem "
+ + operationStatistics.get(CHECK_MEMORY)
+ + System.lineSeparator()
+ + "| |___AllocExcgMem "
+ + operationStatistics.get(ALLOC_EX_MEMORY)
+ + System.lineSeparator()
+ + "| |___FIExec "
+ + operationStatistics.get(CREATE_FI_EXEC)
+ + System.lineSeparator()
+ + "|___SerTsBlock "
+ + operationStatistics.get(SERIALIZE_TSBLOCK)
+ + System.lineSeparator()
+ + " |___WaitForResult "
+ + operationStatistics.get(WAIT_FOR_RESULT)
+ + System.lineSeparator()
+ + " |___GetTsBlock "
+ + operationStatistics.get(LOCAL_SOURCE_HANDLE_GET_TSBLOCK)
+ + System.lineSeparator()
+ + " |___FreeMem "
+ + operationStatistics.get(FREE_MEM)
+ + System.lineSeparator()
+ + "Query Execution Thread:"
+ + System.lineSeparator()
+ + System.lineSeparator()
+ + "|___QueryResourceInit "
+ + operationStatistics.get(QUERY_RESOURCE_INIT)
+ + System.lineSeparator()
+ + "| |___TsFileList "
+ + operationStatistics.get(QUERY_RESOURCE_LIST)
+ + System.lineSeparator()
+ + "| |___AddRef "
+ + operationStatistics.get(ADD_REFERENCE)
+ + System.lineSeparator()
+ + "| |___InitSourceOp "
+ + operationStatistics.get(INIT_SOURCE_OP)
+ + System.lineSeparator()
+ + "|___DriverInternalProcess "
+ + operationStatistics.get(DRIVER_INTERNAL_PROCESS)
+ + System.lineSeparator()
+ + "| |___AggScanOperator "
+ + operationStatistics.get(AGG_SCAN_OPERATOR)
+ + System.lineSeparator()
+ + "| | |___CalcNextAggRes "
+ + operationStatistics.get(CAL_NEXT_AGG_RES)
+ + System.lineSeparator()
+ + "| | |___loadTSMeta "
+ + operationStatistics.get(LOAD_TIME_SERIES_METADATA_ALIGNED)
+ + System.lineSeparator()
+ + "| | |___AggFromFile "
+ + operationStatistics.get(CAL_AGG_FROM_FILE)
+ + System.lineSeparator()
+ + "| | | |___loadChunkMeta "
+ + operationStatistics.get(LOAD_CHUNK_METADATA_LIST)
+ + System.lineSeparator()
+ + "| | | |___AggFromChunk "
+ + operationStatistics.get(CAL_AGG_FROM_CHUNK)
+ + System.lineSeparator()
+ + "| | | |___loadChunk "
+ + operationStatistics.get(LOAD_PAGE_READER_LIST)
+ + System.lineSeparator()
+ + "| | | |___AggFromPage "
+ + operationStatistics.get(CAL_AGG_FROM_PAGE)
+ + System.lineSeparator()
+ + "| | | |___loadPage "
+ + operationStatistics.get(PAGE_READER)
+ + System.lineSeparator()
+ + "| | | |___AggFromRawData "
+ + operationStatistics.get(CAL_AGG_FROM_RAW_DATA)
+ + System.lineSeparator()
+ + "| | | |___AggProcTsBlock "
+ + operationStatistics.get(AGGREGATOR_PROCESS_TSBLOCK)
+ + System.lineSeparator()
+ + "| | |___BuildAggRes "
+ + operationStatistics.get(BUILD_AGG_RES)
+ + System.lineSeparator()
+ + "| |___SendTsBlock "
+ + operationStatistics.get(SEND_TSBLOCK)
+ + System.lineSeparator()
+ + "| |___ReserveMem "
+ + operationStatistics.get(RESERVE_MEMORY)
+ + System.lineSeparator()
+ + "| |___NotifyNewTsBlock "
+ + operationStatistics.get(NOTIFY_NEW_TSBLOCK)
+ + System.lineSeparator()
+ + "|___SetNoMoreTsBlock "
+ + operationStatistics.get(SET_NO_MORE_TSBLOCK)
+ + System.lineSeparator()
+ + " |___NotifyEnd "
+ + operationStatistics.get(NOTIFY_END)
+ + System.lineSeparator()
+ + " |___EndListener "
+ + operationStatistics.get(SINK_HANDLE_END_LISTENER)
+ + System.lineSeparator()
+ + " |___CkAndInvOnFinished "
+ + operationStatistics.get(CHECK_AND_INVOKE_ON_FINISHED)
+ + System.lineSeparator()
+ + " |___FinishListener "
+ + operationStatistics.get(SINK_HANDLE_FINISH_LISTENER)
+ + System.lineSeparator();
+
+ QUERY_STATISTICS_LOGGER.info(res);
QUERY_STATISTICS_LOGGER.info("");
}
}
[iotdb] 01/04: Add more metrics
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 73b7ec32071286e7c9c072fbd67a8c646a73536a
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Tue Dec 20 19:50:49 2022 +0800
Add more metrics
---
.../iotdb/db/engine/storagegroup/DataRegion.java | 5 ++++
.../iotdb/db/mpp/aggregation/Aggregator.java | 3 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 33 +++++++++++++++-------
.../AbstractSeriesAggregationScanOperator.java | 20 +++++++++----
.../execution/operator/source/SeriesScanUtil.java | 4 ++-
.../iotdb/db/mpp/statistics/QueryStatistics.java | 9 ++++++
.../iotdb/tsfile/read/filter/GroupByFilter.java | 18 +-----------
7 files changed, 57 insertions(+), 35 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 353f546c92..5b5a88b0c0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -1745,11 +1745,16 @@ public class DataRegion implements IDataRegionForQuery {
}
}
+ boolean meet = false;
for (TsFileResource tsFileResource : tsFileResources) {
if (!tsFileResource.isSatisfied(
singleDeviceId, timeFilter, isSeq, dataTTL, context.isDebug())) {
+ if (meet) {
+ break;
+ }
continue;
}
+ meet = true;
closeQueryLock.readLock().lock();
try {
if (tsFileResource.isClosed()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/aggregation/Aggregator.java b/server/src/main/java/org/apache/iotdb/db/mpp/aggregation/Aggregator.java
index c893e4015f..dd2896556d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/aggregation/Aggregator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/aggregation/Aggregator.java
@@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.List;
import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.AGGREGATOR_PROCESS_TSBLOCK;
public class Aggregator {
@@ -80,7 +81,7 @@ public class Aggregator {
Math.max(lastReadReadIndex, accumulator.addInput(controlTimeAndValueColumn, curWindow));
}
QueryStatistics.getInstance()
- .addCost("AggregationScan: calcFromRawData", System.nanoTime() - startTime);
+ .addCost(AGGREGATOR_PROCESS_TSBLOCK, System.nanoTime() - startTime);
return lastReadReadIndex;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
index f766943236..893cdcd063 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/DataDriver.java
@@ -40,7 +40,10 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.ADD_REFERENCE;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.INIT_SOURCE_OP;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.QUERY_RESOURCE_INIT;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.QUERY_RESOURCE_LIST;
/**
* One dataDriver is responsible for one FragmentInstance which is for data query, which may
@@ -105,16 +108,19 @@ public class DataDriver extends Driver {
((DataDriverContext) driverContext).getSourceOperators();
if (sourceOperators != null && !sourceOperators.isEmpty()) {
QueryDataSource dataSource = initQueryDataSource();
- sourceOperators.forEach(
- sourceOperator -> {
- // construct QueryDataSource for source operator
- QueryDataSource queryDataSource =
- new QueryDataSource(dataSource.getSeqResources(), dataSource.getUnseqResources());
+ long start = System.nanoTime();
+ for (DataSourceOperator sourceOperator : sourceOperators) {
+ // construct QueryDataSource for source operator
+ QueryDataSource queryDataSource =
+ new QueryDataSource(dataSource.getSeqResources(), dataSource.getUnseqResources());
- queryDataSource.setDataTTL(dataSource.getDataTTL());
+ queryDataSource.setDataTTL(dataSource.getDataTTL());
- sourceOperator.initQueryDataSource(queryDataSource);
- });
+ sourceOperator.initQueryDataSource(queryDataSource);
+ }
+ driverContext
+ .getFragmentInstanceContext()
+ .addOperationTime(INIT_SOURCE_OP, System.nanoTime() - start);
}
this.init = true;
@@ -142,16 +148,24 @@ public class DataDriver extends Driver {
pathList.stream().map(PartialPath::getDevice).collect(Collectors.toSet());
Filter timeFilter = context.getTimeFilter();
+ long startTime = System.nanoTime();
QueryDataSource dataSource =
dataRegion.query(
pathList,
selectedDeviceIdSet.size() == 1 ? selectedDeviceIdSet.iterator().next() : null,
driverContext.getFragmentInstanceContext(),
timeFilter != null ? timeFilter.copy() : null);
+ driverContext
+ .getFragmentInstanceContext()
+ .addOperationTime(QUERY_RESOURCE_LIST, System.nanoTime() - startTime);
// used files should be added before mergeLock is unlocked, or they may be deleted by
// running merge
+ startTime = System.nanoTime();
addUsedFilesForQuery(dataSource);
+ driverContext
+ .getFragmentInstanceContext()
+ .addOperationTime(ADD_REFERENCE, System.nanoTime() - startTime);
return dataSource;
} finally {
@@ -196,8 +210,7 @@ public class DataDriver extends Driver {
*/
private void addFilePathToMap(TsFileResource tsFile, boolean isClosed) {
Set<TsFileResource> pathSet = isClosed ? closedFilePaths : unClosedFilePaths;
- if (!pathSet.contains(tsFile)) {
- pathSet.add(tsFile);
+ if (pathSet.add(tsFile)) {
FileReaderManager.getInstance().increaseFileReaderReference(tsFile, isClosed);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index aeaf19b3a1..d398966eae 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -43,6 +43,7 @@ import static org.apache.iotdb.db.mpp.execution.operator.AggregationUtil.appendA
import static org.apache.iotdb.db.mpp.execution.operator.AggregationUtil.calculateAggregationFromRawData;
import static org.apache.iotdb.db.mpp.execution.operator.AggregationUtil.isAllAggregatorsHasFinalResult;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.AGG_SCAN_OPERATOR;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.BUILD_AGG_RES;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_CHUNK;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_FILE;
import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.CAL_AGG_FROM_PAGE;
@@ -70,7 +71,9 @@ public abstract class AbstractSeriesAggregationScanOperator implements DataSourc
protected final List<Aggregator> aggregators;
// using for building result tsBlock
- protected final TsBlockBuilder resultTsBlockBuilder;
+ private TsBlockBuilder resultTsBlockBuilder;
+
+ private final List<TSDataType> dataTypes;
protected boolean finished = false;
@@ -96,11 +99,10 @@ public abstract class AbstractSeriesAggregationScanOperator implements DataSourc
this.aggregators = aggregators;
this.timeRangeIterator = timeRangeIterator;
- List<TSDataType> dataTypes = new ArrayList<>();
+ this.dataTypes = new ArrayList<>();
for (Aggregator aggregator : aggregators) {
dataTypes.addAll(Arrays.asList(aggregator.getOutputType()));
}
- this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
this.maxRetainedSize =
(1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
@@ -144,6 +146,12 @@ public abstract class AbstractSeriesAggregationScanOperator implements DataSourc
@Override
public TsBlock next() {
+ if (resultTsBlockBuilder == null) {
+ resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
+ } else {
+ // lazy reset until next `next` call
+ resultTsBlockBuilder.reset();
+ }
// start stopwatch
long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
long start = System.nanoTime();
@@ -166,9 +174,7 @@ public abstract class AbstractSeriesAggregationScanOperator implements DataSourc
}
if (resultTsBlockBuilder.getPositionCount() > 0) {
- TsBlock resultTsBlock = resultTsBlockBuilder.build();
- resultTsBlockBuilder.reset();
- return resultTsBlock;
+ return resultTsBlockBuilder.build();
} else {
return null;
}
@@ -214,8 +220,10 @@ public abstract class AbstractSeriesAggregationScanOperator implements DataSourc
}
protected void updateResultTsBlock() {
+ long startTime = System.nanoTime();
appendAggregationResult(
resultTsBlockBuilder, aggregators, timeRangeIterator.currentOutputTime());
+ operatorContext.addOperatorTime(BUILD_AGG_RES, System.nanoTime() - startTime);
}
protected boolean calcFromCachedData() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
index 160a794230..49ced2de96 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java
@@ -160,7 +160,9 @@ public class SeriesScanUtil {
}
public void initQueryDataSource(QueryDataSource dataSource) {
- QueryUtils.fillOrderIndexes(dataSource, seriesPath.getDevice(), orderUtils.getAscending());
+ if (!dataSource.getUnseqResources().isEmpty()) {
+ QueryUtils.fillOrderIndexes(dataSource, seriesPath.getDevice(), orderUtils.getAscending());
+ }
this.dataSource = dataSource;
this.timeFilter = dataSource.updateFilterUsingTTL(timeFilter);
if (this.valueFilter != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 0f22ada07b..043dde1a72 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -62,6 +62,11 @@ public class QueryStatistics {
public static final String QUERY_RESOURCE_INIT = "QueryResourceInit";
+ public static final String INIT_SOURCE_OP = "InitSourceOp";
+
+ public static final String QUERY_RESOURCE_LIST = "TsFileList";
+ public static final String ADD_REFERENCE = "AddRef";
+
public static final String LOCAL_SOURCE_HANDLE_GET_TSBLOCK = "LocalSourceHandleGetTsBlock";
public static final String LOCAL_SOURCE_HANDLE_SER_TSBLOCK = "LocalSourceHandleSerializeTsBlock";
@@ -83,12 +88,16 @@ public class QueryStatistics {
public static final String CAL_AGG_FROM_RAW_DATA = "CalcAggFromRawData";
+ public static final String AGGREGATOR_PROCESS_TSBLOCK = "AggProcTsBlock";
+
public static final String CAL_AGG_FROM_PAGE = "CalcAggFromPage";
public static final String CAL_AGG_FROM_CHUNK = "CalcAggFromChunk";
public static final String CAL_AGG_FROM_FILE = "CalcAggFromFile";
+ public static final String BUILD_AGG_RES = "BuildAggRes";
+
public static final String FILTER_AND_PROJECT_OPERATOR = "FilterAndProjectOperator";
public static final String SINGLE_INPUT_AGG_OPERATOR = "SingleInputAggregationOperator";
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
index 726f112c61..2bd7c14c2d 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
@@ -60,23 +60,7 @@ public class GroupByFilter implements Filter, Serializable {
@Override
public boolean satisfyStartEndTime(long startTime, long endTime) {
- if (endTime < this.startTime || startTime >= this.endTime) {
- return false;
- } else if (startTime <= this.startTime) {
- return true;
- } else {
- long minTime = startTime - this.startTime;
- long count = minTime / slidingStep;
- if (minTime <= interval + count * slidingStep) {
- return true;
- } else {
- if (this.endTime <= (count + 1) * slidingStep + this.startTime) {
- return false;
- } else {
- return endTime >= (count + 1) * slidingStep + this.startTime;
- }
- }
- }
+ return startTime <= this.endTime && endTime >= this.startTime;
}
@Override