You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/01/13 08:19:50 UTC
[iotdb] branch master updated: Add query metrics doc (#8853)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e0d1cc9e08 Add query metrics doc (#8853)
e0d1cc9e08 is described below
commit e0d1cc9e08f9120d812da5828f01b17dee231abe
Author: liuminghui233 <36...@users.noreply.github.com>
AuthorDate: Fri Jan 13 16:19:43 2023 +0800
Add query metrics doc (#8853)
---
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 94 +++++++++++++---
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 94 +++++++++++++---
.../iotdb/commons/concurrent/ThreadName.java | 1 -
.../apache/iotdb/commons/conf/IoTDBConstant.java | 2 -
.../commons/service/metric/enums/Operation.java | 40 -------
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 79 --------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 30 ------
.../db/mpp/execution/exchange/LocalSinkHandle.java | 2 +-
.../mpp/execution/exchange/LocalSourceHandle.java | 4 +-
.../execution/exchange/MPPDataExchangeManager.java | 14 ++-
.../db/mpp/execution/exchange/SinkHandle.java | 6 +-
.../db/mpp/execution/exchange/SourceHandle.java | 14 ++-
...tricSet.java => DataExchangeCostMetricSet.java} | 10 +-
.../db/mpp/metric/DataExchangeCountMetricSet.java | 119 +++++++++++++++++++++
.../db/mpp/metric/QueryExecutionMetricSet.java | 5 -
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 13 +--
.../db/mpp/plan/execution/QueryExecution.java | 5 -
.../db/service/metrics/DataNodeMetricsHelper.java | 6 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 -
19 files changed, 317 insertions(+), 223 deletions(-)
diff --git a/docs/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
index f52ac121db..c0ee62faaa 100644
--- a/docs/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -167,18 +167,18 @@ Core-level metrics are enabled by default during system operation. The addition
| cache | name="DataPartition", type="all" | Counter | The access number of SDataPartition Cache |
#### 4.2.5. Interface
-| Metric | Tags | Type | Description |
-| --------------------- | ---------------------------------- | --------- | -------------------------------------------------------------- |
-| operation | name = "{{name}}" | Histogram | The time consumed of operations in client |
-| entry | name="{{interface}}" | Timer | The time consumed of thrift operations |
-| thrift_connections | name="ConfigNodeRPC" | AutoGauge | The number of thrift internal connections in ConfigNode |
-| thrift_connections | name="Internal" | AutoGauge | The number of thrift internal connections in DataNode |
-| thrift_connections | name="MPPDataExchange" | AutoGauge | The number of thrift internal connections in MPP |
-| thrift_connections | name="RPC" | AutoGauge | The number of thrift connections of Client |
-| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | The number of thrift active internal connections in ConfigNode |
-| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | The number of thrift active internal connections in DataNode |
-| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | The number of thrift active internal connections in MPP |
-| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | The number of thrift active connections of client |
+| Metric | Tags | Type | Description |
+| --------------------- | ---------------------------------------------------- | --------- | -------------------------------------------------------------- |
+| statement_execution | interface="{{interface}}", type="{{statement_type}}" | Timer | The time consumed of operations in client |
+| entry | name="{{interface}}" | Timer | The time consumed of thrift operations |
+| thrift_connections | name="ConfigNodeRPC" | AutoGauge | The number of thrift internal connections in ConfigNode |
+| thrift_connections | name="Internal" | AutoGauge | The number of thrift internal connections in DataNode |
+| thrift_connections | name="MPPDataExchange" | AutoGauge | The number of thrift internal connections in MPP |
+| thrift_connections | name="RPC" | AutoGauge | The number of thrift connections of Client |
+| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | The number of thrift active internal connections in ConfigNode |
+| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | The number of thrift active internal connections in DataNode |
+| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | The number of thrift active internal connections in MPP |
+| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | The number of thrift active connections of client |
#### 4.2.6. Memory
| Metric | Tags | Type | Description |
@@ -196,7 +196,6 @@ Core-level metrics are enabled by default during system operation. The addition
| queue | name="compaction_cross", status="running/waiting" | Gauge | The number of cross compatcion tasks |
| cost_task | name="inner_compaction/cross_compaction/flush" | Gauge | The time consumed of compaction tasks |
| queue | name="flush",status="running/waiting" | AutoGauge | The number of flush tasks |
-| queue | name="Sub_RawQuery",status="running/waiting" | AutoGauge | The number of Sub_RawQuery |
#### 4.2.8. Compaction
| Metric | Tags | Type | Description |
@@ -281,6 +280,75 @@ Core-level metrics are enabled by default during system operation. The addition
| ----------------------- | --------------------------------------------- | --------- | -------------------------------- |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | The time consumed in compilation |
+#### 4.2.17. Query Planning
+| Metric | Tags | Type | Description |
+|-----------------|------------------------------|-------|-----------------------------------------------------|
+| query_plan_cost | stage="sql_parser" | Timer | The SQL parsing time-consuming |
+| query_plan_cost | stage="analyzer" | Timer | The query statement analysis time-consuming |
+| query_plan_cost | stage="logical_planner" | Timer | The query logical plan planning time-consuming |
+| query_plan_cost | stage="distribution_planner" | Timer | The query distribution plan planning time-consuming |
+| query_plan_cost | stage="partition_fetcher" | Timer | The partition information fetching time-consuming |
+| query_plan_cost | stage="schema_fetcher" | Timer | The schema information fetching time-consuming |
+
+#### 4.2.18. Plan Dispatcher
+| Metric | Tags | Type | Description |
+|------------|---------------------------|-------|--------------------------------------------------------------|
+| dispatcher | stage="wait_for_dispatch" | Timer | The distribution plan dispatcher time-consuming |
+| dispatcher | stage="dispatch_read" | Timer | The distribution plan dispatcher time-consuming (only query) |
+
+#### 4.2.19. Query Resource
+| Metric | Tags | Type | Description |
+|----------------|--------------------------|------|--------------------------------------------|
+| query_resource | type="sequence_tsfile" | Rate | The access frequency of sequence tsfiles |
+| query_resource | type="unsequence_tsfile" | Rate | The access frequency of unsequence tsfiles |
+| query_resource | type="flushing_memtable" | Rate | The access frequency of flushing memtables |
+| query_resource | type="working_memtable" | Rate | The access frequency of working memtables |
+
+#### 4.2.20. Data Exchange
+| Metric | Tags | Type | Description |
+|---------------------|------------------------------------------------------------------------|-----------|-----------------------------------------------------------------|
+| data_exchange_cost | operation="source_handle_get_tsblock", type="local/remote" | Timer | The time-consuming that source handles receive TsBlock |
+| data_exchange_cost | operation="source_handle_deserialize_tsblock", type="local/remote" | Timer | The time-consuming that source handles deserialize TsBlock |
+| data_exchange_cost | operation="sink_handle_send_tsblock", type="local/remote" | Timer | The time-consuming that sink handles send TsBlock |
+| data_exchange_cost | operation="send_new_data_block_event_task", type="server/caller" | Timer | The RPC time-consuming that sink handles send TsBlock |
+| data_exchange_cost | operation="get_data_block_task", type="server/caller" | Timer | The RPC time-consuming that source handles receive TsBlock |
+| data_exchange_cost | operation="on_acknowledge_data_block_event_task", type="server/caller" | Timer | The RPC time-consuming that source handles ack received TsBlock |
+| data_exchange_count | name="send_new_data_block_num", type="server/caller" | Histogram | The number of sent TsBlocks by sink handles |
+| data_exchange_count | name="get_data_block_num", type="server/caller" | Histogram | The number of received TsBlocks by source handles |
+| data_exchange_count | name="on_acknowledge_data_block_num", type="server/caller" | Histogram | The number of acknowledged TsBlocks by source handles |
+
+#### 4.2.21. Query Task Schedule
+| Metric | Tags | Type | Description |
+|------------------|--------------------------------|-----------|--------------------------------------------------|
+| driver_scheduler | name="ready_queued_time" | Timer | The queuing time of ready queue |
+| driver_scheduler | name="block_queued_time" | Timer | The queuing time of blocking queue |
+| driver_scheduler | name="ready_queue_task_count" | AutoGauge | The number of tasks queued in the ready queue |
+| driver_scheduler | name="block_queued_task_count" | AutoGauge | The number of tasks queued in the blocking queue |
+
+#### 4.2.22. Query Execution
+| Metric | Tags | Type | Description |
+|--------------------------|-------------------------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------|
+| query_execution | stage="local_execution_planner" | Timer | The time-consuming of operator tree construction |
+| query_execution | stage="query_resource_init" | Timer | The time-consuming of query resource initialization |
+| query_execution | stage="get_query_resource_from_mem" | Timer | The time-consuming of query resource memory query and construction |
+| query_execution | stage="driver_internal_process" | Timer | The time-consuming of driver execution |
+| query_execution | stage="wait_for_result" | Timer | The time-consuming of getting query result from result handle |
+| operator_execution_cost | name="{{operator_name}}" | Timer | The operator execution time |
+| operator_execution_count | name="{{operator_name}}" | Counter | The number of operator calls (counted by the number of next method calls) |
+| aggregation | from="raw_data" | Timer | The time-consuming of performing an aggregation calculation from a batch of raw data |
+| aggregation | from="statistics" | Timer | The time-consuming of updating an aggregated value with statistics |
+| series_scan_cost | stage="load_timeseries_metadata", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of loading TimeseriesMetadata |
+| series_scan_cost | stage="read_timeseries_metadata", type="", from="cache/file" | Timer | The time-consuming of reading TimeseriesMetadata of a tsfile |
+| series_scan_cost | stage="timeseries_metadata_modification", type="aligned/non_aligned", from="null" | Timer | The time-consuming of filtering TimeseriesMetadata by mods |
+| series_scan_cost | stage="load_chunk_metadata_list", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of loading ChunkMetadata list |
+| series_scan_cost | stage="chunk_metadata_modification", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of filtering ChunkMetadata by mods |
+| series_scan_cost | stage="chunk_metadata_filter", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of filtering ChunkMetadata by query filter |
+| series_scan_cost | stage="construct_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of constructing ChunkReader |
+| series_scan_cost | stage="read_chunk", type="", from="cache/file" | Timer | The time-consuming of reading Chunk |
+| series_scan_cost | stage="init_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of initializing ChunkReader (constructing PageReader) |
+| series_scan_cost | stage="build_tsblock_from_page_reader", type="aligned/non_aligned", from="mem/disk" | Timer | The time-consuming of constructing Tsblock from PageReader |
+| series_scan_cost | stage="build_tsblock_from_merge_reader", type="aligned/non_aligned", from="null" | Timer | The time-consuming of constructing Tsblock from MergeReader (handling overlapping data) |
+
### 4.3. Normal level Metrics
#### 4.3.1. Cluster
diff --git a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
index 31dfbd6173..cdc4a349c3 100644
--- a/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
+++ b/docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md
@@ -163,18 +163,18 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| cache | name="DataPartition", type="all" | Counter | DataPartition Cache 的访问次数 |
#### 4.2.5. 接口层统计
-| Metric | Tags | Type | Description |
-| --------------------- | ---------------------------------- | --------- | ----------------------------------- |
-| operation | name = "{{name}}" | Histogram | 客户端执行的操作的耗时情况 |
-| entry | name="{{interface}}" | Timer | Client 建立的 Thrift 的耗时情况 |
-| thrift_connections | name="ConfigNodeRPC" | AutoGauge | ConfigNode 的内部 Thrift 连接数 |
-| thrift_connections | name="Internal" | AutoGauge | DataNode 的内部 Thrift 连接数 |
-| thrift_connections | name="MPPDataExchange" | AutoGauge | MPP 框架的内部 Thrift 连接数 |
-| thrift_connections | name="RPC" | AutoGauge | Client 建立的 Thrift 连接数 |
-| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | ConfigNode 的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | DataNode 的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | MPP 框架的内部活跃 Thrift 连接数 |
-| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | Client 建立的活跃 Thrift 连接数 |
+| Metric | Tags | Type | Description |
+| --------------------- | ---------------------------------------------------- | --------- | ----------------------------------- |
+| statement_execution | interface="{{interface}}", type="{{statement_type}}" | Timer | 客户端执行的操作的耗时情况 |
+| entry | name="{{interface}}" | Timer | Client 建立的 Thrift 的耗时情况 |
+| thrift_connections | name="ConfigNodeRPC" | AutoGauge | ConfigNode 的内部 Thrift 连接数 |
+| thrift_connections | name="Internal" | AutoGauge | DataNode 的内部 Thrift 连接数 |
+| thrift_connections | name="MPPDataExchange" | AutoGauge | MPP 框架的内部 Thrift 连接数 |
+| thrift_connections | name="RPC" | AutoGauge | Client 建立的 Thrift 连接数 |
+| thrift_active_threads | name="ConfigNodeRPC-Service" | AutoGauge | ConfigNode 的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="DataNodeInternalRPC-Service" | AutoGauge | DataNode 的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="MPPDataExchangeRPC-Service" | AutoGauge | MPP 框架的内部活跃 Thrift 连接数 |
+| thrift_active_threads | name="ClientRPC-Service" | AutoGauge | Client 建立的活跃 Thrift 连接数 |
#### 4.2.6. 内存统计
| Metric | Tags | Type | Description |
@@ -192,7 +192,6 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| queue | name="compaction_cross", status="running/waiting" | Gauge | 跨空间合并任务数 |
| cost_task | name="inner_compaction/cross_compaction/flush" | Gauge | 任务耗时情况 |
| queue | name="flush",status="running/waiting" | AutoGauge | 刷盘任务数 |
-| queue | name="Sub_RawQuery",status="running/waiting" | AutoGauge | Sub_RawQuery任务数 |
#### 4.2.8. 合并统计
| Metric | Tags | Type | Description |
@@ -276,6 +275,75 @@ Core 级别的监控指标在系统运行中默认开启，每一个 Core 级别
| ----------------------- | --------------------------------------------- | --------- | ------------------ |
| jvm_compilation_time_ms | {compiler="HotSpot 64-Bit Tiered Compilers",} | AutoGauge | 耗费在编译上的时间 |
+#### 4.2.17. 查询规划耗时统计
+| Metric | Tags | Type | Description |
+|-----------------|------------------------------|-------|----------------------------|
+| query_plan_cost | stage="sql_parser" | Timer | SQL 解析耗时 |
+| query_plan_cost | stage="analyzer" | Timer | 查询语句分析耗时 |
+| query_plan_cost | stage="logical_planner" | Timer | 查询逻辑计划规划耗时 |
+| query_plan_cost | stage="distribution_planner" | Timer | 查询分布式执行计划规划耗时 |
+| query_plan_cost | stage="partition_fetcher" | Timer | 分区信息拉取耗时 |
+| query_plan_cost | stage="schema_fetcher" | Timer | 元数据信息拉取耗时 |
+
+#### 4.2.18. 执行计划分发耗时统计
+| Metric | Tags | Type | Description |
+|------------|---------------------------|-------|----------------------|
+| dispatcher | stage="wait_for_dispatch" | Timer | 分发执行计划耗时 |
+| dispatcher | stage="dispatch_read" | Timer | 查询执行计划发送耗时 |
+
+#### 4.2.19. 查询资源访问统计
+| Metric | Tags | Type | Description |
+|----------------|--------------------------|------|----------------------------|
+| query_resource | type="sequence_tsfile" | Rate | 顺序文件访问频率 |
+| query_resource | type="unsequence_tsfile" | Rate | 乱序文件访问频率 |
+| query_resource | type="flushing_memtable" | Rate | flushing memtable 访问频率 |
+| query_resource | type="working_memtable" | Rate | working memtable 访问频率 |
+
+#### 4.2.20. 数据传输模块统计
+| Metric | Tags | Type | Description |
+|---------------------|------------------------------------------------------------------------|-----------|-----------------------------------------|
+| data_exchange_cost | operation="source_handle_get_tsblock", type="local/remote" | Timer | source handle 接收 TsBlock 耗时 |
+| data_exchange_cost | operation="source_handle_deserialize_tsblock", type="local/remote" | Timer | source handle 反序列化 TsBlock 耗时 |
+| data_exchange_cost | operation="sink_handle_send_tsblock", type="local/remote" | Timer | sink handle 发送 TsBlock 耗时 |
+| data_exchange_cost | operation="send_new_data_block_event_task", type="server/caller" | Timer | sink handle 发送 TsBlock RPC 耗时 |
+| data_exchange_cost | operation="get_data_block_task", type="server/caller" | Timer | source handle 接收 TsBlock RPC 耗时 |
+| data_exchange_cost | operation="on_acknowledge_data_block_event_task", type="server/caller" | Timer | source handle 确认接收 TsBlock RPC 耗时 |
+| data_exchange_count | name="send_new_data_block_num", type="server/caller" | Histogram | sink handle 发送 TsBlock数量 |
+| data_exchange_count | name="get_data_block_num", type="server/caller" | Histogram | source handle 接收 TsBlock 数量 |
+| data_exchange_count | name="on_acknowledge_data_block_num", type="server/caller" | Histogram | source handle 确认接收 TsBlock 数量 |
+
+#### 4.2.21. 查询任务调度统计
+| Metric | Tags | Type | Description |
+|------------------|--------------------------------|-----------|--------------------|
+| driver_scheduler | name="ready_queued_time" | Timer | 就绪队列排队时间 |
+| driver_scheduler | name="block_queued_time" | Timer | 阻塞队列排队时间 |
+| driver_scheduler | name="ready_queue_task_count" | AutoGauge | 就绪队列排队任务数 |
+| driver_scheduler | name="block_queued_task_count" | AutoGauge | 阻塞队列排队任务数 |
+
+#### 4.2.22. 查询执行耗时统计
+| Metric | Tags | Type | Description |
+|--------------------------|-------------------------------------------------------------------------------------|---------|------------------------------------------------|
+| query_execution | stage="local_execution_planner" | Timer | 算子树构造耗时 |
+| query_execution | stage="query_resource_init" | Timer | 查询资源初始化耗时 |
+| query_execution | stage="get_query_resource_from_mem" | Timer | 查询资源内存查询与构造耗时 |
+| query_execution | stage="driver_internal_process" | Timer | Driver 执行耗时 |
+| query_execution | stage="wait_for_result" | Timer | 从resultHandle 获取一次查询结果的耗时 |
+| operator_execution_cost | name="{{operator_name}}" | Timer | 算子执行耗时 |
+| operator_execution_count | name="{{operator_name}}" | Counter | 算子调用次数（以 next 方法调用次数计算） |
+| aggregation | from="raw_data" | Timer | 从一批原始数据进行一次聚合计算的耗时 |
+| aggregation | from="statistics" | Timer | 使用统计信息更新一次聚合值的耗时 |
+| series_scan_cost | stage="load_timeseries_metadata", type="aligned/non_aligned", from="mem/disk" | Timer | 加载 TimeseriesMetadata 耗时 |
+| series_scan_cost | stage="read_timeseries_metadata", type="", from="cache/file" | Timer | 读取一个文件的 Metadata 耗时 |
+| series_scan_cost | stage="timeseries_metadata_modification", type="aligned/non_aligned", from="null" | Timer | 过滤删除的 TimeseriesMetadata 耗时 |
+| series_scan_cost | stage="load_chunk_metadata_list", type="aligned/non_aligned", from="mem/disk" | Timer | 加载 ChunkMetadata 列表耗时 |
+| series_scan_cost | stage="chunk_metadata_modification", type="aligned/non_aligned", from="mem/disk" | Timer | 过滤删除的 ChunkMetadata 耗时 |
+| series_scan_cost | stage="chunk_metadata_filter", type="aligned/non_aligned", from="mem/disk" | Timer | 根据查询过滤条件过滤 ChunkMetadata 耗时 |
+| series_scan_cost | stage="construct_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 构造 ChunkReader 耗时 |
+| series_scan_cost | stage="read_chunk", type="", from="cache/file" | Timer | 读取 Chunk 的耗时 |
+| series_scan_cost | stage="init_chunk_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 初始化 ChunkReader（构造 PageReader） 耗时 |
+| series_scan_cost | stage="build_tsblock_from_page_reader", type="aligned/non_aligned", from="mem/disk" | Timer | 从 PageReader 构造 Tsblock 耗时 |
+| series_scan_cost | stage="build_tsblock_from_merge_reader", type="aligned/non_aligned", from="null" | Timer | 从 MergeReader 构造 Tsblock （解乱序数据）耗时 |
+
### 4.3. Normal 级别监控指标
#### 4.3.1. 集群
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
index f0d3bac43a..9cdf23ae5a 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
@@ -45,7 +45,6 @@ public enum ThreadName {
SYNC_CLIENT("Sync-Client"),
SYNC_SERVER("Sync"),
QUERY_SERVICE("Query"),
- SUB_RAW_QUERY_SERVICE("Sub_RawQuery"),
INSERTION_SERVICE("MultithreadingInsertionPool"),
WINDOW_EVALUATION_SERVICE("WindowEvaluationTaskPoolManager"),
TTL_CHECK_SERVICE("TTL-CHECK"),
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index ca68d8283c..eb019d7cab 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -220,8 +220,6 @@ public class IoTDBConstant {
public static final String LOAD_TSFILE_FOLDER_NAME = "load";
public static final String SYNC_FOLDER_NAME = "sync";
public static final String QUERY_FOLDER_NAME = "query";
- public static final String TRACING_FOLDER_NAME = "tracing";
- public static final String TRACING_LOG = "tracing.txt";
public static final String EXT_FOLDER_NAME = "ext";
public static final String UDF_FOLDER_NAME = "udf";
public static final String TRIGGER_FOLDER_NAME = "trigger";
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java
deleted file mode 100644
index d4a8651e5e..0000000000
--- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Operation.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.commons.service.metric.enums;
-
-public enum Operation {
- EXECUTE_JDBC_BATCH("EXECUTE_JDBC_BATCH"),
- EXECUTE_ONE_SQL_IN_BATCH("EXECUTE_ONE_SQL_IN_BATCH"),
- EXECUTE_ROWS_PLAN_IN_BATCH("EXECUTE_ROWS_PLAN_IN_BATCH"),
- EXECUTE_MULTI_TIMESERIES_PLAN_IN_BATCH("EXECUTE_MULTI_TIMESERIES_PLAN_IN_BATCH"),
- EXECUTE_RPC_BATCH_INSERT("EXECUTE_RPC_BATCH_INSERT"),
- EXECUTE_QUERY("EXECUTE_QUERY"),
- EXECUTE_SELECT_INTO("EXECUTE_SELECT_INTO");
-
- public String getName() {
- return name;
- }
-
- String name;
-
- Operation(String name) {
- this.name = name;
- }
-}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index acafb58bb7..e5092a0b1f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -251,10 +251,6 @@ public class IoTDBConfig {
+ File.separator
+ IoTDBConstant.SCHEMA_FOLDER_NAME;
- /** Performance tracing directory, stores performance tracing files */
- private String tracingDir =
- IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.TRACING_FOLDER_NAME;
-
/** Query directory, stores temporary files of query */
private String queryDir =
IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.QUERY_FOLDER_NAME;
@@ -325,14 +321,6 @@ public class IoTDBConfig {
/** How many queries can be concurrently executed. When <= 0, use 1000. */
private int maxAllowedConcurrentQueries = 1000;
- /**
- * How many threads can concurrently read data for raw data query. When <= 0, use CPU core number.
- */
- private int subRawQueryThreadCount = 8;
-
- /** Blocking queue size for read task in raw data query. */
- private int rawQueryBlockingQueueCapacity = 5;
-
/** How many threads can concurrently evaluate windows. When <= 0, use CPU core number. */
private int windowEvaluationThreadCount = Runtime.getRuntime().availableProcessors();
@@ -530,24 +518,9 @@ public class IoTDBConfig {
/** Whether to enable Last cache */
private boolean lastCacheEnable = true;
- /** Set true to enable statistics monitor service, false to disable statistics service. */
- private boolean enableStatMonitor = false;
-
- /** Set true to enable writing monitor time series. */
- private boolean enableMonitorSeriesWrite = false;
-
/** Cache size of {@code checkAndGetDataTypeCache}. */
private int mRemoteSchemaCacheSize = 100000;
- /** Is external sort enable. */
- private boolean enableExternalSort = true;
-
- /**
- * The threshold of items in external sort. If the number of chunks participating in sorting
- * exceeds this threshold, external sorting is enabled, otherwise memory sorting is used.
- */
- private int externalSortThreshold = 1000;
-
/** White list for sync */
private String ipWhiteList = "127.0.0.1/32";
@@ -792,9 +765,6 @@ public class IoTDBConfig {
private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB);
- /** The cached record size (in MB) of each series in group by fill query */
- private float groupByFillCacheSizeInMB = (float) 1.0;
-
// time in nanosecond precision when starting up
private long startUpNanosecond = System.nanoTime();
@@ -1085,14 +1055,6 @@ public class IoTDBConfig {
this.udfMemoryBudgetInMB = udfMemoryBudgetInMB;
}
- public float getGroupByFillCacheSizeInMB() {
- return groupByFillCacheSizeInMB;
- }
-
- public void setGroupByFillCacheSizeInMB(float groupByFillCacheSizeInMB) {
- this.groupByFillCacheSizeInMB = groupByFillCacheSizeInMB;
- }
-
public float getUdfReaderMemoryBudgetInMB() {
return udfReaderMemoryBudgetInMB;
}
@@ -1168,7 +1130,6 @@ public class IoTDBConfig {
systemDir = addDataHomeDir(systemDir);
schemaDir = addDataHomeDir(schemaDir);
loadTsFileDir = addDataHomeDir(loadTsFileDir);
- tracingDir = addDataHomeDir(tracingDir);
consensusDir = addDataHomeDir(consensusDir);
dataRegionConsensusDir = addDataHomeDir(dataRegionConsensusDir);
ratisDataRegionSnapshotDir = addDataHomeDir(ratisDataRegionSnapshotDir);
@@ -1362,14 +1323,6 @@ public class IoTDBConfig {
this.schemaDir = schemaDir;
}
- public String getTracingDir() {
- return tracingDir;
- }
-
- void setTracingDir(String tracingDir) {
- this.tracingDir = tracingDir;
- }
-
public String getQueryDir() {
return queryDir;
}
@@ -1524,14 +1477,6 @@ public class IoTDBConfig {
this.maxAllowedConcurrentQueries = maxAllowedConcurrentQueries;
}
- public int getSubRawQueryThreadCount() {
- return subRawQueryThreadCount;
- }
-
- void setSubRawQueryThreadCount(int subRawQueryThreadCount) {
- this.subRawQueryThreadCount = subRawQueryThreadCount;
- }
-
public long getMaxBytesPerFragmentInstance() {
return maxBytesPerFragmentInstance;
}
@@ -1541,14 +1486,6 @@ public class IoTDBConfig {
this.maxBytesPerFragmentInstance = maxBytesPerFragmentInstance;
}
- public int getRawQueryBlockingQueueCapacity() {
- return rawQueryBlockingQueueCapacity;
- }
-
- public void setRawQueryBlockingQueueCapacity(int rawQueryBlockingQueueCapacity) {
- this.rawQueryBlockingQueueCapacity = rawQueryBlockingQueueCapacity;
- }
-
public int getWindowEvaluationThreadCount() {
return windowEvaluationThreadCount;
}
@@ -1883,22 +1820,6 @@ public class IoTDBConfig {
- allocateMemoryForSchema;
}
- public boolean isEnableExternalSort() {
- return enableExternalSort;
- }
-
- void setEnableExternalSort(boolean enableExternalSort) {
- this.enableExternalSort = enableExternalSort;
- }
-
- public int getExternalSortThreshold() {
- return externalSortThreshold;
- }
-
- void setExternalSortThreshold(int externalSortThreshold) {
- this.externalSortThreshold = externalSortThreshold;
- }
-
public boolean isEnablePartialInsert() {
return enablePartialInsert;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index b8ef20d717..128ea8233c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -331,8 +331,6 @@ public class IoTDBDescriptor {
conf.setQueryDir(
FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME));
- conf.setTracingDir(properties.getProperty("dn_tracing_dir", conf.getTracingDir()));
-
conf.setDataDirs(properties.getProperty("dn_data_dirs", conf.getDataDirs()[0]).split(","));
conf.setConsensusDir(properties.getProperty("dn_consensus_dir", conf.getConsensusDir()));
@@ -553,21 +551,6 @@ public class IoTDBDescriptor {
conf.setMaxAllowedConcurrentQueries(1000);
}
- conf.setSubRawQueryThreadCount(
- Integer.parseInt(
- properties.getProperty(
- "sub_rawQuery_thread_count", Integer.toString(conf.getSubRawQueryThreadCount()))));
-
- if (conf.getSubRawQueryThreadCount() <= 0) {
- conf.setSubRawQueryThreadCount(Runtime.getRuntime().availableProcessors());
- }
-
- conf.setRawQueryBlockingQueueCapacity(
- Integer.parseInt(
- properties.getProperty(
- "raw_query_blocking_queue_capacity",
- Integer.toString(conf.getRawQueryBlockingQueueCapacity()))));
-
conf.setmRemoteSchemaCacheSize(
Integer.parseInt(
properties
@@ -583,14 +566,6 @@ public class IoTDBDescriptor {
Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable")));
}
- conf.setEnableExternalSort(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_external_sort", Boolean.toString(conf.isEnableExternalSort()))));
- conf.setExternalSortThreshold(
- Integer.parseInt(
- properties.getProperty(
- "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold()))));
conf.setUpgradeThreadCount(
Integer.parseInt(
properties.getProperty(
@@ -1713,11 +1688,6 @@ public class IoTDBDescriptor {
Math.min(Float.parseFloat(memoryBudgetInMb), 0.2 * conf.getAllocateMemoryForRead()));
}
- String groupByFillCacheSizeInMB = properties.getProperty("group_by_fill_cache_size_in_mb");
- if (groupByFillCacheSizeInMB != null) {
- conf.setGroupByFillCacheSizeInMB(Float.parseFloat(groupByFillCacheSizeInMB));
- }
-
String readerTransformerCollectorMemoryProportion =
properties.getProperty("udf_reader_transformer_collector_memory_proportion");
if (readerTransformerCollectorMemoryProportion != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
index 0a1d666409..8afb07fe6e 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSinkHandle.java
@@ -33,7 +33,7 @@ import java.util.List;
import java.util.Optional;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SINK_HANDLE_SEND_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SINK_HANDLE_SEND_TSBLOCK_LOCAL;
public class LocalSinkHandle implements ISinkHandle {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
index 6487a20399..f47dea04d6 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/LocalSourceHandle.java
@@ -37,8 +37,8 @@ import java.nio.ByteBuffer;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeManager.createFullIdFrom;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_LOCAL;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_GET_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_LOCAL;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_GET_TSBLOCK_LOCAL;
public class LocalSourceHandle implements ISourceHandle {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
index 885447d0c0..b3d89c0340 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/MPPDataExchangeManager.java
@@ -54,9 +54,12 @@ import java.util.concurrent.ExecutorService;
import java.util.function.Supplier;
import static org.apache.iotdb.db.mpp.common.FragmentInstanceId.createFullId;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_TASK_SERVER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.GET_DATA_BLOCK_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.GET_DATA_BLOCK_NUM_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.SEND_NEW_DATA_BLOCK_NUM_SERVER;
public class MPPDataExchangeManager implements IMPPDataExchangeManager {
@@ -118,6 +121,8 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
GET_DATA_BLOCK_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ GET_DATA_BLOCK_NUM_SERVER, req.getEndSequenceId() - req.getStartSequenceId());
}
}
@@ -150,6 +155,8 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER, e.getEndSequenceId() - e.getStartSequenceId());
}
}
@@ -192,6 +199,7 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager {
} finally {
QUERY_METRICS.recordDataExchangeCost(
SEND_NEW_DATA_BLOCK_EVENT_TASK_SERVER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(SEND_NEW_DATA_BLOCK_NUM_SERVER, e.getBlockSizes().size());
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
index e2494af0fb..139387d510 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SinkHandle.java
@@ -51,8 +51,9 @@ import java.util.concurrent.ExecutorService;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.common.FragmentInstanceId.createFullId;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SINK_HANDLE_SEND_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SINK_HANDLE_SEND_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.SEND_NEW_DATA_BLOCK_NUM_CALLER;
import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
public class SinkHandle implements ISinkHandle {
@@ -428,6 +429,7 @@ public class SinkHandle implements ISinkHandle {
} finally {
QUERY_METRICS.recordDataExchangeCost(
SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(SEND_NEW_DATA_BLOCK_NUM_CALLER, blockSizes.size());
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
index 2e1a4b8a6d..d164e9d1de 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/exchange/SourceHandle.java
@@ -51,10 +51,12 @@ import java.util.concurrent.ExecutorService;
import static com.google.common.util.concurrent.Futures.nonCancellationPropagating;
import static org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeManager.createFullIdFrom;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_REMOTE;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.SOURCE_HANDLE_GET_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.GET_DATA_BLOCK_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_DESERIALIZE_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet.SOURCE_HANDLE_GET_TSBLOCK_REMOTE;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.GET_DATA_BLOCK_NUM_CALLER;
+import static org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet.ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER;
public class SourceHandle implements ISourceHandle {
@@ -467,7 +469,7 @@ public class SourceHandle implements ISourceHandle {
tsBlocks.addAll(resp.getTsBlocks());
logger.debug("[EndPullTsBlocksFromRemote] Count:{}", tsBlockNum);
- QUERY_METRICS.recordDataBlockNum(tsBlockNum);
+ QUERY_METRICS.recordDataBlockNum(GET_DATA_BLOCK_NUM_CALLER, tsBlockNum);
executorService.submit(
new SendAcknowledgeDataBlockEventTask(startSequenceId, endSequenceId));
synchronized (SourceHandle.this) {
@@ -581,6 +583,8 @@ public class SourceHandle implements ISourceHandle {
} finally {
QUERY_METRICS.recordDataExchangeCost(
ON_ACKNOWLEDGE_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime);
+ QUERY_METRICS.recordDataBlockNum(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER, endSequenceId - startSequenceId);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
similarity index 94%
rename from server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java
rename to server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
index 004b640d71..aab7d1ba5d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCostMetricSet.java
@@ -30,7 +30,7 @@ import org.apache.iotdb.metrics.utils.MetricType;
import java.util.HashMap;
import java.util.Map;
-public class DataExchangeMetricSet implements IMetricSet {
+public class DataExchangeCostMetricSet implements IMetricSet {
private static final String metric = Metric.DATA_EXCHANGE_COST.toString();
@@ -170,19 +170,12 @@ public class DataExchangeMetricSet implements IMetricSet {
"server"));
}
- public static final String GET_DATA_BLOCK_NUM = "get_data_block_num";
-
@Override
public void bindTo(AbstractMetricService metricService) {
for (MetricInfo metricInfo : metricInfoMap.values()) {
metricService.getOrCreateTimer(
metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
}
- metricService.getOrCreateHistogram(
- Metric.DATA_EXCHANGE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- GET_DATA_BLOCK_NUM);
}
@Override
@@ -190,6 +183,5 @@ public class DataExchangeMetricSet implements IMetricSet {
for (MetricInfo metricInfo : metricInfoMap.values()) {
metricService.remove(MetricType.TIMER, metric, metricInfo.getTagsInArray());
}
- metricService.remove(MetricType.HISTOGRAM, metric, Tag.NAME.toString(), GET_DATA_BLOCK_NUM);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
new file mode 100644
index 0000000000..153270cf79
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/DataExchangeCountMetricSet.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.metric;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricInfo;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class DataExchangeCountMetricSet implements IMetricSet {
+
+ private static final String metric = Metric.DATA_EXCHANGE_COUNT.toString();
+
+ public static final Map<String, MetricInfo> metricInfoMap = new HashMap<>();
+
+ public static final String SEND_NEW_DATA_BLOCK_NUM_CALLER = "send_new_data_block_num_caller";
+ public static final String SEND_NEW_DATA_BLOCK_NUM_SERVER = "send_new_data_block_num_server";
+ public static final String ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER =
+ "on_acknowledge_data_block_num_caller";
+ public static final String ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER =
+ "on_acknowledge_data_block_num_server";
+ public static final String GET_DATA_BLOCK_NUM_CALLER = "get_data_block_num_caller";
+ public static final String GET_DATA_BLOCK_NUM_SERVER = "get_data_block_num_server";
+
+ static {
+ metricInfoMap.put(
+ SEND_NEW_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "send_new_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ SEND_NEW_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "send_new_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ metricInfoMap.put(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "on_acknowledge_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ ON_ACKNOWLEDGE_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "on_acknowledge_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ metricInfoMap.put(
+ GET_DATA_BLOCK_NUM_CALLER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "get_data_block_num",
+ Tag.TYPE.toString(),
+ "caller"));
+ metricInfoMap.put(
+ GET_DATA_BLOCK_NUM_SERVER,
+ new MetricInfo(
+ MetricType.HISTOGRAM,
+ metric,
+ Tag.NAME.toString(),
+ "get_data_block_num",
+ Tag.TYPE.toString(),
+ "server"));
+ }
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ for (MetricInfo metricInfo : metricInfoMap.values()) {
+ metricService.getOrCreateHistogram(
+ metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
+ }
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ for (MetricInfo metricInfo : metricInfoMap.values()) {
+ metricService.remove(MetricType.HISTOGRAM, metric, metricInfo.getTagsInArray());
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
index adaf851310..2c3f5d7bc9 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
@@ -34,15 +34,10 @@ public class QueryExecutionMetricSet implements IMetricSet {
public static final Map<String, MetricInfo> metricInfoMap = new HashMap<>();
- public static final String SCHEDULE = "schedule";
public static final String WAIT_FOR_DISPATCH = "wait_for_dispatch";
public static final String DISPATCH_READ = "dispatch_read";
static {
- metricInfoMap.put(
- SCHEDULE,
- new MetricInfo(
- MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), SCHEDULE));
metricInfoMap.put(
WAIT_FOR_DISPATCH,
new MetricInfo(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
index 206125bb2e..b5bcc5be3f 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
@@ -27,8 +27,6 @@ import org.apache.iotdb.metrics.utils.MetricLevel;
import java.util.concurrent.TimeUnit;
-import static org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet.GET_DATA_BLOCK_NUM;
-
public class QueryMetricsManager {
private final MetricService metricService = MetricService.getInstance();
@@ -88,7 +86,7 @@ public class QueryMetricsManager {
}
public void recordDataExchangeCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = DataExchangeMetricSet.metricInfoMap.get(stage);
+ MetricInfo metricInfo = DataExchangeCostMetricSet.metricInfoMap.get(stage);
metricService.timer(
costTimeInNanos,
TimeUnit.NANOSECONDS,
@@ -97,13 +95,10 @@ public class QueryMetricsManager {
metricInfo.getTagsInArray());
}
- public void recordDataBlockNum(int num) {
+ public void recordDataBlockNum(String type, int num) {
+ MetricInfo metricInfo = DataExchangeCountMetricSet.metricInfoMap.get(type);
metricService.histogram(
- num,
- Metric.DATA_EXCHANGE_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- GET_DATA_BLOCK_NUM);
+ num, metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
}
public void recordTaskQueueTime(String name, long queueTimeInNanos) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
index 9cf25c433c..b0326403b0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java
@@ -81,7 +81,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static org.apache.iotdb.db.mpp.common.DataNodeEndPoints.isSameNode;
-import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.SCHEDULE;
import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.WAIT_FOR_RESULT;
import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetricSet.DISTRIBUTION_PLANNER;
@@ -274,7 +273,6 @@ public class QueryExecution implements IQueryExecution {
}
// TODO: (xingtanzjr) initialize the query scheduler according to configuration
- long startTime = System.nanoTime();
this.scheduler =
new ClusterScheduler(
context,
@@ -286,9 +284,6 @@ public class QueryExecution implements IQueryExecution {
scheduledExecutor,
internalServiceClientManager);
this.scheduler.start();
- if (rawStatement.isQuery()) {
- QUERY_METRICS.recordExecutionCost(SCHEDULE, System.nanoTime() - startTime);
- }
}
// Use LogicalPlanner to do the logical query plan and logical optimization
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index 34ded3bb24..074b7345a6 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -20,7 +20,8 @@
package org.apache.iotdb.db.service.metrics;
import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.db.mpp.metric.DataExchangeMetricSet;
+import org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet;
+import org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet;
import org.apache.iotdb.db.mpp.metric.DriverSchedulerMetricSet;
import org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet;
import org.apache.iotdb.db.mpp.metric.QueryPlanCostMetricSet;
@@ -43,7 +44,8 @@ public class DataNodeMetricsHelper {
MetricService.getInstance().addMetricSet(new SeriesScanCostMetricSet());
MetricService.getInstance().addMetricSet(new QueryExecutionMetricSet());
MetricService.getInstance().addMetricSet(new QueryResourceMetricSet());
- MetricService.getInstance().addMetricSet(new DataExchangeMetricSet());
+ MetricService.getInstance().addMetricSet(new DataExchangeCostMetricSet());
+ MetricService.getInstance().addMetricSet(new DataExchangeCountMetricSet());
MetricService.getInstance().addMetricSet(new DriverSchedulerMetricSet());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index d334e495be..0596cf07e8 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -241,8 +241,6 @@ public class EnvironmentUtils {
cleanDir(config.getSystemDir());
// delete query
cleanDir(config.getQueryDir());
- // delete tracing
- cleanDir(config.getTracingDir());
// delete ulog
cleanDir(config.getUdfDir());
// delete tlog