You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/12/14 04:33:31 UTC
(iotdb) branch beyyes/query_metrics created (now 5ccb7c7a8db)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/query_metrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 5ccb7c7a8db tmp
This branch includes the following new commits:
new 8fe5ce18a85 tmp
new 5ccb7c7a8db tmp
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/02: tmp
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/query_metrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8fe5ce18a85c2be32e823fa64ef0bc736748aa07
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Tue Dec 12 22:24:00 2023 +0800
tmp
---
.../metric/QueryExecutionMetricSet.java | 24 +++++------
.../queryengine/metric/QueryPlanCostMetricSet.java | 10 +++--
.../metric/QueryRelatedResourceMetricSet.java | 50 ++++++++++++++++++++--
.../queryengine/plan/execution/QueryExecution.java | 6 +++
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
5 files changed, 72 insertions(+), 19 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java
index 91d7857aa58..ed493ab738b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java
@@ -64,10 +64,9 @@ public class QueryExecutionMetricSet implements IMetricSet {
dispatchReadTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList(WAIT_FOR_DISPATCH, DISPATCH_READ)
.forEach(
- stage -> {
- metricService.remove(
- MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), stage);
- });
+ stage ->
+ metricService.remove(
+ MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), stage));
}
// endregion
@@ -130,10 +129,12 @@ public class QueryExecutionMetricSet implements IMetricSet {
DRIVER_INTERNAL_PROCESS,
WAIT_FOR_RESULT)
.forEach(
- stage -> {
- metricService.remove(
- MetricType.TIMER, Metric.QUERY_EXECUTION.toString(), Tag.STAGE.toString(), stage);
- });
+ stage ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.QUERY_EXECUTION.toString(),
+ Tag.STAGE.toString(),
+ stage));
}
// endregion
@@ -160,10 +161,9 @@ public class QueryExecutionMetricSet implements IMetricSet {
aggregationFromStatisticsTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList("raw_data", "statistics")
.forEach(
- from -> {
- metricService.remove(
- MetricType.TIMER, Metric.AGGREGATION.toString(), Tag.FROM.toString(), from);
- });
+ from ->
+ metricService.remove(
+ MetricType.TIMER, Metric.AGGREGATION.toString(), Tag.FROM.toString(), from));
}
// endregion
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java
index 251670bc3e4..da775515688 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java
@@ -110,10 +110,12 @@ public class QueryPlanCostMetricSet implements IMetricSet {
Arrays.asList(
ANALYZER, LOGICAL_PLANNER, DISTRIBUTION_PLANNER, PARTITION_FETCHER, SCHEMA_FETCHER)
.forEach(
- stage -> {
- metricService.remove(
- MetricType.TIMER, Metric.QUERY_PLAN_COST.toString(), Tag.STAGE.toString(), stage);
- });
+ stage ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.QUERY_PLAN_COST.toString(),
+ Tag.STAGE.toString(),
+ stage));
}
public static QueryPlanCostMetricSet getInstance() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
index b8280fe399f..93626b06290 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
@@ -27,14 +27,25 @@ import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Gauge;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
+import static org.apache.iotdb.commons.service.metric.enums.Metric.FRAGMENT_INSTANCE_STATISTICS;
+
public class QueryRelatedResourceMetricSet implements IMetricSet {
+
+ private static final QueryRelatedResourceMetricSet INSTANCE = new QueryRelatedResourceMetricSet();
+
+ private QueryRelatedResourceMetricSet() {
+ // empty constructor
+ }
+
// Coordinator
private static final Coordinator coordinator = Coordinator.getInstance();
- private static final String COORDINATOR = Metric.COORDINATOR.toString();
+ private static final String METRIC_COORDINATOR = Metric.COORDINATOR.toString();
private static final String QUERY_EXECUTION_MAP_SIZE = "query_execution_map_size";
// FragmentInstanceManager
@@ -63,11 +74,25 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
private static final String LOCAL_EXECUTION_PLANNER = Metric.LOCAL_EXECUTION_PLANNER.toString();
private static final String FREE_MEMORY_FOR_OPERATORS = "free_memory_for_operators";
+ // FragmentInstanceStatistics
+ public static final String QUERY_FRAGMENT_INSTANCE_COUNT = "query_fragment_instance_count";
+ private Gauge fragmentInstanceCountGauge = DoNothingMetricManager.DO_NOTHING_GAUGE;
+
+ public void recordExecutionCount(String stage, long count) {
+ switch (stage) {
+ case QUERY_FRAGMENT_INSTANCE_COUNT:
+ fragmentInstanceCountGauge.set(count);
+ break;
+ default:
+ break;
+ }
+ }
+
@Override
public void bindTo(AbstractMetricService metricService) {
// Coordinator
metricService.createAutoGauge(
- COORDINATOR,
+ METRIC_COORDINATOR,
MetricLevel.IMPORTANT,
coordinator,
Coordinator::getQueryExecutionMapSize,
@@ -124,13 +149,21 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
LocalExecutionPlanner::getFreeMemoryForOperators,
Tag.NAME.toString(),
FREE_MEMORY_FOR_OPERATORS);
+
+ // FragmentInstanceStatistics
+ fragmentInstanceCountGauge =
+ metricService.getOrCreateGauge(
+ FRAGMENT_INSTANCE_STATISTICS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ QUERY_FRAGMENT_INSTANCE_COUNT);
}
@Override
public void unbindFrom(AbstractMetricService metricService) {
// Coordinator
metricService.remove(
- MetricType.AUTO_GAUGE, COORDINATOR, Tag.NAME.toString(), QUERY_EXECUTION_MAP_SIZE);
+ MetricType.AUTO_GAUGE, METRIC_COORDINATOR, Tag.NAME.toString(), QUERY_EXECUTION_MAP_SIZE);
// FragmentInstanceManager
metricService.remove(
@@ -158,5 +191,16 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
LOCAL_EXECUTION_PLANNER,
Tag.NAME.toString(),
FREE_MEMORY_FOR_OPERATORS);
+
+ // FragmentInstanceStatistics
+ metricService.remove(
+ MetricType.GAUGE,
+ FRAGMENT_INSTANCE_STATISTICS.toString(),
+ Tag.NAME.toString(),
+ QUERY_FRAGMENT_INSTANCE_COUNT);
+ }
+
+ public static QueryRelatedResourceMetricSet getInstance() {
+ return INSTANCE;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
index ce33e169b36..8f365ccb886 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle;
import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle;
import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet;
import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet;
+import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.Analyzer;
import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
@@ -92,6 +93,7 @@ import static com.google.common.base.Throwables.throwIfUnchecked;
import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode;
import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.WAIT_FOR_RESULT;
import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.DISTRIBUTION_PLANNER;
+import static org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet.QUERY_FRAGMENT_INSTANCE_COUNT;
/**
* QueryExecution stores all the status of a query which is being prepared or running inside the MPP
@@ -382,6 +384,10 @@ public class QueryExecution implements IQueryExecution {
distributedPlan.getInstances().size(),
printFragmentInstances(distributedPlan.getInstances()));
}
+
+ QueryRelatedResourceMetricSet.getInstance()
+ .recordExecutionCount(QUERY_FRAGMENT_INSTANCE_COUNT, distributedPlan.getInstances().size());
+
// check timeout after building distribution plan because it could be time-consuming in some
// cases.
checkTimeOutForQuery();
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index 14afbbcb927..872f873fd3c 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -96,6 +96,7 @@ public enum Metric {
DRIVER_SCHEDULER("driver_scheduler"),
COORDINATOR("coordinator"),
FRAGMENT_INSTANCE_MANAGER("fragment_instance_manager"),
+ FRAGMENT_INSTANCE_STATISTICS("fragment_instance_statistics"),
MEMORY_POOL("memory_pool"),
LOCAL_EXECUTION_PLANNER("local_execution_planner"),
// file related
(iotdb) 02/02: tmp
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/query_metrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5ccb7c7a8dbe27863c312617e33efa7ee410acc8
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu Dec 14 12:33:12 2023 +0800
tmp
---
.../confignode1conf/iotdb-common.properties | 4 +-
.../fragment/FragmentInstanceContext.java | 21 ++
.../fragment/FragmentInstanceManager.java | 3 +
.../execution/fragment/QueryContext.java | 18 +
.../operator/source/AlignedSeriesScanUtil.java | 5 +-
.../execution/operator/source/SeriesScanUtil.java | 11 +-
.../metric/QueryRelatedResourceMetricSet.java | 64 +++-
.../metric/SeriesScanCostMetricSet.java | 368 ++++++++++++---------
.../db/service/metrics/CompactionMetrics.java | 2 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 57 +++-
.../datanode1conf/iotdb-datanode.properties | 2 +-
.../datanode2conf/iotdb-datanode.properties | 2 +-
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
14 files changed, 368 insertions(+), 192 deletions(-)
diff --git a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties
index 8981b21285b..0ad328914e3 100644
--- a/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties
+++ b/iotdb-core/confignode/src/test/resources/confignode1conf/iotdb-common.properties
@@ -20,8 +20,8 @@
timestamp_precision=ms
data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus
schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus
-schema_replication_factor=3
-data_replication_factor=3
+schema_replication_factor=1
+data_replication_factor=1
udf_lib_dir=target/confignode1/ext/udf
trigger_lib_dir=target/confignode1/ext/trigger
pipe_lib_dir=target/confignode1/ext/pipe
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 195986aae3f..525cbb297c5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -25,6 +25,8 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
+import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet;
+import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery;
@@ -49,6 +51,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+import static org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet.QUERY_FRAGMENT_EXECUTION_TIME;
+
public class FragmentInstanceContext extends QueryContext {
private static final Logger LOGGER = LoggerFactory.getLogger(FragmentInstanceContext.class);
@@ -450,6 +454,23 @@ public class FragmentInstanceContext extends QueryContext {
sourcePaths = null;
sharedQueryDataSource = null;
releaseDataNodeQueryContext();
+
+ // record fragment instance execution time and metadata get time to metrics
+ long durationTime = System.currentTimeMillis() - executionStartTime.get();
+ QueryRelatedResourceMetricSet.getInstance()
+ .recordExecutionTimeCost(QUERY_FRAGMENT_EXECUTION_TIME, durationTime);
+ SeriesScanCostMetricSet.getInstance()
+ .recordNonAlignedSeriesExecutionTime(
+ loadTimeSeriesMetadataDiskSeqTime.get(),
+ loadTimeSeriesMetadataDiskUnSeqTime.get(),
+ loadTimeSeriesMetadataMemSeqTime.get(),
+ loadTimeSeriesMetadataMemUnSeqTime.get());
+ SeriesScanCostMetricSet.getInstance()
+ .recordAlignedSeriesExecutionTime(
+ loadTimeSeriesMetadataAlignedDiskSeqTime.get(),
+ loadTimeSeriesMetadataAlignedDiskUnSeqTime.get(),
+ loadTimeSeriesMetadataAlignedMemSeqTime.get(),
+ loadTimeSeriesMetadataAlignedMemUnSeqTime.get());
}
private void releaseDataNodeQueryContext() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
index 945a59d3894..54cd72c41b1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISink;
import org.apache.iotdb.db.queryengine.execution.schedule.DriverScheduler;
import org.apache.iotdb.db.queryengine.execution.schedule.IDriverScheduler;
import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet;
+import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
import org.apache.iotdb.db.queryengine.plan.planner.PipelineDriverFactory;
import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
@@ -204,6 +205,8 @@ public class FragmentInstanceManager {
return createFailedInstanceInfo(instanceId);
}
} finally {
+ QueryRelatedResourceMetricSet.getInstance()
+ .updateFragmentInstance(instanceContext.size(), instanceExecution.size());
QUERY_EXECUTION_METRIC_SET.recordExecutionCost(
LOCAL_EXECUTION_PLANNER, System.nanoTime() - startTime);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
index b22f2b5f3cc..c91d81a8efb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
@@ -37,9 +37,27 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
/** QueryContext contains the shared information with in a query. */
public class QueryContext {
+ public AtomicLong loadTimeSeriesMetadataDiskSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataDiskUnSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataMemSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataMemUnSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedDiskSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedMemSeqCount = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqCount = new AtomicLong(0);
+
+ public AtomicLong loadTimeSeriesMetadataDiskSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataDiskUnSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataMemSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataMemUnSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedDiskSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedDiskUnSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedMemSeqTime = new AtomicLong(0);
+ public AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqTime = new AtomicLong(0);
/**
* The key is the path of a ModificationFile and the value is all Modifications in this file. We
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
index f37fe7c23f0..98007f50c07 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
@@ -142,10 +142,11 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil {
PartialPath seriesPath,
QueryContext context,
Filter filter,
- Set<String> allSensors)
+ Set<String> allSensors,
+ boolean isSeq)
throws IOException {
return FileLoaderUtils.loadTimeSeriesMetadata(
- resource, (AlignedPath) seriesPath, context, filter, queryAllSensors);
+ resource, (AlignedPath) seriesPath, context, filter, queryAllSensors, isSeq);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index 4d7f65947d3..ba84d821bdf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -1088,7 +1088,8 @@ public class SeriesScanUtil {
seriesPath,
context,
getGlobalTimeFilter(),
- scanOptions.getAllSensors());
+ scanOptions.getAllSensors(),
+ true);
if (timeseriesMetadata != null) {
timeseriesMetadata.setSeq(true);
seqTimeSeriesMetadata.add(timeseriesMetadata);
@@ -1102,7 +1103,8 @@ public class SeriesScanUtil {
seriesPath,
context,
getGlobalTimeFilter(),
- scanOptions.getAllSensors());
+ scanOptions.getAllSensors(),
+ false);
if (timeseriesMetadata != null) {
timeseriesMetadata.setModified(true);
timeseriesMetadata.setSeq(false);
@@ -1115,10 +1117,11 @@ public class SeriesScanUtil {
PartialPath seriesPath,
QueryContext context,
Filter filter,
- Set<String> allSensors)
+ Set<String> allSensors,
+ boolean isSeq)
throws IOException {
return FileLoaderUtils.loadTimeSeriesMetadata(
- resource, seriesPath, context, filter, allSensors);
+ resource, seriesPath, context, filter, allSensors, isSeq);
}
protected List<TSDataType> getTsDataTypeList() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
index 93626b06290..9f63480fc54 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
import org.apache.iotdb.metrics.type.Gauge;
+import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
@@ -43,12 +44,16 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
// empty constructor
}
+ /////////////////////////////////////////////////////////////////////////////////////////////////
// Coordinator
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final Coordinator coordinator = Coordinator.getInstance();
private static final String METRIC_COORDINATOR = Metric.COORDINATOR.toString();
private static final String QUERY_EXECUTION_MAP_SIZE = "query_execution_map_size";
+ /////////////////////////////////////////////////////////////////////////////////////////////////
// FragmentInstanceManager
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final FragmentInstanceManager fragmentInstanceManager =
FragmentInstanceManager.getInstance();
private static final String FRAGMENT_INSTANCE_MANAGER =
@@ -56,7 +61,9 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
private static final String INSTANCE_CONTEXT_SIZE = "instance_context_size";
private static final String INSTANCE_EXECUTION_SIZE = "instance_execution_size";
+ /////////////////////////////////////////////////////////////////////////////////////////////////
// MemoryPool
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final MemoryPool memoryPool =
MPPDataExchangeService.getInstance()
.getMPPDataExchangeManager()
@@ -68,15 +75,30 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
private static final String QUERY_MEMORY_RESERVATION_SIZE = "query_memory_reservation_size";
private static final String MEMORY_RESERVATION_SIZE = "memory_reservation_size";
+ /////////////////////////////////////////////////////////////////////////////////////////////////
// LocalExecutionPlanner
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final LocalExecutionPlanner localExecutionPlanner =
LocalExecutionPlanner.getInstance();
private static final String LOCAL_EXECUTION_PLANNER = Metric.LOCAL_EXECUTION_PLANNER.toString();
private static final String FREE_MEMORY_FOR_OPERATORS = "free_memory_for_operators";
+ /////////////////////////////////////////////////////////////////////////////////////////////////
// FragmentInstanceStatistics
+ /////////////////////////////////////////////////////////////////////////////////////////////////
public static final String QUERY_FRAGMENT_INSTANCE_COUNT = "query_fragment_instance_count";
+ public static final String QUERY_FRAGMENT_EXECUTION_TIME =
+ "query_fragment_instance_execution_time";
+ private Gauge fragmentInstanceContextSizeGauge = DoNothingMetricManager.DO_NOTHING_GAUGE;
+ private Gauge fragmentInstanceExecutionSizeGauge = DoNothingMetricManager.DO_NOTHING_GAUGE;
private Gauge fragmentInstanceCountGauge = DoNothingMetricManager.DO_NOTHING_GAUGE;
+ private Timer fragmentInstanceExecutionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+
+ public void updateFragmentInstance(
+ long fragmentInstanceContextSize, long fragmentInstanceExecutionSize) {
+ fragmentInstanceContextSizeGauge.set(fragmentInstanceContextSize);
+ fragmentInstanceExecutionSizeGauge.set(fragmentInstanceExecutionSize);
+ }
public void recordExecutionCount(String stage, long count) {
switch (stage) {
@@ -88,6 +110,16 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
}
}
+ public void recordExecutionTimeCost(String type, long cost) {
+ switch (type) {
+ case QUERY_FRAGMENT_EXECUTION_TIME:
+ fragmentInstanceExecutionTimer.updateMillis(cost);
+ break;
+ default:
+ break;
+ }
+ }
+
@Override
public void bindTo(AbstractMetricService metricService) {
// Coordinator
@@ -100,20 +132,18 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
QUERY_EXECUTION_MAP_SIZE);
// FragmentInstanceManager
- metricService.createAutoGauge(
- FRAGMENT_INSTANCE_MANAGER,
- MetricLevel.IMPORTANT,
- fragmentInstanceManager,
- FragmentInstanceManager::getInstanceContextSize,
- Tag.NAME.toString(),
- INSTANCE_CONTEXT_SIZE);
- metricService.createAutoGauge(
- FRAGMENT_INSTANCE_MANAGER,
- MetricLevel.IMPORTANT,
- fragmentInstanceManager,
- FragmentInstanceManager::getInstanceExecutionSize,
- Tag.NAME.toString(),
- INSTANCE_EXECUTION_SIZE);
+ fragmentInstanceContextSizeGauge =
+ metricService.getOrCreateGauge(
+ FRAGMENT_INSTANCE_MANAGER,
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ INSTANCE_CONTEXT_SIZE);
+ fragmentInstanceExecutionSizeGauge =
+ metricService.getOrCreateGauge(
+ FRAGMENT_INSTANCE_MANAGER,
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ INSTANCE_EXECUTION_SIZE);
// MemoryPool
metricService
@@ -157,6 +187,12 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
QUERY_FRAGMENT_INSTANCE_COUNT);
+ fragmentInstanceExecutionTimer =
+ metricService.getOrCreateTimer(
+ Metric.FRAGMENT_INSTANCE_STATISTICS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ QUERY_FRAGMENT_EXECUTION_TIME);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
index dffdef2e5f3..7ac0667867b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
@@ -28,101 +28,179 @@ import org.apache.iotdb.metrics.type.Timer;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.MetricType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Arrays;
public class SeriesScanCostMetricSet implements IMetricSet {
- private static final SeriesScanCostMetricSet INSTANCE = new SeriesScanCostMetricSet();
- private SeriesScanCostMetricSet() {
- // empty constructor
+ private static final Logger LOGGER = LoggerFactory.getLogger(SeriesScanCostMetricSet.class);
+
+ public static SeriesScanCostMetricSet getInstance() {
+ return INSTANCE;
+ }
+
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // load timeseries metadata
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ public static final String LOAD_TIMESERIES_METADATA = "load_timeseries_metadata";
+ public static final String LOAD_ALIGNED_TIMESERIES_METADATA = "load_aligned_timeseries_metadata";
+ public static final String ALIGNED = "aligned";
+ public static final String NON_ALIGNED = "non_aligned";
+ public static final String MEM = "mem";
+ public static final String DISK = "disk";
+ public static final String SEQUENCE = "sequence";
+ public static final String UNSEQUENCE = "unsequence";
+
+ public Timer loadTimeSeriesMetadataDiskSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataDiskUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataMemSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataMemUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+
+ public Timer loadTimeSeriesMetadataAlignedDiskSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataAlignedDiskUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataAlignedMemSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public Timer loadTimeSeriesMetadataAlignedMemUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+
+ public void recordNonAlignedSeriesExecutionTime(long t1, long t2, long t3, long t4) {
+ loadTimeSeriesMetadataDiskSeqTime.updateNanos(t1);
+ loadTimeSeriesMetadataDiskUnSeqTime.updateNanos(t2);
+ loadTimeSeriesMetadataMemSeqTime.updateNanos(t3);
+ loadTimeSeriesMetadataMemUnSeqTime.updateNanos(t4);
}
- // region load timeseries metadata
- private static final String LOAD_TIMESERIES_METADATA = "load_timeseries_metadata";
- private static final String ALIGNED = "aligned";
- private static final String NON_ALIGNED = "non_aligned";
- private static final String MEM = "mem";
- private static final String DISK = "disk";
- public static final String LOAD_TIMESERIES_METADATA_ALIGNED_MEM =
- LOAD_TIMESERIES_METADATA + "_" + ALIGNED + "_" + MEM;
- public static final String LOAD_TIMESERIES_METADATA_ALIGNED_DISK =
- LOAD_TIMESERIES_METADATA + "_" + ALIGNED + "_" + DISK;
- public static final String LOAD_TIMESERIES_METADATA_NONALIGNED_MEM =
- LOAD_TIMESERIES_METADATA + "_" + NON_ALIGNED + "_" + MEM;
- public static final String LOAD_TIMESERIES_METADATA_NONALIGNED_DISK =
- LOAD_TIMESERIES_METADATA + "_" + NON_ALIGNED + "_" + DISK;
- private Timer loadTimeseriesMetadataAlignedMemTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private Timer loadTimeseriesMetadataAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private Timer loadTimeseriesMetadataNonAlignedMemTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- private Timer loadTimeseriesMetadataNonAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+ public void recordAlignedSeriesExecutionTime(long t1, long t2, long t3, long t4) {
+ loadTimeSeriesMetadataAlignedDiskSeqTime.updateNanos(t1);
+ loadTimeSeriesMetadataAlignedDiskUnSeqTime.updateNanos(t2);
+ loadTimeSeriesMetadataAlignedMemSeqTime.updateNanos(t3);
+ loadTimeSeriesMetadataAlignedMemUnSeqTime.updateNanos(t4);
+ }
private void bindTimeseriesMetadata(AbstractMetricService metricService) {
- loadTimeseriesMetadataAlignedMemTimer =
+ loadTimeSeriesMetadataDiskSeqTime =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
LOAD_TIMESERIES_METADATA,
Tag.TYPE.toString(),
- ALIGNED,
+ SEQUENCE,
Tag.FROM.toString(),
- MEM);
- loadTimeseriesMetadataAlignedDiskTimer =
+ DISK);
+ loadTimeSeriesMetadataDiskUnSeqTime =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
LOAD_TIMESERIES_METADATA,
Tag.TYPE.toString(),
- ALIGNED,
+ UNSEQUENCE,
Tag.FROM.toString(),
DISK);
- loadTimeseriesMetadataNonAlignedMemTimer =
+ loadTimeSeriesMetadataMemSeqTime =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
LOAD_TIMESERIES_METADATA,
Tag.TYPE.toString(),
- NON_ALIGNED,
+ SEQUENCE,
Tag.FROM.toString(),
MEM);
- loadTimeseriesMetadataNonAlignedDiskTimer =
+ loadTimeSeriesMetadataMemUnSeqTime =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
LOAD_TIMESERIES_METADATA,
Tag.TYPE.toString(),
- NON_ALIGNED,
+ UNSEQUENCE,
+ Tag.FROM.toString(),
+ MEM);
+ }
+
+ private void bindAlignedTimeseriesMetadata(AbstractMetricService metricService) {
+ loadTimeSeriesMetadataAlignedDiskSeqTime =
+ metricService.getOrCreateTimer(
+ Metric.SERIES_SCAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ LOAD_ALIGNED_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ SEQUENCE,
+ Tag.FROM.toString(),
+ DISK);
+ loadTimeSeriesMetadataAlignedDiskUnSeqTime =
+ metricService.getOrCreateTimer(
+ Metric.SERIES_SCAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ LOAD_ALIGNED_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ UNSEQUENCE,
Tag.FROM.toString(),
DISK);
+ loadTimeSeriesMetadataAlignedMemSeqTime =
+ metricService.getOrCreateTimer(
+ Metric.SERIES_SCAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ LOAD_ALIGNED_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ SEQUENCE,
+ Tag.FROM.toString(),
+ MEM);
+ loadTimeSeriesMetadataAlignedMemUnSeqTime =
+ metricService.getOrCreateTimer(
+ Metric.SERIES_SCAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ LOAD_ALIGNED_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ UNSEQUENCE,
+ Tag.FROM.toString(),
+ MEM);
}
private void unbindTimeseriesMetadata(AbstractMetricService metricService) {
- loadTimeseriesMetadataAlignedMemTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- loadTimeseriesMetadataAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- loadTimeseriesMetadataNonAlignedMemTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- loadTimeseriesMetadataNonAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- Arrays.asList(ALIGNED, NON_ALIGNED)
- .forEach(
- type ->
- Arrays.asList(MEM, DISK)
- .forEach(
- from ->
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- LOAD_TIMESERIES_METADATA,
- Tag.TYPE.toString(),
- type,
- Tag.FROM.toString(),
- from)));
+ loadTimeSeriesMetadataDiskSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataDiskUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataMemSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataMemUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataAlignedDiskSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataAlignedDiskUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataAlignedMemSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataAlignedMemUnSeqTime = DoNothingMetricManager.DO_NOTHING_TIMER;
+
+ for (String type : Arrays.asList(ALIGNED, NON_ALIGNED)) {
+ for (String from : Arrays.asList(MEM, DISK)) {
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ LOAD_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ from);
+
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ LOAD_ALIGNED_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ from);
+ }
+ }
}
- // endregion
- // region read timeseries metadata
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // read timeseries metadata
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String READ_TIMESERIES_METADATA = "read_timeseries_metadata";
private static final String CACHE = "cache";
private static final String FILE = "file";
@@ -161,21 +239,21 @@ public class SeriesScanCostMetricSet implements IMetricSet {
readTimeseriesMetadataFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList(CACHE, FILE)
.forEach(
- from -> {
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- READ_TIMESERIES_METADATA,
- Tag.TYPE.toString(),
- NULL,
- Tag.FROM.toString(),
- from);
- });
+ from ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ READ_TIMESERIES_METADATA,
+ Tag.TYPE.toString(),
+ NULL,
+ Tag.FROM.toString(),
+ from));
}
- // endregion
- // region read timeseries metadata aligned
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // read timeseries metadata aligned
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String TIMESERIES_METADATA_MODIFICATION = "timeseries_metadata_modification";
public static final String TIMESERIES_METADATA_MODIFICATION_ALIGNED =
TIMESERIES_METADATA_MODIFICATION + "_" + ALIGNED;
@@ -214,21 +292,21 @@ public class SeriesScanCostMetricSet implements IMetricSet {
timeseriesMetadataModificationNonAlignedTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList(ALIGNED, NON_ALIGNED)
.forEach(
- type -> {
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- TIMESERIES_METADATA_MODIFICATION,
- Tag.TYPE.toString(),
- type,
- Tag.FROM.toString(),
- NULL);
- });
+ type ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ TIMESERIES_METADATA_MODIFICATION,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ NULL));
}
- // endregion
- // region load chunk metadata list
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // load chunk metadata list
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String LOAD_CHUNK_METADATA_LIST = "load_chunk_metadata_list";
public static final String LOAD_CHUNK_METADATA_LIST_ALIGNED_MEM =
LOAD_CHUNK_METADATA_LIST + "_" + ALIGNED + "_" + MEM;
@@ -293,26 +371,24 @@ public class SeriesScanCostMetricSet implements IMetricSet {
loadChunkMetadataListNonAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList(ALIGNED, NON_ALIGNED)
.forEach(
- type -> {
- Arrays.asList(MEM, DISK)
- .forEach(
- from -> {
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- LOAD_CHUNK_METADATA_LIST,
- Tag.TYPE.toString(),
- type,
- Tag.FROM.toString(),
- from);
- });
- });
+ type ->
+ Arrays.asList(MEM, DISK)
+ .forEach(
+ from ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ LOAD_CHUNK_METADATA_LIST,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ from)));
}
- // endregion
-
- // region chunk metadata modification
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // chunk metadata modification
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String CHUNK_METADATA_MODIFICATION = "chunk_metadata_modification";
public static final String CHUNK_METADATA_MODIFICATION_ALIGNED_MEM =
CHUNK_METADATA_MODIFICATION + "_" + ALIGNED + "_" + MEM;
@@ -393,9 +469,10 @@ public class SeriesScanCostMetricSet implements IMetricSet {
Tag.FROM.toString(),
from)));
}
- // endregion
- // region chunk metadata filter
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // chunk metadata filter
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String CHUNK_METADATA_FILTER = "chunk_metadata_filter";
public static final String CHUNK_METADATA_FILTER_ALIGNED_MEM =
CHUNK_METADATA_FILTER + "_" + ALIGNED + "_" + MEM;
@@ -476,9 +553,9 @@ public class SeriesScanCostMetricSet implements IMetricSet {
from)));
}
- // endregion
-
- // region construct chunk reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // construct chunk reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String CONSTRUCT_CHUNK_READER = "construct_chunk_reader";
public static final String CONSTRUCT_CHUNK_READER_ALIGNED_MEM =
CONSTRUCT_CHUNK_READER + "_" + ALIGNED + "_" + MEM;
@@ -546,22 +623,21 @@ public class SeriesScanCostMetricSet implements IMetricSet {
type ->
Arrays.asList(MEM, DISK)
.forEach(
- from -> {
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- CONSTRUCT_CHUNK_READER,
- Tag.TYPE.toString(),
- type,
- Tag.FROM.toString(),
- from);
- }));
+ from ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ CONSTRUCT_CHUNK_READER,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ from)));
}
- // endregion
-
- // region read chunk
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // read chunk
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String READ_CHUNK = "read_chunk";
private static final String ALL = "all";
public static final String READ_CHUNK_ALL = READ_CHUNK + "_" + ALL;
@@ -608,9 +684,10 @@ public class SeriesScanCostMetricSet implements IMetricSet {
Tag.FROM.toString(),
from));
}
- // endregion
- // region init chunk reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // init chunk reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String INIT_CHUNK_READER = "init_chunk_reader";
public static final String INIT_CHUNK_READER_ALIGNED_MEM =
INIT_CHUNK_READER + "_" + ALIGNED + "_" + MEM;
@@ -689,9 +766,10 @@ public class SeriesScanCostMetricSet implements IMetricSet {
Tag.FROM.toString(),
from)));
}
- // endregion
- // region build tsblock from page reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // build tsblock from page reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String BUILD_TSBLOCK_FROM_PAGE_READER = "build_tsblock_from_page_reader";
public static final String BUILD_TSBLOCK_FROM_PAGE_READER_ALIGNED_MEM =
BUILD_TSBLOCK_FROM_PAGE_READER + "_" + ALIGNED + "_" + MEM;
@@ -760,25 +838,24 @@ public class SeriesScanCostMetricSet implements IMetricSet {
buildTsBlockFromPageReaderNonAlignedDiskTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
Arrays.asList(ALIGNED, NON_ALIGNED)
.forEach(
- type -> {
- Arrays.asList(MEM, DISK)
- .forEach(
- from -> {
- metricService.remove(
- MetricType.TIMER,
- Metric.SERIES_SCAN_COST.toString(),
- Tag.STAGE.toString(),
- BUILD_TSBLOCK_FROM_PAGE_READER,
- Tag.TYPE.toString(),
- type,
- Tag.FROM.toString(),
- from);
- });
- });
+ type ->
+ Arrays.asList(MEM, DISK)
+ .forEach(
+ from ->
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.SERIES_SCAN_COST.toString(),
+ Tag.STAGE.toString(),
+ BUILD_TSBLOCK_FROM_PAGE_READER,
+ Tag.TYPE.toString(),
+ type,
+ Tag.FROM.toString(),
+ from)));
}
- // endregion
- // region build tsblock from merge reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // build tsblock from merge reader
+ /////////////////////////////////////////////////////////////////////////////////////////////////
private static final String BUILD_TSBLOCK_FROM_MERGE_READER = "build_tsblock_from_merge_reader";
public static final String BUILD_TSBLOCK_FROM_MERGE_READER_ALIGNED =
BUILD_TSBLOCK_FROM_MERGE_READER + "_" + ALIGNED;
@@ -828,11 +905,12 @@ public class SeriesScanCostMetricSet implements IMetricSet {
Tag.TYPE.toString(),
type));
}
- // endregion
@Override
public void bindTo(AbstractMetricService metricService) {
bindTimeseriesMetadata(metricService);
+ bindAlignedTimeseriesMetadata(metricService);
+
bindReadTimeseriesMetadata(metricService);
bindTimeseriesMetadataModification(metricService);
bindLoadChunkMetadataList(metricService);
@@ -862,18 +940,6 @@ public class SeriesScanCostMetricSet implements IMetricSet {
public void recordSeriesScanCost(String type, long cost) {
switch (type) {
- case LOAD_TIMESERIES_METADATA_ALIGNED_MEM:
- loadTimeseriesMetadataAlignedMemTimer.updateNanos(cost);
- break;
- case LOAD_TIMESERIES_METADATA_ALIGNED_DISK:
- loadTimeseriesMetadataAlignedDiskTimer.updateNanos(cost);
- break;
- case LOAD_TIMESERIES_METADATA_NONALIGNED_MEM:
- loadTimeseriesMetadataNonAlignedMemTimer.updateNanos(cost);
- break;
- case LOAD_TIMESERIES_METADATA_NONALIGNED_DISK:
- loadTimeseriesMetadataNonAlignedDiskTimer.updateNanos(cost);
- break;
case TIMESERIES_METADATA_MODIFICATION_ALIGNED:
timeseriesMetadataModificationAlignedTimer.updateNanos(cost);
break;
@@ -975,7 +1041,9 @@ public class SeriesScanCostMetricSet implements IMetricSet {
}
}
- public static SeriesScanCostMetricSet getInstance() {
- return INSTANCE;
+ private static final SeriesScanCostMetricSet INSTANCE = new SeriesScanCostMetricSet();
+
+ private SeriesScanCostMetricSet() {
+ // empty constructor
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java
index f7ca16b32aa..102a23756b4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/CompactionMetrics.java
@@ -443,7 +443,7 @@ public class CompactionMetrics implements IMetricSet {
Metric.COMPACTION_TASK_COUNT.toString(),
MetricLevel.IMPORTANT,
this,
- (metrics) -> {
+ metrics -> {
updateCompactionTaskInfo();
return finishSeqInnerCompactionTaskNum.get();
},
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index bc7f28c184e..6e6e2afa2e0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -75,7 +75,7 @@ public class DataNodeMetricsHelper {
MetricService.getInstance().addMetricSet(DataExchangeCostMetricSet.getInstance());
MetricService.getInstance().addMetricSet(DataExchangeCountMetricSet.getInstance());
MetricService.getInstance().addMetricSet(DriverSchedulerMetricSet.getInstance());
- MetricService.getInstance().addMetricSet(new QueryRelatedResourceMetricSet());
+ MetricService.getInstance().addMetricSet(QueryRelatedResourceMetricSet.getInstance());
// bind performance overview related metrics
MetricService.getInstance().addMetricSet(PerformanceOverviewMetrics.getInstance());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 7a08d5b3e39..1ca8e6b0142 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -55,10 +55,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_DISK;
-import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_ALIGNED_MEM;
-import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_DISK;
-import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.LOAD_TIMESERIES_METADATA_NONALIGNED_MEM;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_ALIGNED;
import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.TIMESERIES_METADATA_MODIFICATION_NONALIGNED;
@@ -119,6 +115,7 @@ public class FileLoaderUtils {
* @param seriesPath Timeseries path
* @param allSensors measurements queried at the same time of this device
* @param filter any filter, only used to check time range
+ * @param isSeq if it is a sequence file
* @throws IOException IOException may be thrown while reading it from disk.
*/
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
@@ -127,7 +124,8 @@ public class FileLoaderUtils {
PartialPath seriesPath,
QueryContext context,
Filter filter,
- Set<String> allSensors)
+ Set<String> allSensors,
+ boolean isSeq)
throws IOException {
long t1 = System.nanoTime();
boolean loadFromMem = false;
@@ -185,11 +183,24 @@ public class FileLoaderUtils {
}
return timeSeriesMetadata;
} finally {
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- loadFromMem
- ? LOAD_TIMESERIES_METADATA_NONALIGNED_MEM
- : LOAD_TIMESERIES_METADATA_NONALIGNED_DISK,
- System.nanoTime() - t1);
+ long costTime = System.nanoTime() - t1;
+ if (loadFromMem) {
+ if (isSeq) {
+ context.loadTimeSeriesMetadataMemSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataMemSeqTime.getAndAdd(costTime);
+ } else {
+ context.loadTimeSeriesMetadataMemUnSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataMemUnSeqTime.getAndAdd(costTime);
+ }
+ } else {
+ if (isSeq) {
+ context.loadTimeSeriesMetadataDiskSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataDiskSeqTime.getAndAdd(costTime);
+ } else {
+ context.loadTimeSeriesMetadataDiskUnSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataDiskUnSeqTime.getAndAdd(costTime);
+ }
+ }
}
}
@@ -206,7 +217,8 @@ public class FileLoaderUtils {
AlignedPath alignedPath,
QueryContext context,
Filter filter,
- boolean queryAllSensors)
+ boolean queryAllSensors,
+ boolean isSeq)
throws IOException {
final long t1 = System.nanoTime();
boolean loadFromMem = false;
@@ -248,11 +260,24 @@ public class FileLoaderUtils {
}
return alignedTimeSeriesMetadata;
} finally {
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- loadFromMem
- ? LOAD_TIMESERIES_METADATA_ALIGNED_MEM
- : LOAD_TIMESERIES_METADATA_ALIGNED_DISK,
- System.nanoTime() - t1);
+ long costTime = System.nanoTime() - t1;
+ if (loadFromMem) {
+ if (isSeq) {
+ context.loadTimeSeriesMetadataAlignedMemSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataAlignedMemSeqTime.getAndAdd(costTime);
+ } else {
+ context.loadTimeSeriesMetadataAlignedMemUnSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataAlignedMemUnSeqTime.getAndAdd(costTime);
+ }
+ } else {
+ if (isSeq) {
+ context.loadTimeSeriesMetadataAlignedDiskSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataAlignedDiskSeqTime.getAndAdd(costTime);
+ } else {
+ context.loadTimeSeriesMetadataAlignedDiskUnSeqCount.getAndAdd(1);
+ context.loadTimeSeriesMetadataAlignedDiskUnSeqTime.getAndAdd(costTime);
+ }
+ }
}
}
diff --git a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties
index 72acbce4157..6f02870b280 100644
--- a/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties
+++ b/iotdb-core/datanode/src/test/resources/datanode1conf/iotdb-datanode.properties
@@ -39,4 +39,4 @@ sort_tmp_dir=target/datanode1/tmp
dn_metric_reporter_list=PROMETHEUS
dn_metric_level=IMPORTANT
dn_metric_async_collect_period=5
-dn_metric_prometheus_reporter_port=9097
\ No newline at end of file
+dn_metric_prometheus_reporter_port=9093
\ No newline at end of file
diff --git a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties
index b1037c8457b..f766356f275 100644
--- a/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties
+++ b/iotdb-core/datanode/src/test/resources/datanode2conf/iotdb-datanode.properties
@@ -39,4 +39,4 @@ sort_tmp_dir=target/datanode2/tmp
dn_metric_reporter_list=PROMETHEUS
dn_metric_level=IMPORTANT
dn_metric_async_collect_period=5
-dn_metric_prometheus_reporter_port=9099
+dn_metric_prometheus_reporter_port=9094
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index 872f873fd3c..fe0dc34aef2 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -86,6 +86,7 @@ public enum Metric {
OPERATOR_EXECUTION_COST("operator_execution_cost"),
OPERATOR_EXECUTION_COUNT("operator_execution_count"),
SERIES_SCAN_COST("series_scan_cost"),
+ QUERY_METADATA_COST("query_metadata_cost"),
DISPATCHER("dispatcher"),
QUERY_EXECUTION("query_execution"),
AGGREGATION("aggregation"),