You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/12/20 01:15:48 UTC
[iotdb] 02/05: add metrics: query_resource
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/addQueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1e52c006e1e7b5e62cd6ef353788d022be16e49a
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Dec 19 11:45:44 2022 +0800
add metrics: query_resource
---
.../iotdb/commons/service/metric/enums/Metric.java | 3 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 9 ++
.../db/engine/storagegroup/TsFileProcessor.java | 108 ++++++++++++---------
.../db/mpp/metric/QueryExecutionMetricSet.java | 8 ++
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 5 +
.../db/mpp/metric/QueryResourceMetricSet.java | 58 +++++++++++
6 files changed, 144 insertions(+), 47 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index 3d70034cd6..0fd3e8f258 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -68,7 +68,8 @@ public enum Metric {
SERIES_SCAN_COST,
DISPATCHER,
QUERY_EXECUTION,
- AGGREGATION;
+ AGGREGATION,
+ QUERY_RESOURCE;
@Override
public String toString() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 964c423f07..b602eab8b6 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache;
import org.apache.iotdb.db.metadata.idtable.IDTable;
import org.apache.iotdb.db.metadata.idtable.IDTableManager;
+import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertMultiTabletsNode;
@@ -132,6 +133,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
import static org.apache.iotdb.db.engine.storagegroup.TsFileResource.TEMP_SUFFIX;
+import static org.apache.iotdb.db.mpp.metric.QueryResourceMetricSet.SEQUENCE_TSFILE;
+import static org.apache.iotdb.db.mpp.metric.QueryResourceMetricSet.UNSEQUENCE_TSFILE;
import static org.apache.iotdb.db.qp.executor.PlanExecutor.operateClearCache;
import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@@ -267,6 +270,8 @@ public class DataRegion implements IDataRegionForQuery {
private IDTable idTable;
+ private final QueryMetricsManager QUERY_METRICS = QueryMetricsManager.getInstance();
+
/**
* constrcut a database processor
*
@@ -1718,6 +1723,10 @@ public class DataRegion implements IDataRegionForQuery {
context,
timeFilter,
false);
+
+ QUERY_METRICS.recordQueryResourceNum(SEQUENCE_TSFILE, seqResources.size());
+ QUERY_METRICS.recordQueryResourceNum(UNSEQUENCE_TSFILE, unseqResources.size());
+
QueryDataSource dataSource = new QueryDataSource(seqResources, unseqResources);
dataSource.setDataTTL(dataTTL);
return dataSource;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 04ec438712..cc0d2c14e8 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -48,6 +48,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory;
import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID;
import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils;
+import org.apache.iotdb.db.mpp.metric.QueryMetricsManager;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertTabletNode;
@@ -88,6 +89,10 @@ import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.iotdb.db.mpp.metric.QueryExecutionMetricSet.GET_QUERY_RESOURCE_FROM_MEM;
+import static org.apache.iotdb.db.mpp.metric.QueryResourceMetricSet.FLUSHING_MEMTABLE;
+import static org.apache.iotdb.db.mpp.metric.QueryResourceMetricSet.WORKING_MEMTABLE;
+
@SuppressWarnings("java:S1135") // ignore todos
public class TsFileProcessor {
@@ -163,6 +168,8 @@ public class TsFileProcessor {
/** flush file listener */
private List<FlushListener> flushListeners = new ArrayList<>();
+ private final QueryMetricsManager QUERY_METRICS = QueryMetricsManager.getInstance();
+
@SuppressWarnings("squid:S107")
TsFileProcessor(
String storageGroupName,
@@ -1349,61 +1356,70 @@ public class TsFileProcessor {
QueryContext context,
List<TsFileResource> tsfileResourcesForQuery)
throws IOException {
- Map<PartialPath, List<IChunkMetadata>> pathToChunkMetadataListMap = new HashMap<>();
- Map<PartialPath, List<ReadOnlyMemChunk>> pathToReadOnlyMemChunkMap = new HashMap<>();
-
- flushQueryLock.readLock().lock();
+ long startTime = System.nanoTime();
try {
- for (PartialPath seriesPath : seriesPaths) {
- List<ReadOnlyMemChunk> readOnlyMemChunks = new ArrayList<>();
- for (IMemTable flushingMemTable : flushingMemTables) {
- if (flushingMemTable.isSignalMemTable()) {
- continue;
- }
- ReadOnlyMemChunk memChunk =
- flushingMemTable.query(seriesPath, context.getQueryTimeLowerBound(), modsToMemtable);
- if (memChunk != null) {
- readOnlyMemChunks.add(memChunk);
+ Map<PartialPath, List<IChunkMetadata>> pathToChunkMetadataListMap = new HashMap<>();
+ Map<PartialPath, List<ReadOnlyMemChunk>> pathToReadOnlyMemChunkMap = new HashMap<>();
+
+ flushQueryLock.readLock().lock();
+ try {
+ for (PartialPath seriesPath : seriesPaths) {
+ List<ReadOnlyMemChunk> readOnlyMemChunks = new ArrayList<>();
+ for (IMemTable flushingMemTable : flushingMemTables) {
+ if (flushingMemTable.isSignalMemTable()) {
+ continue;
+ }
+ ReadOnlyMemChunk memChunk =
+ flushingMemTable.query(
+ seriesPath, context.getQueryTimeLowerBound(), modsToMemtable);
+ if (memChunk != null) {
+ readOnlyMemChunks.add(memChunk);
+ }
}
- }
- if (workMemTable != null) {
- ReadOnlyMemChunk memChunk =
- workMemTable.query(seriesPath, context.getQueryTimeLowerBound(), null);
- if (memChunk != null) {
- readOnlyMemChunks.add(memChunk);
+ if (workMemTable != null) {
+ ReadOnlyMemChunk memChunk =
+ workMemTable.query(seriesPath, context.getQueryTimeLowerBound(), null);
+ if (memChunk != null) {
+ readOnlyMemChunks.add(memChunk);
+ }
}
- }
- List<IChunkMetadata> chunkMetadataList =
- ResourceByPathUtils.getResourceInstance(seriesPath)
- .getVisibleMetadataListFromWriter(writer, tsFileResource, context);
+ List<IChunkMetadata> chunkMetadataList =
+ ResourceByPathUtils.getResourceInstance(seriesPath)
+ .getVisibleMetadataListFromWriter(writer, tsFileResource, context);
- // get in memory data
- if (!readOnlyMemChunks.isEmpty() || !chunkMetadataList.isEmpty()) {
- pathToReadOnlyMemChunkMap.put(seriesPath, readOnlyMemChunks);
- pathToChunkMetadataListMap.put(seriesPath, chunkMetadataList);
+ // get in memory data
+ if (!readOnlyMemChunks.isEmpty() || !chunkMetadataList.isEmpty()) {
+ pathToReadOnlyMemChunkMap.put(seriesPath, readOnlyMemChunks);
+ pathToChunkMetadataListMap.put(seriesPath, chunkMetadataList);
+ }
}
- }
- } catch (QueryProcessException | MetadataException e) {
- logger.error(
- "{}: {} get ReadOnlyMemChunk has error",
- storageGroupName,
- tsFileResource.getTsFile().getName(),
- e);
- } finally {
- flushQueryLock.readLock().unlock();
- if (logger.isDebugEnabled()) {
- logger.debug(
- "{}: {} release flushQueryLock",
+ } catch (QueryProcessException | MetadataException e) {
+ logger.error(
+ "{}: {} get ReadOnlyMemChunk has error",
storageGroupName,
- tsFileResource.getTsFile().getName());
+ tsFileResource.getTsFile().getName(),
+ e);
+ } finally {
+ QUERY_METRICS.recordQueryResourceNum(FLUSHING_MEMTABLE, flushingMemTables.size());
+ QUERY_METRICS.recordQueryResourceNum(WORKING_MEMTABLE, workMemTable != null ? 1 : 0);
+
+ flushQueryLock.readLock().unlock();
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "{}: {} release flushQueryLock",
+ storageGroupName,
+ tsFileResource.getTsFile().getName());
+ }
}
- }
- if (!pathToReadOnlyMemChunkMap.isEmpty() || !pathToChunkMetadataListMap.isEmpty()) {
- tsfileResourcesForQuery.add(
- new TsFileResource(
- pathToReadOnlyMemChunkMap, pathToChunkMetadataListMap, tsFileResource));
+ if (!pathToReadOnlyMemChunkMap.isEmpty() || !pathToChunkMetadataListMap.isEmpty()) {
+ tsfileResourcesForQuery.add(
+ new TsFileResource(
+ pathToReadOnlyMemChunkMap, pathToChunkMetadataListMap, tsFileResource));
+ }
+ } finally {
+ QUERY_METRICS.recordExecutionCost(GET_QUERY_RESOURCE_FROM_MEM, System.nanoTime() - startTime);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
index 0667bdc623..adaf851310 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryExecutionMetricSet.java
@@ -58,6 +58,7 @@ public class QueryExecutionMetricSet implements IMetricSet {
public static final String LOCAL_EXECUTION_PLANNER = "local_execution_planner";
public static final String QUERY_RESOURCE_INIT = "query_resource_init";
+ public static final String GET_QUERY_RESOURCE_FROM_MEM = "get_query_resource_from_mem";
public static final String DRIVER_INTERNAL_PROCESS = "driver_internal_process";
public static final String WAIT_FOR_RESULT = "wait_for_result";
@@ -76,6 +77,13 @@ public class QueryExecutionMetricSet implements IMetricSet {
Metric.QUERY_EXECUTION.toString(),
Tag.STAGE.toString(),
QUERY_RESOURCE_INIT));
+ metricInfoMap.put(
+ GET_QUERY_RESOURCE_FROM_MEM,
+ new MetricInfo(
+ MetricType.TIMER,
+ Metric.QUERY_EXECUTION.toString(),
+ Tag.STAGE.toString(),
+ GET_QUERY_RESOURCE_FROM_MEM));
metricInfoMap.put(
DRIVER_INTERNAL_PROCESS,
new MetricInfo(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
index 0cf9bea940..6d6146c90d 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
@@ -80,6 +80,11 @@ public class QueryMetricsManager {
metricInfo.getTagsInArray());
}
+ public void recordQueryResourceNum(String type, int count) {
+ metricService.rate(
+ count, Metric.QUERY_RESOURCE.toString(), MetricLevel.IMPORTANT, Tag.TYPE.toString(), type);
+ }
+
public static QueryMetricsManager getInstance() {
return QueryMetricsManager.QueryMetricsManagerHolder.INSTANCE;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryResourceMetricSet.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryResourceMetricSet.java
new file mode 100644
index 0000000000..06dc34fd60
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryResourceMetricSet.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.mpp.metric;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class QueryResourceMetricSet implements IMetricSet {
+
+ public static final String SEQUENCE_TSFILE = "sequence_tsfile";
+ public static final String UNSEQUENCE_TSFILE = "unsequence_tsfile";
+ public static final String FLUSHING_MEMTABLE = "flushing_memtable";
+ public static final String WORKING_MEMTABLE = "working_memtable";
+
+ private static final String metric = Metric.QUERY_RESOURCE.toString();
+ private final String tagKey = Tag.TYPE.toString();
+
+ private static final List<String> resourceTypes =
+ Arrays.asList(SEQUENCE_TSFILE, UNSEQUENCE_TSFILE, FLUSHING_MEMTABLE, WORKING_MEMTABLE);
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ for (String type : resourceTypes) {
+ metricService.getOrCreateRate(metric, MetricLevel.IMPORTANT, tagKey, type);
+ }
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ for (String type : resourceTypes) {
+ metricService.remove(MetricType.RATE, metric, tagKey, type);
+ }
+ }
+}