You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/12/06 15:16:01 UTC
[iotdb] 02/02: add io metrics
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 72020ef96b74c12de0d9eebd03b216e734872d4f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Dec 6 23:13:16 2022 +0800
add io metrics
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 5 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 6 +
.../iotdb/db/mpp/statistics/QueryStatistics.java | 7 +
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 259 +++++++++++----------
4 files changed, 159 insertions(+), 118 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 0fa63a5fdf..a47f9ecffe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -71,6 +72,7 @@ public class ChunkCache {
.recordStats()
.build(
chunkMetadata -> {
+ long startTime = System.nanoTime();
try {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
@@ -79,6 +81,9 @@ public class ChunkCache {
} catch (IOException e) {
logger.error("Something wrong happened in reading {}", chunkMetadata, e);
throw e;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.CHUNK_CACHE_MISS, System.nanoTime() - startTime);
}
});
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d70b2faf41..c9610d3354 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
@@ -50,6 +51,8 @@ import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.TIME_SERIES_METADATA_CACHE_MISS;
+
/**
* This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is
* LRU.
@@ -137,6 +140,7 @@ public class TimeSeriesMetadataCache {
TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
if (timeseriesMetadata == null) {
+ long startTime = System.nanoTime();
if (debug) {
DEBUG_LOGGER.info(
"Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
@@ -180,6 +184,8 @@ public class TimeSeriesMetadataCache {
}
}
}
+ QueryStatistics.getInstance()
+ .addCost(TIME_SERIES_METADATA_CACHE_MISS, System.nanoTime() - startTime);
}
if (timeseriesMetadata == null) {
if (debug) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 9b0dd7e985..7f6074d709 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -84,6 +84,13 @@ public class QueryStatistics {
public static final String SERVER_RPC_RT = "ServerRpcRT";
+ public static final String LOAD_TIME_SERIES_METADATA_ALIGNED = "loadTimeSeriesMetadata-aligned";
+ public static final String LOAD_TIME_SERIES_METADATA = "loadTimeSeriesMetadata";
+ public static final String LOAD_CHUNK_METADATA_LIST = "loadChunkMetadataList";
+ public static final String LOAD_PAGE_READER_LIST = "loadPageReaderList";
+ public static final String TIME_SERIES_METADATA_CACHE_MISS = "TimeSeriesMetadataCacheMiss";
+ public static final String CHUNK_CACHE_MISS = "ChunkCacheMiss";
+
private QueryStatistics() {
ScheduledExecutorService scheduledExecutor =
IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index bfd6a0bf80..bdaffb8360 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.TimeSeriesMetada
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.chunk.metadata.DiskAlignedChunkMetadataLoader;
import org.apache.iotdb.db.query.reader.chunk.metadata.DiskChunkMetadataLoader;
@@ -124,51 +125,56 @@ public class FileLoaderUtils {
Filter filter,
Set<String> allSensors)
throws IOException {
-
- // common path
- TimeseriesMetadata timeSeriesMetadata;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- timeSeriesMetadata =
- TimeSeriesMetadataCache.getInstance()
- .get(
- new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
- resource.getTsFilePath(),
- seriesPath.getDevice(),
- seriesPath.getMeasurement()),
- allSensors,
- resource.getTimeIndexType() != 1,
- context.isDebug());
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
- }
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ long startTime = System.nanoTime();
+ try {
+ // common path
+ TimeseriesMetadata timeSeriesMetadata;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ timeSeriesMetadata =
+ TimeSeriesMetadataCache.getInstance()
+ .get(
+ new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
+ resource.getTsFilePath(),
+ seriesPath.getDevice(),
+ seriesPath.getMeasurement()),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ context.isDebug());
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
}
- }
- if (timeSeriesMetadata != null) {
- List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(), seriesPath);
- timeSeriesMetadata.setModified(!pathModifications.isEmpty());
- if (timeSeriesMetadata.getStatistics().getStartTime()
- > timeSeriesMetadata.getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- timeSeriesMetadata.getStatistics().getStartTime(),
- timeSeriesMetadata.getStatistics().getEndTime())) {
- return null;
+ if (timeSeriesMetadata != null) {
+ List<Modification> pathModifications =
+ context.getPathModifications(resource.getModFile(), seriesPath);
+ timeSeriesMetadata.setModified(!pathModifications.isEmpty());
+ if (timeSeriesMetadata.getStatistics().getStartTime()
+ > timeSeriesMetadata.getStatistics().getEndTime()) {
+ return null;
+ }
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ timeSeriesMetadata.getStatistics().getStartTime(),
+ timeSeriesMetadata.getStatistics().getEndTime())) {
+ return null;
+ }
}
+ return timeSeriesMetadata;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_TIME_SERIES_METADATA, System.nanoTime() - startTime);
}
- return timeSeriesMetadata;
}
/**
@@ -181,87 +187,94 @@ public class FileLoaderUtils {
public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter)
throws IOException {
- AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // load all the TimeseriesMetadata of vector, the first one is for time column and the
- // remaining is for sub sensors
- // the order of timeSeriesMetadata list is same as subSensorList's order
- TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
- List<String> valueMeasurementList = vectorPath.getMeasurementList();
- Set<String> allSensors = new HashSet<>(valueMeasurementList);
- allSensors.add("");
- boolean isDebug = context.isDebug();
- String filePath = resource.getTsFilePath();
- String deviceId = vectorPath.getDevice();
+ long startTime = System.nanoTime();
+ try {
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // load all the TimeseriesMetadata of vector, the first one is for time column and the
+ // remaining is for sub sensors
+ // the order of timeSeriesMetadata list is same as subSensorList's order
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ List<String> valueMeasurementList = vectorPath.getMeasurementList();
+ Set<String> allSensors = new HashSet<>(valueMeasurementList);
+ allSensors.add("");
+ boolean isDebug = context.isDebug();
+ String filePath = resource.getTsFilePath();
+ String deviceId = vectorPath.getDevice();
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- TimeseriesMetadata timeColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- if (timeColumn != null) {
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- new ArrayList<>(valueMeasurementList.size());
- // if all the queried aligned sensors does not exist, we will return null
- boolean exist = false;
- for (String valueMeasurement : valueMeasurementList) {
- TimeseriesMetadata valueColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- exist = (exist || (valueColumn != null));
- valueTimeSeriesMetadataList.add(valueColumn);
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ TimeseriesMetadata timeColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ if (timeColumn != null) {
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ new ArrayList<>(valueMeasurementList.size());
+ // if all the queried aligned sensors does not exist, we will return null
+ boolean exist = false;
+ for (String valueMeasurement : valueMeasurementList) {
+ TimeseriesMetadata valueColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ exist = (exist || (valueColumn != null));
+ valueTimeSeriesMetadataList.add(valueColumn);
+ }
+ if (exist) {
+ alignedTimeSeriesMetadata =
+ new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
+ alignedTimeSeriesMetadata.setChunkMetadataLoader(
+ new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
+ }
}
- if (exist) {
- alignedTimeSeriesMetadata =
- new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ alignedTimeSeriesMetadata =
+ (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath);
+ if (alignedTimeSeriesMetadata != null) {
alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
+ new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
}
}
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- alignedTimeSeriesMetadata =
- (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath);
- if (alignedTimeSeriesMetadata != null) {
- alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
- }
- }
- if (alignedTimeSeriesMetadata != null) {
- if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
- > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),
- alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {
- return null;
- }
+ if (alignedTimeSeriesMetadata != null) {
+ if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
+ > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
+ return null;
+ }
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {
+ return null;
+ }
- // set modifications to each aligned path
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
- boolean modified = false;
- for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
- if (valueTimeSeriesMetadataList.get(i) != null) {
- List<Modification> pathModifications =
- context.getPathModifications(
- resource.getModFile(), vectorPath.getPathWithMeasurement(i));
- valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
- modified = (modified || !pathModifications.isEmpty());
+ // set modifications to each aligned path
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
+ boolean modified = false;
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ if (valueTimeSeriesMetadataList.get(i) != null) {
+ List<Modification> pathModifications =
+ context.getPathModifications(
+ resource.getModFile(), vectorPath.getPathWithMeasurement(i));
+ valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
+ modified = (modified || !pathModifications.isEmpty());
+ }
}
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
}
- alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
+ return alignedTimeSeriesMetadata;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(
+ QueryStatistics.LOAD_TIME_SERIES_METADATA_ALIGNED, System.nanoTime() - startTime);
}
- return alignedTimeSeriesMetadata;
}
/**
@@ -271,7 +284,11 @@ public class FileLoaderUtils {
*/
public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata)
throws IOException {
- return timeSeriesMetadata.loadChunkMetadataList();
+ long startTime = System.nanoTime();
+ List<IChunkMetadata> chunkMetadataList = timeSeriesMetadata.loadChunkMetadataList();
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_CHUNK_METADATA_LIST, System.nanoTime() - startTime);
+ return chunkMetadataList;
}
/**
@@ -282,11 +299,17 @@ public class FileLoaderUtils {
*/
public static List<IPageReader> loadPageReaderList(
IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException {
- if (chunkMetaData == null) {
- throw new IOException("Can't init null chunkMeta");
+ long startTime = System.nanoTime();
+ try {
+ if (chunkMetaData == null) {
+ throw new IOException("Can't init null chunkMeta");
+ }
+ IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
+ IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);
+ return chunkReader.loadPageReaderList();
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_PAGE_READER_LIST, System.nanoTime() - startTime);
}
- IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
- IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);
- return chunkReader.loadPageReaderList();
}
}