You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/03/15 09:47:30 UTC
[iotdb] 01/01: Add Explain Query Support
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch MasterQueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bb0c0b90c47e970e08d42336be6e50bf8857e77e
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Mar 15 17:46:53 2021 +0800
Add Explain Query Support
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 6 ++++-
.../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++----
.../iotdb/db/engine/modification/Deletion.java | 16 +++++++++++++
.../engine/storagegroup/StorageGroupProcessor.java | 8 ++++---
.../db/engine/storagegroup/TsFileResource.java | 9 ++++----
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++
.../iotdb/db/query/context/QueryContext.java | 11 +++++++++
.../db/query/reader/chunk/DiskChunkLoader.java | 9 +++++---
.../chunk/metadata/DiskChunkMetadataLoader.java | 27 +++++++++++++++++++++-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 8 +++----
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 ++-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 4 ++--
.../tsfile/file/metadata/TimeseriesMetadata.java | 25 ++++++++++++++++++++
13 files changed, 127 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index f3462c6..556075f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -89,6 +89,10 @@ public class ChunkCache {
}
public Chunk get(ChunkMetadata chunkMetaData) throws IOException {
+ return get(chunkMetaData, false);
+ }
+
+ public Chunk get(ChunkMetadata chunkMetaData, boolean debug) throws IOException {
if (!CACHE_ENABLE) {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
@@ -134,7 +138,7 @@ public class ChunkCache {
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData);
}
return new Chunk(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d6ad1f4..0fe95da 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -122,9 +122,14 @@ public class TimeSeriesMetadataCache {
return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE;
}
- @SuppressWarnings("squid:S1860") // Suppress synchronize warning
public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors)
throws IOException {
+ return get(key, allSensors, false);
+ }
+
+ @SuppressWarnings("squid:S1860") // Suppress synchronize warning
+ public TimeseriesMetadata get(
+ TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug) throws IOException {
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@ -150,7 +155,7 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
@@ -174,7 +179,7 @@ public class TimeSeriesMetadataCache {
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
return null;
@@ -202,12 +207,12 @@ public class TimeSeriesMetadataCache {
}
}
if (timeseriesMetadata == null) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
}
return null;
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Get timeseries: {}.{} metadata in file: {} from cache: {}.",
key.device,
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
index 5ed33b3..5d41dbe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
@@ -88,4 +88,20 @@ public class Deletion extends Modification {
public int hashCode() {
return Objects.hash(super.hashCode(), startTime, endTime);
}
+
+ @Override
+ public String toString() {
+ return "Deletion{"
+ + "startTime="
+ + startTime
+ + ", endTime="
+ + endTime
+ + ", type="
+ + type
+ + ", path="
+ + path
+ + ", fileOffset="
+ + fileOffset
+ + '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index b5b76df..d2720f4 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1599,7 +1599,7 @@ public class StorageGroupProcessor {
boolean isSeq)
throws MetadataException {
- if (config.isDebugOn()) {
+ if (context.isDebug()) {
DEBUG_LOGGER.info(
"Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}",
deviceId.getFullPath(),
@@ -1618,7 +1618,8 @@ public class StorageGroupProcessor {
// for upgrade files and old files must be closed
for (TsFileResource tsFileResource : upgradeTsFileResources) {
- if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) {
+ if (!tsFileResource.isSatisfied(
+ deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1630,7 +1631,8 @@ public class StorageGroupProcessor {
}
for (TsFileResource tsFileResource : tsFileResources) {
- if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) {
+ if (!tsFileResource.isSatisfied(
+ deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 8a9cd3a..86ea847 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -541,9 +541,10 @@ public class TsFileResource {
}
/** @return true if the device is contained in the TsFile and it lives beyond TTL */
- public boolean isSatisfied(String deviceId, Filter timeFilter, boolean isSeq, long ttl) {
+ public boolean isSatisfied(
+ String deviceId, Filter timeFilter, boolean isSeq, long ttl, boolean debug) {
if (!getDevices().contains(deviceId)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
}
@@ -554,7 +555,7 @@ public class TsFileResource {
long endTime = closed || !isSeq ? getEndTime(deviceId) : Long.MAX_VALUE;
if (!isAlive(endTime, ttl)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of ttl!", deviceId, file);
}
return false;
@@ -562,7 +563,7 @@ public class TsFileResource {
if (timeFilter != null) {
boolean res = timeFilter.satisfyStartEndTime(startTime, endTime);
- if (config.isDebugOn() && !res) {
+ if (debug && !res) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of time filter!", deviceId, fsFactory);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index 5a5ad27..16719c4 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -75,6 +75,8 @@ public abstract class PhysicalPlan {
// a bridge from a cluster raft log to a physical plan
protected long index;
+ private boolean debug;
+
/** whether the plan can be split into more than one Plans. Only used in the cluster mode. */
public boolean canBeSplit() {
return canBeSplit;
@@ -117,6 +119,14 @@ public abstract class PhysicalPlan {
isQuery = query;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public void setDebug(boolean debug) {
+ this.debug = debug;
+ }
+
/**
* Serialize the plan into the given buffer. All necessary fields will be serialized.
*
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 3febfa9..326e71f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -49,12 +49,19 @@ public class QueryContext {
private long queryTimeLowerBound = Long.MIN_VALUE;
+ private boolean debug;
+
public QueryContext() {}
public QueryContext(long queryId) {
this.queryId = queryId;
}
+ public QueryContext(long queryId, boolean debug) {
+ this.queryId = queryId;
+ this.debug = debug;
+ }
+
/**
* Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
@@ -87,6 +94,10 @@ public class QueryContext {
return queryId;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
public long getQueryTimeLowerBound() {
return queryTimeLowerBound;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 9c02149..36619e2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.query.reader.chunk;
import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
@@ -29,13 +30,15 @@ import java.io.IOException;
/** To read one chunk from disk, and only used in iotdb server module */
public class DiskChunkLoader implements IChunkLoader {
- public DiskChunkLoader() {
- // do nothing
+ private final QueryContext context;
+
+ public DiskChunkLoader(QueryContext context) {
+ this.context = context;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData);
+ return ChunkCache.getInstance().get(chunkMetaData, context.isDebug());
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index ebb9521..cdaf128 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -29,6 +29,9 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.List;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@@ -39,6 +42,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
// time filter or value filter, only used to check time range
private final Filter filter;
+ private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
+
public DiskChunkMetadataLoader(
TsFileResource resource, PartialPath seriesPath, QueryContext context, Filter filter) {
this.resource = resource;
@@ -69,6 +74,12 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
metadata.setVersion(resource.getVersion());
}
}
+
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After removed by filter Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
return chunkMetadataList;
}
@@ -80,9 +91,23 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath);
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info(
+ "Modifications size is {} for file Path: {} ",
+ pathModifications.size(),
+ resource.getTsFilePath());
+ pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
+
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After modification Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
// it is ok, even if it is not thread safe, because the cost of creating a DiskChunkLoader is
// very cheap.
chunkMetadataList.forEach(
@@ -90,7 +115,7 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
if (chunkMetadata.getChunkLoader() == null) {
chunkMetadata.setFilePath(resource.getTsFilePath());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new DiskChunkLoader());
+ chunkMetadata.setChunkLoader(new DiskChunkLoader(context));
}
});
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index edc0725..90987c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -728,7 +728,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (costTime >= config.getSlowQueryThreshold()) {
SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement);
}
- if (config.isDebugOn()) {
+ if (plan.isDebug()) {
SLOW_SQL_LOGGER.info(
"ChunkCache used memory proportion: {}\n"
+ "TimeSeriesMetadataCache used memory proportion: {}",
@@ -1029,14 +1029,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, SQLException, TException, InterruptedException {
- QueryContext context = genQueryContext(queryId);
+ QueryContext context = genQueryContext(queryId, physicalPlan.isDebug());
QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context);
queryId2DataSet.put(queryId, queryDataSet);
return queryDataSet;
}
- protected QueryContext genQueryContext(long queryId) {
- return new QueryContext(queryId);
+ protected QueryContext genQueryContext(long queryId, boolean debug) {
+ return new QueryContext(queryId, debug);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index a829b69..0a55b54 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -104,7 +104,8 @@ public class FileLoaderUtils {
resource.getTsFilePath(),
seriesPath.getDevice(),
seriesPath.getMeasurement()),
- allSensors);
+ allSensors,
+ context.isDebug());
if (timeSeriesMetadata != null) {
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index de6681d..07260bc 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -93,8 +93,8 @@ public class ChunkMetadata implements Accountable {
@Override
public String toString() {
return String.format(
- "measurementId: %s, datatype: %s, version: %d, " + "Statistics: %s, deleteIntervalList: %s",
- measurementUid, tsDataType, version, statistics, deleteIntervalList);
+ "measurementId: %s, datatype: %s, version: %d, Statistics: %s, deleteIntervalList: %s, filePath: %s",
+ measurementUid, tsDataType, version, statistics, deleteIntervalList, filePath);
}
public long getNumOfPoints() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 20dbf05..7344c2e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -231,4 +231,29 @@ public class TimeseriesMetadata implements Accountable {
public void setChunkMetadataList(ArrayList<ChunkMetadata> chunkMetadataList) {
this.chunkMetadataList = chunkMetadataList;
}
+
+ @Override
+ public String toString() {
+ return "TimeseriesMetadata{"
+ + "startOffsetOfChunkMetaDataList="
+ + startOffsetOfChunkMetaDataList
+ + ", timeSeriesMetadataType="
+ + timeSeriesMetadataType
+ + ", chunkMetaDataListDataSize="
+ + chunkMetaDataListDataSize
+ + ", measurementId='"
+ + measurementId
+ + '\''
+ + ", dataType="
+ + dataType
+ + ", statistics="
+ + statistics
+ + ", modified="
+ + modified
+ + ", isSeq="
+ + isSeq
+ + ", chunkMetadataList="
+ + chunkMetadataList
+ + '}';
+ }
}