You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/03/15 09:47:29 UTC
[iotdb] branch MasterQueryDebug created (now bb0c0b9)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch MasterQueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at bb0c0b9 Add Explain Query Support
This branch includes the following new commits:
new bb0c0b9 Add Explain Query Support
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Add Explain Query Support
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch MasterQueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bb0c0b90c47e970e08d42336be6e50bf8857e77e
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Mar 15 17:46:53 2021 +0800
Add Explain Query Support
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 6 ++++-
.../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++----
.../iotdb/db/engine/modification/Deletion.java | 16 +++++++++++++
.../engine/storagegroup/StorageGroupProcessor.java | 8 ++++---
.../db/engine/storagegroup/TsFileResource.java | 9 ++++----
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++
.../iotdb/db/query/context/QueryContext.java | 11 +++++++++
.../db/query/reader/chunk/DiskChunkLoader.java | 9 +++++---
.../chunk/metadata/DiskChunkMetadataLoader.java | 27 +++++++++++++++++++++-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 8 +++----
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 3 ++-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 4 ++--
.../tsfile/file/metadata/TimeseriesMetadata.java | 25 ++++++++++++++++++++
13 files changed, 127 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index f3462c6..556075f 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -89,6 +89,10 @@ public class ChunkCache {
}
public Chunk get(ChunkMetadata chunkMetaData) throws IOException {
+ return get(chunkMetaData, false);
+ }
+
+ public Chunk get(ChunkMetadata chunkMetaData, boolean debug) throws IOException {
if (!CACHE_ENABLE) {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
@@ -134,7 +138,7 @@ public class ChunkCache {
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData);
}
return new Chunk(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d6ad1f4..0fe95da 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -122,9 +122,14 @@ public class TimeSeriesMetadataCache {
return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE;
}
- @SuppressWarnings("squid:S1860") // Suppress synchronize warning
public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors)
throws IOException {
+ return get(key, allSensors, false);
+ }
+
+ @SuppressWarnings("squid:S1860") // Suppress synchronize warning
+ public TimeseriesMetadata get(
+ TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug) throws IOException {
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@ -150,7 +155,7 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
DEBUG_LOGGER.info("Device: {}, all sensors: {}", key.device, allSensors);
@@ -174,7 +179,7 @@ public class TimeSeriesMetadataCache {
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
return null;
@@ -202,12 +207,12 @@ public class TimeSeriesMetadataCache {
}
}
if (timeseriesMetadata == null) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("The file doesn't have this time series {}.", key);
}
return null;
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Get timeseries: {}.{} metadata in file: {} from cache: {}.",
key.device,
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
index 5ed33b3..5d41dbe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
@@ -88,4 +88,20 @@ public class Deletion extends Modification {
public int hashCode() {
return Objects.hash(super.hashCode(), startTime, endTime);
}
+
+ @Override
+ public String toString() {
+ return "Deletion{"
+ + "startTime="
+ + startTime
+ + ", endTime="
+ + endTime
+ + ", type="
+ + type
+ + ", path="
+ + path
+ + ", fileOffset="
+ + fileOffset
+ + '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index b5b76df..d2720f4 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1599,7 +1599,7 @@ public class StorageGroupProcessor {
boolean isSeq)
throws MetadataException {
- if (config.isDebugOn()) {
+ if (context.isDebug()) {
DEBUG_LOGGER.info(
"Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}",
deviceId.getFullPath(),
@@ -1618,7 +1618,8 @@ public class StorageGroupProcessor {
// for upgrade files and old files must be closed
for (TsFileResource tsFileResource : upgradeTsFileResources) {
- if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) {
+ if (!tsFileResource.isSatisfied(
+ deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1630,7 +1631,8 @@ public class StorageGroupProcessor {
}
for (TsFileResource tsFileResource : tsFileResources) {
- if (!tsFileResource.isSatisfied(deviceId.getFullPath(), timeFilter, isSeq, dataTTL)) {
+ if (!tsFileResource.isSatisfied(
+ deviceId.getFullPath(), timeFilter, isSeq, dataTTL, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 8a9cd3a..86ea847 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -541,9 +541,10 @@ public class TsFileResource {
}
/** @return true if the device is contained in the TsFile and it lives beyond TTL */
- public boolean isSatisfied(String deviceId, Filter timeFilter, boolean isSeq, long ttl) {
+ public boolean isSatisfied(
+ String deviceId, Filter timeFilter, boolean isSeq, long ttl, boolean debug) {
if (!getDevices().contains(deviceId)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of no device!", deviceId, file);
}
@@ -554,7 +555,7 @@ public class TsFileResource {
long endTime = closed || !isSeq ? getEndTime(deviceId) : Long.MAX_VALUE;
if (!isAlive(endTime, ttl)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of ttl!", deviceId, file);
}
return false;
@@ -562,7 +563,7 @@ public class TsFileResource {
if (timeFilter != null) {
boolean res = timeFilter.satisfyStartEndTime(startTime, endTime);
- if (config.isDebugOn() && !res) {
+ if (debug && !res) {
DEBUG_LOGGER.info(
"Path: {} file {} is not satisfied because of time filter!", deviceId, fsFactory);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index 5a5ad27..16719c4 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -75,6 +75,8 @@ public abstract class PhysicalPlan {
// a bridge from a cluster raft log to a physical plan
protected long index;
+ private boolean debug;
+
/** whether the plan can be split into more than one Plans. Only used in the cluster mode. */
public boolean canBeSplit() {
return canBeSplit;
@@ -117,6 +119,14 @@ public abstract class PhysicalPlan {
isQuery = query;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public void setDebug(boolean debug) {
+ this.debug = debug;
+ }
+
/**
* Serialize the plan into the given buffer. All necessary fields will be serialized.
*
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 3febfa9..326e71f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -49,12 +49,19 @@ public class QueryContext {
private long queryTimeLowerBound = Long.MIN_VALUE;
+ private boolean debug;
+
public QueryContext() {}
public QueryContext(long queryId) {
this.queryId = queryId;
}
+ public QueryContext(long queryId, boolean debug) {
+ this.queryId = queryId;
+ this.debug = debug;
+ }
+
/**
* Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
@@ -87,6 +94,10 @@ public class QueryContext {
return queryId;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
public long getQueryTimeLowerBound() {
return queryTimeLowerBound;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
index 9c02149..36619e2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/DiskChunkLoader.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.query.reader.chunk;
import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.common.Chunk;
import org.apache.iotdb.tsfile.read.controller.IChunkLoader;
@@ -29,13 +30,15 @@ import java.io.IOException;
/** To read one chunk from disk, and only used in iotdb server module */
public class DiskChunkLoader implements IChunkLoader {
- public DiskChunkLoader() {
- // do nothing
+ private final QueryContext context;
+
+ public DiskChunkLoader(QueryContext context) {
+ this.context = context;
}
@Override
public Chunk loadChunk(ChunkMetadata chunkMetaData) throws IOException {
- return ChunkCache.getInstance().get(chunkMetaData);
+ return ChunkCache.getInstance().get(chunkMetaData, context.isDebug());
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index ebb9521..cdaf128 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -29,6 +29,9 @@ import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.List;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@@ -39,6 +42,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
// time filter or value filter, only used to check time range
private final Filter filter;
+ private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
+
public DiskChunkMetadataLoader(
TsFileResource resource, PartialPath seriesPath, QueryContext context, Filter filter) {
this.resource = resource;
@@ -69,6 +74,12 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
metadata.setVersion(resource.getVersion());
}
}
+
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After removed by filter Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
return chunkMetadataList;
}
@@ -80,9 +91,23 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath);
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info(
+ "Modifications size is {} for file Path: {} ",
+ pathModifications.size(),
+ resource.getTsFilePath());
+ pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
+
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After modification Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
// it is ok, even if it is not thread safe, because the cost of creating a DiskChunkLoader is
// very cheap.
chunkMetadataList.forEach(
@@ -90,7 +115,7 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
if (chunkMetadata.getChunkLoader() == null) {
chunkMetadata.setFilePath(resource.getTsFilePath());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new DiskChunkLoader());
+ chunkMetadata.setChunkLoader(new DiskChunkLoader(context));
}
});
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index edc0725..90987c5 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -728,7 +728,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (costTime >= config.getSlowQueryThreshold()) {
SLOW_SQL_LOGGER.info("Cost: {} ms, sql is {}", costTime, statement);
}
- if (config.isDebugOn()) {
+ if (plan.isDebug()) {
SLOW_SQL_LOGGER.info(
"ChunkCache used memory proportion: {}\n"
+ "TimeSeriesMetadataCache used memory proportion: {}",
@@ -1029,14 +1029,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, SQLException, TException, InterruptedException {
- QueryContext context = genQueryContext(queryId);
+ QueryContext context = genQueryContext(queryId, physicalPlan.isDebug());
QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context);
queryId2DataSet.put(queryId, queryDataSet);
return queryDataSet;
}
- protected QueryContext genQueryContext(long queryId) {
- return new QueryContext(queryId);
+ protected QueryContext genQueryContext(long queryId, boolean debug) {
+ return new QueryContext(queryId, debug);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index a829b69..0a55b54 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -104,7 +104,8 @@ public class FileLoaderUtils {
resource.getTsFilePath(),
seriesPath.getDevice(),
seriesPath.getMeasurement()),
- allSensors);
+ allSensors,
+ context.isDebug());
if (timeSeriesMetadata != null) {
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index de6681d..07260bc 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -93,8 +93,8 @@ public class ChunkMetadata implements Accountable {
@Override
public String toString() {
return String.format(
- "measurementId: %s, datatype: %s, version: %d, " + "Statistics: %s, deleteIntervalList: %s",
- measurementUid, tsDataType, version, statistics, deleteIntervalList);
+ "measurementId: %s, datatype: %s, version: %d, Statistics: %s, deleteIntervalList: %s, filePath: %s",
+ measurementUid, tsDataType, version, statistics, deleteIntervalList, filePath);
}
public long getNumOfPoints() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 20dbf05..7344c2e 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -231,4 +231,29 @@ public class TimeseriesMetadata implements Accountable {
public void setChunkMetadataList(ArrayList<ChunkMetadata> chunkMetadataList) {
this.chunkMetadataList = chunkMetadataList;
}
+
+ @Override
+ public String toString() {
+ return "TimeseriesMetadata{"
+ + "startOffsetOfChunkMetaDataList="
+ + startOffsetOfChunkMetaDataList
+ + ", timeSeriesMetadataType="
+ + timeSeriesMetadataType
+ + ", chunkMetaDataListDataSize="
+ + chunkMetaDataListDataSize
+ + ", measurementId='"
+ + measurementId
+ + '\''
+ + ", dataType="
+ + dataType
+ + ", statistics="
+ + statistics
+ + ", modified="
+ + modified
+ + ", isSeq="
+ + isSeq
+ + ", chunkMetadataList="
+ + chunkMetadataList
+ + '}';
+ }
}