You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/03/15 03:11:15 UTC
[iotdb] 01/01: add explain grammar
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f836c4bb7c60adacad79b22a82cea191e6e22698
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Mar 15 11:10:39 2021 +0800
add explain grammar
---
grafana/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
pom.xml | 6 +++---
.../org/apache/iotdb/db/engine/cache/ChunkCache.java | 7 ++++++-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 9 +++++++--
.../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++++-----
.../iotdb/db/engine/modification/Modification.java | 9 +++++++++
.../engine/storagegroup/StorageGroupProcessor.java | 14 +++++++-------
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++++
.../apache/iotdb/db/query/context/QueryContext.java | 11 +++++++++++
.../groupby/GroupByWithValueFilterDataSet.java | 2 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 20 ++++++++++++++++++--
.../org/apache/iotdb/db/service/TSServiceImpl.java | 8 ++++----
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
14 files changed, 89 insertions(+), 28 deletions(-)
diff --git a/grafana/pom.xml b/grafana/pom.xml
index 357350a..b605e65 100644
--- a/grafana/pom.xml
+++ b/grafana/pom.xml
@@ -165,7 +165,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>${start-class}</mainClass>
</transformer>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index b44bb46..090fdb0 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -223,7 +223,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore />
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
diff --git a/pom.xml b/pom.xml
index 3a15f55..ec39eb2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,7 @@
<sonar.exclusions>**/generated-sources</sonar.exclusions>
<!-- By default, the argLine is empty-->
<gson.version>2.8.6</gson.version>
- <argLine />
+ <argLine/>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
@@ -599,7 +599,7 @@
<id>enforce-version-convergence</id>
<configuration>
<rules>
- <dependencyConvergence />
+ <dependencyConvergence/>
</rules>
</configuration>
<goals>
@@ -645,7 +645,7 @@
</requireJavaVersion>
<!-- Disabled for now as it breaks the ability to build single modules -->
<!--reactorModuleConvergence/-->
- <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies" />
+ <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
</rules>
</configuration>
</execution>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 6fccb8b..545b666 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -84,6 +84,11 @@ public class ChunkCache {
}
public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader) throws IOException {
+ return get(chunkMetaData, reader, false);
+ }
+
+ public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader, boolean debug)
+ throws IOException {
if (!CACHE_ENABLE) {
Chunk chunk = reader.readMemChunk(chunkMetaData);
return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
@@ -118,7 +123,7 @@ public class ChunkCache {
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData);
}
return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeleteIntervalList());
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index f8fd5f3..356da90 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -102,11 +102,16 @@ public class ChunkMetadataCache {
return ChunkMetadataCacheSingleton.INSTANCE;
}
+ public List<ChunkMetadata> get(String filePath, Path seriesPath,
+ TimeseriesMetadata timeseriesMetadata) throws IOException {
+ return get(filePath, seriesPath, timeseriesMetadata, false);
+ }
+
/**
* get {@link ChunkMetadata}. THREAD SAFE.
*/
public List<ChunkMetadata> get(String filePath, Path seriesPath,
- TimeseriesMetadata timeseriesMetadata) throws IOException {
+ TimeseriesMetadata timeseriesMetadata, boolean debug) throws IOException {
if (timeseriesMetadata == null) {
return Collections.emptyList();
}
@@ -144,7 +149,7 @@ public class ChunkMetadataCache {
lock.writeLock().unlock();
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Chunk meta data list size: " + chunkMetadataList.size() + " key is: " + key.getString());
chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d24160e..9aef0db 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -106,9 +106,14 @@ public class TimeSeriesMetadataCache {
return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE;
}
- @SuppressWarnings("squid:S1860") // Suppress synchronize warning
public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors)
throws IOException {
+ return get(key, allSensors, false);
+ }
+
+ @SuppressWarnings("squid:S1860") // Suppress synchronize warning
+ public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug)
+ throws IOException {
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@ -134,7 +139,7 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Cache miss: " + key.device + "." + key.measurement + " metadata in file: "
+ key.filePath);
@@ -159,7 +164,7 @@ public class TimeSeriesMetadataCache {
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
return null;
@@ -185,12 +190,12 @@ public class TimeSeriesMetadataCache {
}
}
if (timeseriesMetadata == null) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("The file doesn't have this time series {}", key);
}
return null;
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Get timeseries: {}. {} metadata in file: {} from cache: {}",
key.device, key.measurement, key.filePath, timeseriesMetadata);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
index 3e3f46e..2c7a757 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
@@ -94,4 +94,13 @@ public abstract class Modification {
public int hashCode() {
return Objects.hash(type, path, versionNum);
}
+
+ @Override
+ public String toString() {
+ return "Modification{" +
+ "type=" + type +
+ ", path=" + path +
+ ", versionNum=" + versionNum +
+ '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 751789f..4161201 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1415,7 +1415,7 @@ public class StorageGroupProcessor {
boolean isSeq)
throws MetadataException {
- if (config.isDebugOn()) {
+ if (context.isDebug()) {
DEBUG_LOGGER
.info("Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}", deviceId.getFullPath(),
measurementId, tsFileResources, isSeq, (timeFilter == null ? "null" : timeFilter));
@@ -1429,7 +1429,7 @@ public class StorageGroupProcessor {
context.setQueryTimeLowerBound(timeLowerBound);
for (TsFileResource tsFileResource : tsFileResources) {
- if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) {
+ if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1450,7 +1450,7 @@ public class StorageGroupProcessor {
}
// for upgrade files and old files must be closed
for (TsFileResource tsFileResource : upgradeTsFileResources) {
- if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) {
+ if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1467,9 +1467,9 @@ public class StorageGroupProcessor {
* @return true if the device is contained in the TsFile and it lives beyond TTL
*/
private boolean isTsFileResourceSatisfied(TsFileResource tsFileResource, String deviceId,
- Filter timeFilter, boolean isSeq) {
+ Filter timeFilter, boolean isSeq, boolean debug) {
if (!tsFileResource.containsDevice(deviceId)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of no device!", deviceId,
tsFileResource);
}
@@ -1482,7 +1482,7 @@ public class StorageGroupProcessor {
: Long.MAX_VALUE;
if (!isAlive(endTime)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER
.info("Path: {} file {} is not satisfied because of ttl!", deviceId, tsFileResource);
}
@@ -1491,7 +1491,7 @@ public class StorageGroupProcessor {
if (timeFilter != null) {
boolean res = timeFilter.satisfyStartEndTime(startTime, endTime);
- if (config.isDebugOn() && !res) {
+ if (debug && !res) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of time filter!", deviceId,
tsFileResource);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index c2842f1..1beb2c9 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -63,6 +63,8 @@ public abstract class PhysicalPlan {
// a bridge from a cluster raft log to a physical plan
protected long index;
+ private boolean debug;
+
/**
* whether the plan can be split into more than one Plans. Only used in the cluster mode.
*/
@@ -171,6 +173,14 @@ public abstract class PhysicalPlan {
this.loginUserName = loginUserName;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public void setDebug(boolean debug) {
+ this.debug = debug;
+ }
+
public static class Factory {
private Factory() {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 863d4c3..a8c11fd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -50,6 +50,8 @@ public class QueryContext {
private long queryTimeLowerBound = Long.MIN_VALUE;
+ private boolean debug;
+
public QueryContext() {
}
@@ -57,6 +59,15 @@ public class QueryContext {
this.queryId = queryId;
}
+ public QueryContext(long queryId, boolean debug) {
+ this.queryId = queryId;
+ this.debug = debug;
+ }
+
+ public boolean isDebug() {
+ return debug;
+ }
+
/**
* Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
index bd37495..64bad5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
@@ -74,7 +74,7 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
}
public GroupByWithValueFilterDataSet(long queryId, GroupByTimePlan groupByTimePlan) {
- super(new QueryContext(queryId), groupByTimePlan);
+ super(new QueryContext(queryId, groupByTimePlan.isDebug()), groupByTimePlan);
this.allDataReaderList = new ArrayList<>();
this.timeStampFetchSize = IoTDBDescriptor.getInstance().getConfig().getBatchSize();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 95466e9..8318365 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -31,9 +31,13 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
+ private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
+
private TsFileResource resource;
private PartialPath seriesPath;
private QueryContext context;
@@ -51,8 +55,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@Override
public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
throws IOException {
- List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
- .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata);
+ List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache.getInstance()
+ .get(resource.getTsFilePath(), seriesPath, timeseriesMetadata, context.isDebug());
setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
@@ -82,10 +86,22 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath);
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("Modifications file Path: {} ", resource.getTsFilePath());
+ pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
+ if (context.isDebug()) {
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After modification Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+ }
+
for (ChunkMetadata data : chunkMetadataList) {
data.setChunkLoader(new DiskChunkLoader(resource));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 500ae7a..da62967 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -767,7 +767,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (costTime >= config.getSlowQueryThreshold()) {
SLOW_SQL_LOGGER.info("Cost: " + costTime + " ms, sql is " + statement);
}
- if (config.isDebugOn()) {
+ if (plan.isDebug()) {
SLOW_SQL_LOGGER.info("ChunkCache used memory proportion: " + ChunkCache.getInstance()
.getUsedMemoryProportion() + "\nChunkMetadataCache used memory proportion: "
+ ChunkMetadataCache.getInstance().getUsedMemoryProportion()
@@ -1113,14 +1113,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, SQLException, TException, InterruptedException {
- QueryContext context = genQueryContext(queryId);
+ QueryContext context = genQueryContext(queryId, physicalPlan.isDebug());
QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context);
queryId2DataSet.put(queryId, queryDataSet);
return queryDataSet;
}
- protected QueryContext genQueryContext(long queryId) {
- return new QueryContext(queryId);
+ protected QueryContext genQueryContext(long queryId, boolean debug) {
+ return new QueryContext(queryId, debug);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index b3af734..dea1b60 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -94,7 +94,7 @@ public class FileLoaderUtils {
}
timeSeriesMetadata = TimeSeriesMetadataCache.getInstance()
.get(new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(resource.getTsFilePath(),
- seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors);
+ seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors, context.isDebug());
if (timeSeriesMetadata != null) {
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, seriesPath, context, filter));