You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2021/03/15 03:11:14 UTC
[iotdb] branch QueryDebug created (now f836c4b)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch QueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at f836c4b add explain grammar
This branch includes the following new commits:
new f836c4b add explain grammar
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: add explain grammar
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryDebug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f836c4bb7c60adacad79b22a82cea191e6e22698
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Mar 15 11:10:39 2021 +0800
add explain grammar
---
grafana/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
pom.xml | 6 +++---
.../org/apache/iotdb/db/engine/cache/ChunkCache.java | 7 ++++++-
.../iotdb/db/engine/cache/ChunkMetadataCache.java | 9 +++++++--
.../db/engine/cache/TimeSeriesMetadataCache.java | 15 ++++++++++-----
.../iotdb/db/engine/modification/Modification.java | 9 +++++++++
.../engine/storagegroup/StorageGroupProcessor.java | 14 +++++++-------
.../apache/iotdb/db/qp/physical/PhysicalPlan.java | 10 ++++++++++
.../apache/iotdb/db/query/context/QueryContext.java | 11 +++++++++++
.../groupby/GroupByWithValueFilterDataSet.java | 2 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 20 ++++++++++++++++++--
.../org/apache/iotdb/db/service/TSServiceImpl.java | 8 ++++----
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 2 +-
14 files changed, 89 insertions(+), 28 deletions(-)
diff --git a/grafana/pom.xml b/grafana/pom.xml
index 357350a..b605e65 100644
--- a/grafana/pom.xml
+++ b/grafana/pom.xml
@@ -165,7 +165,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>${start-class}</mainClass>
</transformer>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index b44bb46..090fdb0 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -223,7 +223,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore />
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
diff --git a/pom.xml b/pom.xml
index 3a15f55..ec39eb2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,7 @@
<sonar.exclusions>**/generated-sources</sonar.exclusions>
<!-- By default, the argLine is empty-->
<gson.version>2.8.6</gson.version>
- <argLine />
+ <argLine/>
</properties>
<!--
if we claim dependencies in dependencyManagement, then we do not claim
@@ -599,7 +599,7 @@
<id>enforce-version-convergence</id>
<configuration>
<rules>
- <dependencyConvergence />
+ <dependencyConvergence/>
</rules>
</configuration>
<goals>
@@ -645,7 +645,7 @@
</requireJavaVersion>
<!-- Disabled for now as it breaks the ability to build single modules -->
<!--reactorModuleConvergence/-->
- <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies" />
+ <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
</rules>
</configuration>
</execution>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 6fccb8b..545b666 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -84,6 +84,11 @@ public class ChunkCache {
}
public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader) throws IOException {
+ return get(chunkMetaData, reader, false);
+ }
+
+ public Chunk get(ChunkMetadata chunkMetaData, TsFileSequenceReader reader, boolean debug)
+ throws IOException {
if (!CACHE_ENABLE) {
Chunk chunk = reader.readMemChunk(chunkMetaData);
return new Chunk(chunk.getHeader(), chunk.getData().duplicate(),
@@ -118,7 +123,7 @@ public class ChunkCache {
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("get chunk from cache whose meta data is: " + chunkMetaData);
}
return new Chunk(chunk.getHeader(), chunk.getData().duplicate(), chunk.getDeleteIntervalList());
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index f8fd5f3..356da90 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -102,11 +102,16 @@ public class ChunkMetadataCache {
return ChunkMetadataCacheSingleton.INSTANCE;
}
+ public List<ChunkMetadata> get(String filePath, Path seriesPath,
+ TimeseriesMetadata timeseriesMetadata) throws IOException {
+ return get(filePath, seriesPath, timeseriesMetadata, false);
+ }
+
/**
* get {@link ChunkMetadata}. THREAD SAFE.
*/
public List<ChunkMetadata> get(String filePath, Path seriesPath,
- TimeseriesMetadata timeseriesMetadata) throws IOException {
+ TimeseriesMetadata timeseriesMetadata, boolean debug) throws IOException {
if (timeseriesMetadata == null) {
return Collections.emptyList();
}
@@ -144,7 +149,7 @@ public class ChunkMetadataCache {
lock.writeLock().unlock();
}
}
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Chunk meta data list size: " + chunkMetadataList.size() + " key is: " + key.getString());
chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d24160e..9aef0db 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -106,9 +106,14 @@ public class TimeSeriesMetadataCache {
return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE;
}
- @SuppressWarnings("squid:S1860") // Suppress synchronize warning
public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors)
throws IOException {
+ return get(key, allSensors, false);
+ }
+
+ @SuppressWarnings("squid:S1860") // Suppress synchronize warning
+ public TimeseriesMetadata get(TimeSeriesMetadataCacheKey key, Set<String> allSensors, boolean debug)
+ throws IOException {
if (!CACHE_ENABLE) {
// bloom filter part
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
@@ -134,7 +139,7 @@ public class TimeSeriesMetadataCache {
cacheHitNum.incrementAndGet();
printCacheLog(true);
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Cache miss: " + key.device + "." + key.measurement + " metadata in file: "
+ key.filePath);
@@ -159,7 +164,7 @@ public class TimeSeriesMetadataCache {
TsFileSequenceReader reader = FileReaderManager.getInstance().get(key.filePath, true);
BloomFilter bloomFilter = reader.readBloomFilter();
if (bloomFilter != null && !bloomFilter.contains(path.getFullPath())) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("TimeSeries meta data {} is filter by bloomFilter!", key);
}
return null;
@@ -185,12 +190,12 @@ public class TimeSeriesMetadataCache {
}
}
if (timeseriesMetadata == null) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("The file doesn't have this time series {}", key);
}
return null;
} else {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info(
"Get timeseries: {}. {} metadata in file: {} from cache: {}",
key.device, key.measurement, key.filePath, timeseriesMetadata);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
index 3e3f46e..2c7a757 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Modification.java
@@ -94,4 +94,13 @@ public abstract class Modification {
public int hashCode() {
return Objects.hash(type, path, versionNum);
}
+
+ @Override
+ public String toString() {
+ return "Modification{" +
+ "type=" + type +
+ ", path=" + path +
+ ", versionNum=" + versionNum +
+ '}';
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 751789f..4161201 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1415,7 +1415,7 @@ public class StorageGroupProcessor {
boolean isSeq)
throws MetadataException {
- if (config.isDebugOn()) {
+ if (context.isDebug()) {
DEBUG_LOGGER
.info("Path: {}.{}, get tsfile list: {} isSeq: {} timefilter: {}", deviceId.getFullPath(),
measurementId, tsFileResources, isSeq, (timeFilter == null ? "null" : timeFilter));
@@ -1429,7 +1429,7 @@ public class StorageGroupProcessor {
context.setQueryTimeLowerBound(timeLowerBound);
for (TsFileResource tsFileResource : tsFileResources) {
- if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) {
+ if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1450,7 +1450,7 @@ public class StorageGroupProcessor {
}
// for upgrade files and old files must be closed
for (TsFileResource tsFileResource : upgradeTsFileResources) {
- if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq)) {
+ if (!isTsFileResourceSatisfied(tsFileResource, deviceId.getFullPath(), timeFilter, isSeq, context.isDebug())) {
continue;
}
closeQueryLock.readLock().lock();
@@ -1467,9 +1467,9 @@ public class StorageGroupProcessor {
* @return true if the device is contained in the TsFile and it lives beyond TTL
*/
private boolean isTsFileResourceSatisfied(TsFileResource tsFileResource, String deviceId,
- Filter timeFilter, boolean isSeq) {
+ Filter timeFilter, boolean isSeq, boolean debug) {
if (!tsFileResource.containsDevice(deviceId)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of no device!", deviceId,
tsFileResource);
}
@@ -1482,7 +1482,7 @@ public class StorageGroupProcessor {
: Long.MAX_VALUE;
if (!isAlive(endTime)) {
- if (config.isDebugOn()) {
+ if (debug) {
DEBUG_LOGGER
.info("Path: {} file {} is not satisfied because of ttl!", deviceId, tsFileResource);
}
@@ -1491,7 +1491,7 @@ public class StorageGroupProcessor {
if (timeFilter != null) {
boolean res = timeFilter.satisfyStartEndTime(startTime, endTime);
- if (config.isDebugOn() && !res) {
+ if (debug && !res) {
DEBUG_LOGGER.info("Path: {} file {} is not satisfied because of time filter!", deviceId,
tsFileResource);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index c2842f1..1beb2c9 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -63,6 +63,8 @@ public abstract class PhysicalPlan {
// a bridge from a cluster raft log to a physical plan
protected long index;
+ private boolean debug;
+
/**
* whether the plan can be split into more than one Plans. Only used in the cluster mode.
*/
@@ -171,6 +173,14 @@ public abstract class PhysicalPlan {
this.loginUserName = loginUserName;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public void setDebug(boolean debug) {
+ this.debug = debug;
+ }
+
public static class Factory {
private Factory() {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index 863d4c3..a8c11fd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -50,6 +50,8 @@ public class QueryContext {
private long queryTimeLowerBound = Long.MIN_VALUE;
+ private boolean debug;
+
public QueryContext() {
}
@@ -57,6 +59,15 @@ public class QueryContext {
this.queryId = queryId;
}
+ public QueryContext(long queryId, boolean debug) {
+ this.queryId = queryId;
+ this.debug = debug;
+ }
+
+ public boolean isDebug() {
+ return debug;
+ }
+
/**
* Find the modifications of timeseries 'path' in 'modFile'. If they are not in the cache, read
* them from 'modFile' and put then into the cache.
diff --git a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
index bd37495..64bad5f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByWithValueFilterDataSet.java
@@ -74,7 +74,7 @@ public class GroupByWithValueFilterDataSet extends GroupByEngineDataSet {
}
public GroupByWithValueFilterDataSet(long queryId, GroupByTimePlan groupByTimePlan) {
- super(new QueryContext(queryId), groupByTimePlan);
+ super(new QueryContext(queryId, groupByTimePlan.isDebug()), groupByTimePlan);
this.allDataReaderList = new ArrayList<>();
this.timeStampFetchSize = IoTDBDescriptor.getInstance().getConfig().getBatchSize();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 95466e9..8318365 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -31,9 +31,13 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
+ private static final Logger DEBUG_LOGGER = LoggerFactory.getLogger("QUERY_DEBUG");
+
private TsFileResource resource;
private PartialPath seriesPath;
private QueryContext context;
@@ -51,8 +55,8 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
@Override
public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
throws IOException {
- List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
- .getInstance().get(resource.getTsFilePath(), seriesPath, timeseriesMetadata);
+ List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache.getInstance()
+ .get(resource.getTsFilePath(), seriesPath, timeseriesMetadata, context.isDebug());
setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
@@ -82,10 +86,22 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
List<Modification> pathModifications =
context.getPathModifications(resource.getModFile(), seriesPath);
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("Modifications file Path: {} ", resource.getTsFilePath());
+ pathModifications.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+
if (!pathModifications.isEmpty()) {
QueryUtils.modifyChunkMetaData(chunkMetadataList, pathModifications);
}
+ if (context.isDebug()) {
+ if (context.isDebug()) {
+ DEBUG_LOGGER.info("After modification Chunk meta data list is: ");
+ chunkMetadataList.forEach(c -> DEBUG_LOGGER.info(c.toString()));
+ }
+ }
+
for (ChunkMetadata data : chunkMetadataList) {
data.setChunkLoader(new DiskChunkLoader(resource));
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 500ae7a..da62967 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -767,7 +767,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
if (costTime >= config.getSlowQueryThreshold()) {
SLOW_SQL_LOGGER.info("Cost: " + costTime + " ms, sql is " + statement);
}
- if (config.isDebugOn()) {
+ if (plan.isDebug()) {
SLOW_SQL_LOGGER.info("ChunkCache used memory proportion: " + ChunkCache.getInstance()
.getUsedMemoryProportion() + "\nChunkMetadataCache used memory proportion: "
+ ChunkMetadataCache.getInstance().getUsedMemoryProportion()
@@ -1113,14 +1113,14 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
throws QueryProcessException, QueryFilterOptimizationException, StorageEngineException,
IOException, MetadataException, SQLException, TException, InterruptedException {
- QueryContext context = genQueryContext(queryId);
+ QueryContext context = genQueryContext(queryId, physicalPlan.isDebug());
QueryDataSet queryDataSet = executor.processQuery(physicalPlan, context);
queryId2DataSet.put(queryId, queryDataSet);
return queryDataSet;
}
- protected QueryContext genQueryContext(long queryId) {
- return new QueryContext(queryId);
+ protected QueryContext genQueryContext(long queryId, boolean debug) {
+ return new QueryContext(queryId, debug);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index b3af734..dea1b60 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -94,7 +94,7 @@ public class FileLoaderUtils {
}
timeSeriesMetadata = TimeSeriesMetadataCache.getInstance()
.get(new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(resource.getTsFilePath(),
- seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors);
+ seriesPath.getDevice(), seriesPath.getMeasurement()), allSensors, context.isDebug());
if (timeSeriesMetadata != null) {
timeSeriesMetadata.setChunkMetadataLoader(
new DiskChunkMetadataLoader(resource, seriesPath, context, filter));