You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/04/23 06:56:15 UTC
[iotdb] 01/01: fix some query code impl to improve query performance
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/imporve_query
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5cdc4a4a96b0c13aea7d8f719dd0c186dd40d87b
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Sun Apr 23 14:55:57 2023 +0800
fix some query code impl to improve query performance
---
.../src/assembly/resources/conf/iotdb-common.properties | 4 ++++
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++
.../iotdb/db/mpp/execution/schedule/DriverTaskThread.java | 7 +++----
.../iotdb/db/mpp/plan/analyze/cache/PartitionCache.java | 10 ++++++----
.../apache/iotdb/db/mpp/plan/parser/StatementGenerator.java | 2 +-
.../java/org/apache/iotdb/db/query/context/QueryContext.java | 2 ++
.../iotdb/db/query/control/clientsession/ClientSession.java | 3 +--
8 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 038b1f894d..0b31292a7f 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -366,6 +366,10 @@ cluster_name=defaultCluster
# Datatype: boolean
# meta_data_cache_enable=true
+# Whether to enable query metrics monitoring or not. Enable this may cause a little query performance degradation.
+# Datatype: boolean
+# query_metric_enable = true
+
# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others.
# The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50
# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 4e39691ed0..8237040620 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -511,6 +511,9 @@ public class IoTDBConfig {
/** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */
private boolean metaDataCacheEnable = true;
+ /** whether to enable query metrics monitoring or not. */
+ private boolean queryMetricsEnable = true;
+
/** Memory allocated for bloomFilter cache in read process */
private long allocateMemoryForBloomFilterCache = allocateMemoryForRead / 1001;
@@ -2065,6 +2068,14 @@ public class IoTDBConfig {
this.metaDataCacheEnable = metaDataCacheEnable;
}
+ public boolean isQueryMetricsEnable() {
+ return queryMetricsEnable;
+ }
+
+ public void setQueryMetricsEnable(boolean queryMetricsEnable) {
+ this.queryMetricsEnable = queryMetricsEnable;
+ }
+
public long getAllocateMemoryForBloomFilterCache() {
return allocateMemoryForBloomFilterCache;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 5581bacde9..e54db728d0 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -338,6 +338,12 @@ public class IoTDBDescriptor {
"meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable()))
.trim()));
+ conf.setQueryMetricsEnable(
+ Boolean.parseBoolean(
+ properties
+ .getProperty("query_metric_enable", Boolean.toString(conf.isQueryMetricsEnable()))
+ .trim()));
+
initMemoryAllocate(properties);
loadWALProps(properties);
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
index 33028af21b..84e113fff4 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.mpp.execution.driver.IDriver;
import org.apache.iotdb.db.mpp.execution.schedule.queue.IndexedBlockingQueue;
import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTask;
import org.apache.iotdb.db.utils.SetThreadName;
-import org.apache.iotdb.db.utils.stats.CpuTimer;
import com.google.common.base.Ticker;
import com.google.common.util.concurrent.ListenableFuture;
@@ -65,9 +64,9 @@ public class DriverTaskThread extends AbstractDriverThread {
return;
}
IDriver driver = task.getDriver();
- CpuTimer timer = new CpuTimer();
+ // CpuTimer timer = new CpuTimer();
ListenableFuture<?> future = driver.processFor(EXECUTION_TIME_SLICE);
- CpuTimer.CpuDuration duration = timer.elapsedTime();
+ // CpuTimer.CpuDuration duration = timer.elapsedTime();
// If the future is cancelled, the task is in an error and should be thrown.
if (future.isCancelled()) {
task.setAbortCause(DriverTaskAbortedException.BY_ALREADY_BEING_CANCELLED);
@@ -76,7 +75,7 @@ public class DriverTaskThread extends AbstractDriverThread {
}
long quantaScheduledNanos = ticker.read() - startNanos;
ExecutionContext context = new ExecutionContext();
- context.setCpuDuration(duration);
+ // context.setCpuDuration(duration);
context.setScheduledTimeInNanos(quantaScheduledNanos);
context.setTimeSlice(EXECUTION_TIME_SLICE);
if (driver.isFinished()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
index 8449e4fc05..ed787adb33 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
@@ -673,10 +673,12 @@ public class PartitionCache {
SeriesPartitionTable cachedSeriesPartitionTable =
cachedStorageGroupPartitionMap.get(seriesPartitionSlot);
if (null == cachedSeriesPartitionTable) {
- logger.debug(
- "[{} Cache] miss when search device {}",
- DATA_PARTITION_CACHE_NAME,
- dataPartitionQueryParam.getDevicePath());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ DATA_PARTITION_CACHE_NAME,
+ dataPartitionQueryParam.getDevicePath());
+ }
return false;
}
Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot =
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index 432b76a718..12c330a300 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -208,7 +208,7 @@ public class StatementGenerator {
SelectComponent selectComponent = new SelectComponent(zoneId);
List<PartialPath> selectPaths = new ArrayList<>();
for (String pathStr : req.getPaths()) {
- selectPaths.add(new PartialPath(pathStr));
+ selectPaths.add(new PartialPath(pathStr.split("\\.")));
}
List<TAggregationType> aggregations = req.getAggregations();
for (int i = 0; i < aggregations.size(); i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index c3583f4070..33dc7af63a 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -93,6 +93,8 @@ public class QueryContext {
* them from 'modFile' and put then into the cache.
*/
public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
+ // TODO if no delete operation exists globally, return false
+
// if the mods file does not exist, do not add it to the cache
if (!modFile.exists()) {
return Collections.emptyList();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
index c52a27b535..49bb8e790f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
@@ -25,7 +25,6 @@ import java.net.Socket;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArraySet;
/** Client Session is the only identity for a connection. */
public class ClientSession extends IClientSession {
@@ -65,7 +64,7 @@ public class ClientSession extends IClientSession {
@Override
public void addStatementId(long statementId) {
- statementIdToQueryId.computeIfAbsent(statementId, sid -> new CopyOnWriteArraySet<>());
+ statementIdToQueryId.computeIfAbsent(statementId, sid -> ConcurrentHashMap.newKeySet());
}
@Override