You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2023/04/26 08:51:07 UTC
[iotdb] branch beyyes/rel_1.1_optimize_query created (now f27b7f8523)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/rel_1.1_optimize_query
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at f27b7f8523 [IOTDB-5803] Improve query performance by reducing cpu consuming
This branch includes the following new commits:
new f27b7f8523 [IOTDB-5803] Improve query performance by reducing cpu consuming
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-5803] Improve query performance by reducing cpu consuming
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/rel_1.1_optimize_query
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f27b7f85234a7bdda9e0daabc9672b71269459ac
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Wed Apr 26 16:42:13 2023 +0800
[IOTDB-5803] Improve query performance by reducing cpu consuming
---
.../org/apache/iotdb/db/mpp/common/QueryId.java | 2 +-
.../mpp/execution/schedule/DriverTaskThread.java | 7 +-
.../iotdb/db/mpp/metric/QueryMetricsManager.java | 140 +++++++++++++--------
.../db/mpp/plan/analyze/cache/PartitionCache.java | 10 +-
.../db/mpp/plan/parser/StatementGenerator.java | 20 ++-
.../iotdb/db/query/context/QueryContext.java | 2 +
.../query/control/clientsession/ClientSession.java | 3 +-
thrift/src/main/thrift/client.thrift | 3 +
8 files changed, 120 insertions(+), 67 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
index be9462696f..2208e00672 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/QueryId.java
@@ -52,7 +52,7 @@ public class QueryId {
}
public PlanNodeId genPlanNodeId() {
- return new PlanNodeId(String.format("%d", nextPlanNodeIndex++));
+ return new PlanNodeId(String.valueOf(nextPlanNodeIndex++));
}
public PlanFragmentId genPlanFragmentId() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
index 33028af21b..84e113fff4 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/schedule/DriverTaskThread.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.mpp.execution.driver.IDriver;
import org.apache.iotdb.db.mpp.execution.schedule.queue.IndexedBlockingQueue;
import org.apache.iotdb.db.mpp.execution.schedule.task.DriverTask;
import org.apache.iotdb.db.utils.SetThreadName;
-import org.apache.iotdb.db.utils.stats.CpuTimer;
import com.google.common.base.Ticker;
import com.google.common.util.concurrent.ListenableFuture;
@@ -65,9 +64,9 @@ public class DriverTaskThread extends AbstractDriverThread {
return;
}
IDriver driver = task.getDriver();
- CpuTimer timer = new CpuTimer();
+ // CpuTimer timer = new CpuTimer();
ListenableFuture<?> future = driver.processFor(EXECUTION_TIME_SLICE);
- CpuTimer.CpuDuration duration = timer.elapsedTime();
+ // CpuTimer.CpuDuration duration = timer.elapsedTime();
// If the future is cancelled, the task is in an error and should be thrown.
if (future.isCancelled()) {
task.setAbortCause(DriverTaskAbortedException.BY_ALREADY_BEING_CANCELLED);
@@ -76,7 +75,7 @@ public class DriverTaskThread extends AbstractDriverThread {
}
long quantaScheduledNanos = ticker.read() - startNanos;
ExecutionContext context = new ExecutionContext();
- context.setCpuDuration(duration);
+ // context.setCpuDuration(duration);
context.setScheduledTimeInNanos(quantaScheduledNanos);
context.setTimeSlice(EXECUTION_TIME_SLICE);
if (driver.isFinished()) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
index c6730d593c..adb6541e8c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java
@@ -22,93 +22,127 @@ package org.apache.iotdb.db.mpp.metric;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.enums.Metric;
import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.MetricInfo;
import org.apache.iotdb.metrics.utils.MetricLevel;
import java.util.concurrent.TimeUnit;
+import static org.apache.iotdb.metrics.utils.MetricLevel.DO_NOTHING;
+
public class QueryMetricsManager {
private final MetricService metricService = MetricService.getInstance();
public void recordPlanCost(String stage, long costTimeInNanos) {
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.QUERY_PLAN_COST.toString(),
- MetricLevel.IMPORTANT,
- Tag.STAGE.toString(),
- stage);
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.QUERY_PLAN_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.STAGE.toString(),
+ stage);
+ }
}
public void recordOperatorExecutionCost(String operatorType, long costTimeInNanos) {
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.OPERATOR_EXECUTION_COST.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- operatorType);
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.OPERATOR_EXECUTION_COST.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ operatorType);
+ }
}
public void recordOperatorExecutionCount(String operatorType, long count) {
- metricService.count(
- count,
- Metric.OPERATOR_EXECUTION_COUNT.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- operatorType);
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.count(
+ count,
+ Metric.OPERATOR_EXECUTION_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ operatorType);
+ }
}
public void recordSeriesScanCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = SeriesScanCostMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = SeriesScanCostMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordExecutionCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = QueryExecutionMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = QueryExecutionMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordQueryResourceNum(String type, int count) {
- metricService.histogram(
- count, Metric.QUERY_RESOURCE.toString(), MetricLevel.IMPORTANT, Tag.TYPE.toString(), type);
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.histogram(
+ count,
+ Metric.QUERY_RESOURCE.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.TYPE.toString(),
+ type);
+ }
}
public void recordDataExchangeCost(String stage, long costTimeInNanos) {
- MetricInfo metricInfo = DataExchangeCostMetricSet.metricInfoMap.get(stage);
- metricService.timer(
- costTimeInNanos,
- TimeUnit.NANOSECONDS,
- metricInfo.getName(),
- MetricLevel.IMPORTANT,
- metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = DataExchangeCostMetricSet.metricInfoMap.get(stage);
+ metricService.timer(
+ costTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ metricInfo.getName(),
+ MetricLevel.IMPORTANT,
+ metricInfo.getTagsInArray());
+ }
}
public void recordDataBlockNum(String type, int num) {
- MetricInfo metricInfo = DataExchangeCountMetricSet.metricInfoMap.get(type);
- metricService.histogram(
- num, metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ MetricInfo metricInfo = DataExchangeCountMetricSet.metricInfoMap.get(type);
+ metricService.histogram(
+ num, metricInfo.getName(), MetricLevel.IMPORTANT, metricInfo.getTagsInArray());
+ }
}
public void recordTaskQueueTime(String name, long queueTimeInNanos) {
- metricService.timer(
- queueTimeInNanos,
- TimeUnit.NANOSECONDS,
- Metric.DRIVER_SCHEDULER.toString(),
- MetricLevel.IMPORTANT,
- Tag.NAME.toString(),
- name);
+ if (!DO_NOTHING.equals(
+ MetricConfigDescriptor.getInstance().getMetricConfig().getMetricLevel())) {
+ metricService.timer(
+ queueTimeInNanos,
+ TimeUnit.NANOSECONDS,
+ Metric.DRIVER_SCHEDULER.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name);
+ }
}
public static QueryMetricsManager getInstance() {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
index 8449e4fc05..ed787adb33 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/cache/PartitionCache.java
@@ -673,10 +673,12 @@ public class PartitionCache {
SeriesPartitionTable cachedSeriesPartitionTable =
cachedStorageGroupPartitionMap.get(seriesPartitionSlot);
if (null == cachedSeriesPartitionTable) {
- logger.debug(
- "[{} Cache] miss when search device {}",
- DATA_PARTITION_CACHE_NAME,
- dataPartitionQueryParam.getDevicePath());
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "[{} Cache] miss when search device {}",
+ DATA_PARTITION_CACHE_NAME,
+ dataPartitionQueryParam.getDevicePath());
+ }
return false;
}
Map<TTimePartitionSlot, List<TConsensusGroupId>> cachedTimePartitionSlot =
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
index e5fa57f57a..f2d98b1204 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java
@@ -130,7 +130,12 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : rawDataQueryReq.getPaths()) {
- PartialPath path = new PartialPath(pathStr);
+ PartialPath path;
+ if (rawDataQueryReq.isLegalPathNodes()) {
+ path = new PartialPath(pathStr.split("\\."));
+ } else {
+ path = new PartialPath(pathStr);
+ }
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
@@ -168,7 +173,12 @@ public class StatementGenerator {
// iterate the path list and add it to from operator
for (String pathStr : lastDataQueryReq.getPaths()) {
- PartialPath path = new PartialPath(pathStr);
+ PartialPath path;
+ if (lastDataQueryReq.isLegalPathNodes()) {
+ path = new PartialPath(pathStr.split("\\."));
+ } else {
+ path = new PartialPath(pathStr);
+ }
fromComponent.addPrefixPath(path);
}
selectComponent.addResultColumn(
@@ -204,7 +214,11 @@ public class StatementGenerator {
SelectComponent selectComponent = new SelectComponent(zoneId);
List<PartialPath> selectPaths = new ArrayList<>();
for (String pathStr : req.getPaths()) {
- selectPaths.add(new PartialPath(pathStr));
+ if (req.isLegalPathNodes()) {
+ selectPaths.add(new PartialPath(pathStr.split("\\.")));
+ } else {
+ selectPaths.add(new PartialPath(pathStr));
+ }
}
List<TAggregationType> aggregations = req.getAggregations();
for (int i = 0; i < aggregations.size(); i++) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
index c3583f4070..2a44762fdd 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/context/QueryContext.java
@@ -93,6 +93,8 @@ public class QueryContext {
* them from 'modFile' and put then into the cache.
*/
public List<Modification> getPathModifications(ModificationFile modFile, PartialPath path) {
+ // TODO change a way to do the existing check to avoid this IO call each time.
+
// if the mods file does not exist, do not add it to the cache
if (!modFile.exists()) {
return Collections.emptyList();
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
index c52a27b535..49bb8e790f 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/clientsession/ClientSession.java
@@ -25,7 +25,6 @@ import java.net.Socket;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArraySet;
/** Client Session is the only identity for a connection. */
public class ClientSession extends IClientSession {
@@ -65,7 +64,7 @@ public class ClientSession extends IClientSession {
@Override
public void addStatementId(long statementId) {
- statementIdToQueryId.computeIfAbsent(statementId, sid -> new CopyOnWriteArraySet<>());
+ statementIdToQueryId.computeIfAbsent(statementId, sid -> ConcurrentHashMap.newKeySet());
}
@Override
diff --git a/thrift/src/main/thrift/client.thrift b/thrift/src/main/thrift/client.thrift
index e437139369..990e0e3358 100644
--- a/thrift/src/main/thrift/client.thrift
+++ b/thrift/src/main/thrift/client.thrift
@@ -323,6 +323,7 @@ struct TSRawDataQueryReq {
7: optional bool enableRedirectQuery
8: optional bool jdbcQuery
9: optional i64 timeout
+ 10: optional bool legalPathNodes
}
struct TSLastDataQueryReq {
@@ -334,6 +335,7 @@ struct TSLastDataQueryReq {
6: optional bool enableRedirectQuery
7: optional bool jdbcQuery
8: optional i64 timeout
+ 9: optional bool legalPathNodes
}
struct TSAggregationQueryReq {
@@ -347,6 +349,7 @@ struct TSAggregationQueryReq {
8: optional i64 slidingStep
9: optional i32 fetchSize
10: optional i64 timeout
+ 11: optional bool legalPathNodes
}
struct TSCreateMultiTimeseriesReq {