You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/11/07 06:22:48 UTC
[iotdb] branch QueryMetrics0.13 updated (6038a25312 -> c1804bf399)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch QueryMetrics0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 6038a25312 [To rel/0.13] Fix String number infer bug (#7886)
new 1f87529758 add QueryStatistics
new 5cfe9b8bcf support TRACING ON/OFF
new e8375a03da add metrics (Parser/Planner)
new c1804bf399 add metrics (execution)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 +-
server/src/assembly/resources/conf/logback.xml | 20 +++
.../threadpool/ScheduledExecutorUtil.java | 196 +++++++++++++++++++++
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../main/java/org/apache/iotdb/db/qp/Planner.java | 14 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 19 +-
.../iotdb/db/qp/logical/sys/TracingOperator.java | 2 +-
.../iotdb/db/qp/physical/sys/TracingPlan.java | 2 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 7 +
.../iotdb/db/query/control/QueryStatistics.java | 138 +++++++++++++++
.../reader/series/SeriesRawDataBatchReader.java | 20 +++
.../iotdb/db/query/reader/series/SeriesReader.java | 7 +-
.../db/service/thrift/impl/TSServiceImpl.java | 4 +
13 files changed, 431 insertions(+), 6 deletions(-)
create mode 100644 server/src/main/java/org/apache/iotdb/db/concurrent/threadpool/ScheduledExecutorUtil.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
[iotdb] 03/04: add metrics (Parser/Planner)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e8375a03dac5464e3e4054604eeff3fc5fe84d03
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Nov 7 14:00:53 2022 +0800
add metrics (Parser/Planner)
---
server/src/main/java/org/apache/iotdb/db/qp/Planner.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
index 06bb083801..a06f3b8c46 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/Planner.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.db.qp.strategy.optimizer.ConcatPathOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.DnfFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.MergeSingleFilterOptimizer;
import org.apache.iotdb.db.qp.strategy.optimizer.RemoveNotOptimizer;
+import org.apache.iotdb.db.query.control.QueryStatistics;
import org.apache.iotdb.service.rpc.thrift.TSLastDataQueryReq;
import org.apache.iotdb.service.rpc.thrift.TSRawDataQueryReq;
@@ -59,8 +60,19 @@ public class Planner {
String sqlStr, ZoneId zoneId, IoTDBConstant.ClientVersion clientVersion)
throws QueryProcessException {
// from SQL to logical operator
+ long t1 = System.nanoTime();
Operator operator = LogicalGenerator.generate(sqlStr, zoneId, clientVersion);
- return generatePhysicalPlanFromOperator(operator, clientVersion);
+ if (operator.isQuery()) {
+ QueryStatistics.getInstance().addCost(QueryStatistics.PARSER, System.nanoTime() - t1);
+ }
+
+ long t2 = System.nanoTime();
+ PhysicalPlan physicalPlan = generatePhysicalPlanFromOperator(operator, clientVersion);
+ if (operator.isQuery()) {
+ QueryStatistics.getInstance().addCost(QueryStatistics.PLANNER, System.nanoTime() - t2);
+ }
+
+ return physicalPlan;
}
/** convert raw data query to physical plan directly */
[iotdb] 02/04: support TRACING ON/OFF
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5cfe9b8bcf4a34ebf44d93e393e16bfd07ea8032
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Nov 7 13:56:53 2022 +0800
support TRACING ON/OFF
---
.../antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 7 ++++++-
.../java/org/apache/iotdb/db/qp/executor/PlanExecutor.java | 13 +++++++++++++
.../org/apache/iotdb/db/qp/logical/sys/TracingOperator.java | 2 +-
.../org/apache/iotdb/db/qp/physical/sys/TracingPlan.java | 2 +-
.../java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 7 +++++++
5 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 7996345033..70e0920cde 100644
--- a/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -60,7 +60,7 @@ dclStatement
;
utilityStatement
- : merge | fullMerge | flush | clearCache | settle
+ : merge | fullMerge | flush | clearCache | settle | tracing
| setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource
| showQueryProcesslist | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding
| loadConfiguration | loadTimeseries | loadFile | removeFile | unloadFile;
@@ -643,6 +643,11 @@ settle
: SETTLE (prefixPath|tsFilePath=STRING_LITERAL)
;
+// Tracing on/off
+tracing
+ : TRACING (ON | OFF)
+ ;
+
// Set System To ReadOnly/Writable
setSystemStatus
: SET SYSTEM TO (READONLY|WRITABLE)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index a6989293c5..3d6104045e 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -132,11 +132,13 @@ import org.apache.iotdb.db.qp.physical.sys.ShowTTLPlan;
import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
import org.apache.iotdb.db.qp.physical.sys.StartTriggerPlan;
import org.apache.iotdb.db.qp.physical.sys.StopTriggerPlan;
+import org.apache.iotdb.db.qp.physical.sys.TracingPlan;
import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
import org.apache.iotdb.db.qp.utils.DateTimeUtils;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.control.QueryResourceManager;
+import org.apache.iotdb.db.query.control.QueryStatistics;
import org.apache.iotdb.db.query.control.QueryTimeManager;
import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet;
import org.apache.iotdb.db.query.dataset.ListDataSet;
@@ -427,12 +429,23 @@ public class PlanExecutor implements IPlanExecutor {
case PAUSE_ARCHIVING:
operatePauseArchiving((PauseArchivingPlan) plan);
return true;
+ case TRACING:
+ controlTracing((TracingPlan) plan);
+ return true;
default:
throw new UnsupportedOperationException(
String.format("operation %s is not supported", plan.getOperatorName()));
}
}
+ private void controlTracing(TracingPlan plan) {
+ if (plan.isTracingOn()) {
+ QueryStatistics.getInstance().enableTracing();
+ } else {
+ QueryStatistics.getInstance().disableTracing();
+ }
+ }
+
private boolean createTemplate(CreateTemplatePlan createTemplatePlan)
throws QueryProcessException {
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java
index fcc3f0f3c8..82e18f80a1 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/TracingOperator.java
@@ -26,7 +26,7 @@ import org.apache.iotdb.db.qp.strategy.PhysicalGenerator;
public class TracingOperator extends Operator {
- private boolean isTracingOn;
+ private final boolean isTracingOn;
public TracingOperator(int tokenIntType, boolean isTracingOn) {
super(tokenIntType);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java
index 0e0f9fc1fe..b51b13debe 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/TracingPlan.java
@@ -27,7 +27,7 @@ import java.util.List;
public class TracingPlan extends PhysicalPlan {
- private boolean isTracingOn;
+ private final boolean isTracingOn;
public TracingPlan(boolean isTracingOn) {
super(OperatorType.TRACING);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
index bae92be936..babfa1a8be 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java
@@ -2127,6 +2127,13 @@ public class IoTDBSqlVisitor extends IoTDBSqlParserBaseVisitor<Operator> {
return settleOperator;
}
+ // Tracing on/off
+
+ @Override
+ public Operator visitTracing(IoTDBSqlParser.TracingContext ctx) {
+ return new TracingOperator(SQLConstant.TOK_TRACING, ctx.ON() != null);
+ }
+
// Set System To ReadOnly/Writable
@Override
[iotdb] 01/04: add QueryStatistics
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1f87529758b2a02f88a89eeeb5cbe16774a7315d
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Nov 7 13:48:35 2022 +0800
add QueryStatistics
---
server/src/assembly/resources/conf/logback.xml | 20 +++
.../threadpool/ScheduledExecutorUtil.java | 196 +++++++++++++++++++++
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 +
.../iotdb/db/query/control/QueryStatistics.java | 130 ++++++++++++++
4 files changed, 347 insertions(+)
diff --git a/server/src/assembly/resources/conf/logback.xml b/server/src/assembly/resources/conf/logback.xml
index 7349c2b0b2..310053054d 100644
--- a/server/src/assembly/resources/conf/logback.xml
+++ b/server/src/assembly/resources/conf/logback.xml
@@ -265,6 +265,23 @@
<level>INFO</level>
</filter>
</appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="QUERY_STATISTICS">
+ <file>${IOTDB_LOG_DIR}/log_query_statistics.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_LOG_DIR}/log-query-statistics-%d{yyyyMMdd}.%i.log.gz</fileNamePattern>
+ <maxFileSize>10MB</maxFileSize>
+ <maxHistory>168</maxHistory>
+ <totalSizeCap>512MB</totalSizeCap>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
<root level="info">
<appender-ref ref="FILETRACE"/>
<appender-ref ref="FILEDEBUG"/>
@@ -299,4 +316,7 @@
<logger level="info" name="COMPACTION">
<appender-ref ref="COMPACTION"/>
</logger>
+ <logger level="info" name="QUERY_STATISTICS">
+ <appender-ref ref="QUERY_STATISTICS"/>
+ </logger>
</configuration>
diff --git a/server/src/main/java/org/apache/iotdb/db/concurrent/threadpool/ScheduledExecutorUtil.java b/server/src/main/java/org/apache/iotdb/db/concurrent/threadpool/ScheduledExecutorUtil.java
new file mode 100644
index 0000000000..950947331d
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/concurrent/threadpool/ScheduledExecutorUtil.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.concurrent.threadpool;
+
+import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+public class ScheduledExecutorUtil {
+
+ private static final Logger logger = LoggerFactory.getLogger(ScheduledExecutorUtil.class);
+
+ /**
+ * A safe wrapper method to make sure the exception thrown by the previous running will not affect
+ * the next one. Please reference the javadoc of {@link
+ * ScheduledExecutorService#scheduleAtFixedRate(Runnable, long, long, TimeUnit)} for more details.
+ *
+ * @param executor the ScheduledExecutorService instance.
+ * @param command same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @param initialDelay same parameter in {@link
+ * ScheduledExecutorService#scheduleAtFixedRate(Runnable, long, long, TimeUnit)}.
+ * @param period same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @param unit same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @return the same return value of {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ */
+ @SuppressWarnings("unsafeThreadSchedule")
+ public static ScheduledFuture<?> safelyScheduleAtFixedRate(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long period,
+ TimeUnit unit) {
+ return scheduleAtFixedRate(executor, command, initialDelay, period, unit, false);
+ }
+
+ /**
+ * A safe wrapper method to make sure the exception thrown by the previous running will not affect
+ * the next one. Please reference the javadoc of {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)} for more
+ * details.
+ *
+ * @param executor the ScheduledExecutorService instance.
+ * @param command same parameter in {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ * @param initialDelay same parameter in {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ * @param delay same parameter in {@link ScheduledExecutorService#scheduleWithFixedDelay(Runnable,
+ * long, long, TimeUnit)}.
+ * @param unit same parameter in {@link ScheduledExecutorService#scheduleWithFixedDelay(Runnable,
+ * long, long, TimeUnit)}.
+ * @return the same return value of {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ */
+ @SuppressWarnings("unsafeThreadSchedule")
+ public static ScheduledFuture<?> safelyScheduleWithFixedDelay(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long delay,
+ TimeUnit unit) {
+ return scheduleWithFixedDelay(executor, command, initialDelay, delay, unit, false);
+ }
+
+ /**
+ * A wrapper method to have the same semantic with {@link
+ * ScheduledExecutorService#scheduleAtFixedRate(Runnable, long, long, TimeUnit)}, except for
+ * logging an error log when any uncaught exception happens.
+ *
+ * @param executor the ScheduledExecutorService instance.
+ * @param command same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @param initialDelay same parameter in {@link
+ * ScheduledExecutorService#scheduleAtFixedRate(Runnable, long, long, TimeUnit)}.
+ * @param period same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @param unit same parameter in {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ * @return the same return value of {@link ScheduledExecutorService#scheduleAtFixedRate(Runnable,
+ * long, long, TimeUnit)}.
+ */
+ @SuppressWarnings("unsafeThreadSchedule")
+ public static ScheduledFuture<?> unsafelyScheduleAtFixedRate(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long period,
+ TimeUnit unit) {
+ return scheduleAtFixedRate(executor, command, initialDelay, period, unit, true);
+ }
+
+ /**
+ * A wrapper method to have the same semantic with {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}, except for
+ * logging an error log when any uncaught exception happens.
+ *
+ * @param executor the ScheduledExecutorService instance.
+ * @param command same parameter in {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ * @param initialDelay same parameter in {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ * @param delay same parameter in {@link ScheduledExecutorService#scheduleWithFixedDelay(Runnable,
+ * long, long, TimeUnit)}.
+ * @param unit same parameter in {@link ScheduledExecutorService#scheduleWithFixedDelay(Runnable,
+ * long, long, TimeUnit)}.
+ * @return the same return value of {@link
+ * ScheduledExecutorService#scheduleWithFixedDelay(Runnable, long, long, TimeUnit)}.
+ */
+ @SuppressWarnings("unsafeThreadSchedule")
+ public static ScheduledFuture<?> unsafelyScheduleWithFixedDelay(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long delay,
+ TimeUnit unit) {
+ return scheduleWithFixedDelay(executor, command, initialDelay, delay, unit, true);
+ }
+
+ @SuppressWarnings("unsafeThreadSchedule")
+ private static ScheduledFuture<?> scheduleAtFixedRate(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long period,
+ TimeUnit unit,
+ boolean unsafe) {
+ return executor.scheduleAtFixedRate(
+ () -> {
+ try {
+ command.run();
+ } catch (Throwable t) {
+ logger.error("Schedule task failed", t);
+ if (unsafe) {
+ throw t;
+ }
+ }
+ },
+ initialDelay,
+ period,
+ unit);
+ }
+
+ @SuppressWarnings("unsafeThreadSchedule")
+ private static ScheduledFuture<?> scheduleWithFixedDelay(
+ ScheduledExecutorService executor,
+ Runnable command,
+ long initialDelay,
+ long delay,
+ TimeUnit unit,
+ boolean unsafe) {
+ return executor.scheduleWithFixedDelay(
+ () -> {
+ try {
+ command.run();
+ } catch (Throwable t) {
+ logger.error("Schedule task failed", t);
+ if (unsafe) {
+ throw t;
+ }
+ }
+ },
+ initialDelay,
+ delay,
+ unit);
+ }
+
+ public static RuntimeException propagate(Throwable throwable) {
+ logger.error("Run thread failed", throwable);
+ Throwables.throwIfUnchecked(throwable);
+ throw new RuntimeException(throwable);
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 6d09cb4ac6..49059934f3 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -40,6 +40,7 @@ public class IoTDBConstant {
public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
public static final String DOUBLE_LIVE_LOGGER_NAME = "DOUBLE_LIVE";
+ public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS";
public static final String IOTDB_JMX_PORT = "iotdb.jmx.port";
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
new file mode 100644
index 0000000000..3ce09646f0
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.control;
+
+import org.apache.iotdb.db.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.db.concurrent.threadpool.ScheduledExecutorUtil;
+import org.apache.iotdb.db.conf.IoTDBConstant;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class QueryStatistics {
+
+ private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 10_000;
+
+ private static final Logger QUERY_STATISTICS_LOGGER =
+ LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME);
+
+ private final AtomicBoolean tracing = new AtomicBoolean(false);
+
+ private final Map<String, OperationStatistic> operationStatistics = new ConcurrentHashMap<>();
+
+ public static final String PARSER = "Parser";
+ public static final String PLANNER = "Planner";
+
+ private QueryStatistics() {
+ ScheduledExecutorService scheduledExecutor =
+ IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
+ ScheduledExecutorUtil.safelyScheduleAtFixedRate(
+ scheduledExecutor,
+ this::printQueryStatistics,
+ 0,
+ QUERY_STATISTICS_PRINT_INTERVAL_IN_MS,
+ TimeUnit.MILLISECONDS);
+ }
+
+ private void printQueryStatistics() {
+ if (tracing.get()) {
+ operationStatistics.forEach(
+ (k, v) -> {
+ QUERY_STATISTICS_LOGGER.info("Operation: {}, Statistics: {}", k, v);
+ });
+ }
+ }
+
+ public static QueryStatistics getInstance() {
+ return QueryStatisticsHolder.INSTANCE;
+ }
+
+ public void addCost(String key, long costTimeInNanos) {
+ if (tracing.get()) {
+ operationStatistics
+ .computeIfAbsent(key, k -> new OperationStatistic())
+ .addTimeCost(costTimeInNanos);
+ }
+ }
+
+ public void disableTracing() {
+ tracing.set(false);
+ operationStatistics.clear();
+ }
+
+ public void enableTracing() {
+ tracing.set(true);
+ operationStatistics.clear();
+ }
+
+ private static class OperationStatistic {
+ // accumulated operation time in ns
+ private final AtomicLong totalTime;
+ private final AtomicLong totalCount;
+
+ public OperationStatistic() {
+ this.totalTime = new AtomicLong(0);
+ this.totalCount = new AtomicLong(0);
+ }
+
+ public void addTimeCost(long costTimeInNanos) {
+ totalTime.addAndGet(costTimeInNanos);
+ totalCount.incrementAndGet();
+ }
+
+ @Override
+ public String toString() {
+ long time = totalTime.get() / 1_000;
+ long count = totalCount.get();
+ return "{"
+ + "totalTime="
+ + time
+ + "us"
+ + ", totalCount="
+ + count
+ + ", avgOperationTime="
+ + (time / count)
+ + "us"
+ + '}';
+ }
+ }
+
+ private static class QueryStatisticsHolder {
+
+ private static final QueryStatistics INSTANCE = new QueryStatistics();
+
+ private QueryStatisticsHolder() {}
+ }
+}
[iotdb] 04/04: add metrics (execution)
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c1804bf3997d876e3f29b1043d8f511765d7500d
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Mon Nov 7 14:16:51 2022 +0800
add metrics (execution)
---
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 6 +++++-
.../iotdb/db/query/control/QueryStatistics.java | 8 ++++++++
.../reader/series/SeriesRawDataBatchReader.java | 20 ++++++++++++++++++++
.../iotdb/db/query/reader/series/SeriesReader.java | 7 ++++++-
.../iotdb/db/service/thrift/impl/TSServiceImpl.java | 4 ++++
5 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 3d6104045e..b8a49f847f 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -274,8 +274,12 @@ public class PlanExecutor implements IPlanExecutor {
public QueryDataSet processQuery(PhysicalPlan queryPlan, QueryContext context)
throws IOException, StorageEngineException, QueryFilterOptimizationException,
QueryProcessException, MetadataException, InterruptedException {
+ long startTime = System.nanoTime();
if (queryPlan instanceof QueryPlan) {
- return processDataQuery((QueryPlan) queryPlan, context);
+ QueryDataSet queryDataSet = processDataQuery((QueryPlan) queryPlan, context);
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.INIT_QUERY_DATASET, System.nanoTime() - startTime);
+ return queryDataSet;
} else if (queryPlan instanceof AuthorPlan) {
return processAuthorQuery((AuthorPlan) queryPlan);
} else if (queryPlan instanceof ShowPlan) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
index 3ce09646f0..af40cc1bd4 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/control/QueryStatistics.java
@@ -47,6 +47,14 @@ public class QueryStatistics {
public static final String PARSER = "Parser";
public static final String PLANNER = "Planner";
+ public static final String INIT_QUERY_DATASET = "InitQueryDataset";
+ public static final String QUERY_EXECUTION = "QueryExecution";
+ public static final String SERIES_RAW_DATA_BATCH_READER_NEXT =
+ "SeriesRawDataBatchReader.nextBatch()";
+ public static final String SERIES_RAW_DATA_BATCH_READER_HAS_NEXT =
+ "SeriesRawDataBatchReader.hasNextBatch()";
+ public static final String PAGE_READER = "PageReader";
+
private QueryStatistics() {
ScheduledExecutorService scheduledExecutor =
IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
index ca3aab0074..693cee134b 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesRawDataBatchReader.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.querycontext.QueryDataSource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryStatistics;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -43,6 +44,8 @@ public class SeriesRawDataBatchReader implements ManagedSeriesReader {
private BatchData batchData;
private boolean hasCachedBatchData = false;
+ private final QueryStatistics QUERY_STATISTICS = QueryStatistics.getInstance();
+
public SeriesRawDataBatchReader(SeriesReader seriesReader) {
this.seriesReader = seriesReader;
}
@@ -99,8 +102,11 @@ public class SeriesRawDataBatchReader implements ManagedSeriesReader {
*/
@Override
public boolean hasNextBatch() throws IOException {
+ long startTime = System.nanoTime();
if (hasCachedBatchData) {
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_HAS_NEXT, System.nanoTime() - startTime);
return true;
}
@@ -109,6 +115,8 @@ public class SeriesRawDataBatchReader implements ManagedSeriesReader {
*/
if (readPageData()) {
hasCachedBatchData = true;
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_HAS_NEXT, System.nanoTime() - startTime);
return true;
}
@@ -117,6 +125,8 @@ public class SeriesRawDataBatchReader implements ManagedSeriesReader {
*/
if (readChunkData()) {
hasCachedBatchData = true;
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_HAS_NEXT, System.nanoTime() - startTime);
return true;
}
@@ -126,16 +136,26 @@ public class SeriesRawDataBatchReader implements ManagedSeriesReader {
while (seriesReader.hasNextFile()) {
if (readChunkData()) {
hasCachedBatchData = true;
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_HAS_NEXT, System.nanoTime() - startTime);
return true;
}
}
+
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_HAS_NEXT, System.nanoTime() - startTime);
return hasCachedBatchData;
}
@Override
public BatchData nextBatch() throws IOException {
+ long startTime = System.nanoTime();
+
if (hasCachedBatchData || hasNextBatch()) {
hasCachedBatchData = false;
+
+ QUERY_STATISTICS.addCost(
+ QueryStatistics.SERIES_RAW_DATA_BATCH_READER_NEXT, System.nanoTime() - startTime);
return batchData;
}
throw new IOException("no next batch");
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index ab5fbb742a..f4d1ae15c6 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.metadata.idtable.IDTable;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryStatistics;
import org.apache.iotdb.db.query.control.QueryTimeManager;
import org.apache.iotdb.db.query.control.tracing.TracingManager;
import org.apache.iotdb.db.query.filter.TsFileFilter;
@@ -1172,7 +1173,11 @@ public class SeriesReader {
}
BatchData getAllSatisfiedPageData(boolean ascending) throws IOException {
- return data.getAllSatisfiedPageData(ascending);
+ long startTime = System.nanoTime();
+ BatchData batchData = data.getAllSatisfiedPageData(ascending);
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.PAGE_READER, System.nanoTime() - startTime);
+ return batchData;
}
void setFilter(Filter filter) {
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
index 3f98f7a06d..e0b99aae43 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/TSServiceImpl.java
@@ -65,6 +65,7 @@ import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
import org.apache.iotdb.db.qp.physical.sys.ShowQueryProcesslistPlan;
import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
import org.apache.iotdb.db.query.context.QueryContext;
+import org.apache.iotdb.db.query.control.QueryStatistics;
import org.apache.iotdb.db.query.control.clientsession.IClientSession;
import org.apache.iotdb.db.query.control.tracing.TracingConstant;
import org.apache.iotdb.db.query.dataset.DirectAlignByTimeDataSet;
@@ -854,7 +855,10 @@ public class TSServiceImpl implements TSIService.Iface {
resp.setNonAlignQueryDataSet(fillRpcNonAlignReturnData(fetchSize, newDataSet, username));
} else {
try {
+ long startTime = System.nanoTime();
TSQueryDataSet tsQueryDataSet = fillRpcReturnData(fetchSize, newDataSet, username);
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.QUERY_EXECUTION, System.nanoTime() - startTime);
resp.setQueryDataSet(tsQueryDataSet);
} catch (RedirectException e) {
if (plan.isEnableRedirect()) {