You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/11/05 10:32:48 UTC
[iotdb] branch QueryMetrics created (now 7ef20b3d1e)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 7ef20b3d1e Add query metrics
This branch includes the following new commits:
new 7ef20b3d1e Add query metrics
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Add query metrics
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7ef20b3d1e5a499df27d7fb41a2c9a5f6b3e6c08
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Sat Nov 5 18:32:35 2022 +0800
Add query metrics
---
.../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +
.../assembly/resources/conf/logback-datanode.xml | 20 ++++
.../iotdb/db/mpp/statistics/QueryStatistics.java | 118 +++++++++++++++++++++
3 files changed, 141 insertions(+)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index d5a26f4982..da571d8c68 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -77,6 +77,9 @@ public class IoTDBConstant {
public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER";
public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
+
+ public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS";
+
public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
public static final String IOTDB_JMX_PORT = "iotdb.jmx.port";
diff --git a/server/src/assembly/resources/conf/logback-datanode.xml b/server/src/assembly/resources/conf/logback-datanode.xml
index 4f7373e1c2..aeae186f94 100644
--- a/server/src/assembly/resources/conf/logback-datanode.xml
+++ b/server/src/assembly/resources/conf/logback-datanode.xml
@@ -265,6 +265,23 @@
<level>INFO</level>
</filter>
</appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="QUERY_STATISTICS">
+ <file>${IOTDB_HOME}/logs/log_datanode_query_statistics.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_HOME}/logs/log-datanode-slow-sql-%d{yyyyMMdd}.%i.log.gz</fileNamePattern>
+ <maxFileSize>10MB</maxFileSize>
+ <maxHistory>168</maxHistory>
+ <totalSizeCap>512MB</totalSizeCap>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
<root level="info">
<appender-ref ref="FILETRACE"/>
<appender-ref ref="FILEDEBUG"/>
@@ -299,4 +316,7 @@
<logger level="info" name="COMPACTION">
<appender-ref ref="COMPACTION"/>
</logger>
+ <logger level="info" name="QUERY_STATISTICS">
+ <appender-ref ref="QUERY_STATISTICS"/>
+ </logger>
</configuration>
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
new file mode 100644
index 0000000000..f6ba8b3050
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.mpp.statistics;
+
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@ThreadSafe
+public class QueryStatistics {
+
+ private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 10_000;
+
+ private static final Logger QUERY_STATISTICS_LOGGER =
+ LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME);
+
+ private final AtomicBoolean tracing = new AtomicBoolean(false);
+
+ private final Map<String, OperationStatistic> operationStatistics = new ConcurrentHashMap<>();
+
+ private QueryStatistics() {
+ ScheduledExecutorService scheduledExecutor =
+ IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
+ ScheduledExecutorUtil.safelyScheduleAtFixedRate(
+ scheduledExecutor,
+ this::printQueryStatistics,
+ 0,
+ QUERY_STATISTICS_PRINT_INTERVAL_IN_MS,
+ TimeUnit.MILLISECONDS);
+ }
+
+ private void printQueryStatistics() {
+ if (tracing.get()) {
+ operationStatistics.forEach(
+ (k, v) -> {
+ QUERY_STATISTICS_LOGGER.info("Operation: {}, Statistics: {}", k, v);
+ });
+ }
+ }
+
+ public void addCost(String key, long costTimeInNanos) {
+ if (tracing.get()) {
+ operationStatistics
+ .computeIfAbsent(key, k -> new OperationStatistic())
+ .addTimeCost(costTimeInNanos);
+ }
+ }
+
+ public void traceOff() {
+ tracing.set(false);
+ operationStatistics.clear();
+ }
+
+ public void tradeOn() {
+ tracing.set(true);
+ operationStatistics.clear();
+ }
+
+ private static class OperationStatistic {
+ // accumulated operation time in ns
+ private final AtomicLong totalTime;
+ private final AtomicLong totalCount;
+
+ public OperationStatistic() {
+ this.totalTime = new AtomicLong(0);
+ this.totalCount = new AtomicLong(0);
+ }
+
+ public void addTimeCost(long costTimeInNanos) {
+ totalTime.addAndGet(costTimeInNanos);
+ totalCount.incrementAndGet();
+ }
+
+ @Override
+ public String toString() {
+ long time = totalTime.get() / 1_000;
+ long count = totalCount.get();
+ return "{"
+ + "totalTime="
+ + time
+ + "us"
+ + ", totalCount="
+ + count
+ + ", avgOperationTime="
+ + (time / count)
+ + "us"
+ + '}';
+ }
+ }
+}