You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/11/05 10:32:49 UTC
[iotdb] 01/01: Add query metrics
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7ef20b3d1e5a499df27d7fb41a2c9a5f6b3e6c08
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Sat Nov 5 18:32:35 2022 +0800
Add query metrics
---
.../apache/iotdb/commons/conf/IoTDBConstant.java | 3 +
.../assembly/resources/conf/logback-datanode.xml | 20 ++++
.../iotdb/db/mpp/statistics/QueryStatistics.java | 118 +++++++++++++++++++++
3 files changed, 141 insertions(+)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index d5a26f4982..da571d8c68 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -77,6 +77,9 @@ public class IoTDBConstant {
public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER";
public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
+
+ public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS";
+
public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
public static final String IOTDB_JMX_PORT = "iotdb.jmx.port";
diff --git a/server/src/assembly/resources/conf/logback-datanode.xml b/server/src/assembly/resources/conf/logback-datanode.xml
index 4f7373e1c2..aeae186f94 100644
--- a/server/src/assembly/resources/conf/logback-datanode.xml
+++ b/server/src/assembly/resources/conf/logback-datanode.xml
@@ -265,6 +265,23 @@
<level>INFO</level>
</filter>
</appender>
+ <appender class="ch.qos.logback.core.rolling.RollingFileAppender" name="QUERY_STATISTICS">
+ <file>${IOTDB_HOME}/logs/log_datanode_query_statistics.log</file>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>${IOTDB_HOME}/logs/log-datanode-slow-sql-%d{yyyyMMdd}.%i.log.gz</fileNamePattern>
+ <maxFileSize>10MB</maxFileSize>
+ <maxHistory>168</maxHistory>
+ <totalSizeCap>512MB</totalSizeCap>
+ </rollingPolicy>
+ <append>true</append>
+ <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+ <pattern>%d [%t] %-5p %C{25}:%L - %m %n</pattern>
+ <charset>utf-8</charset>
+ </encoder>
+ <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
+ <level>INFO</level>
+ </filter>
+ </appender>
<root level="info">
<appender-ref ref="FILETRACE"/>
<appender-ref ref="FILEDEBUG"/>
@@ -299,4 +316,7 @@
<logger level="info" name="COMPACTION">
<appender-ref ref="COMPACTION"/>
</logger>
+ <logger level="info" name="QUERY_STATISTICS">
+ <appender-ref ref="QUERY_STATISTICS"/>
+ </logger>
</configuration>
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
new file mode 100644
index 0000000000..f6ba8b3050
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.mpp.statistics;
+
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+@ThreadSafe
+public class QueryStatistics {
+
+ private static final long QUERY_STATISTICS_PRINT_INTERVAL_IN_MS = 10_000;
+
+ private static final Logger QUERY_STATISTICS_LOGGER =
+ LoggerFactory.getLogger(IoTDBConstant.QUERY_STATISTICS_LOGGER_NAME);
+
+ private final AtomicBoolean tracing = new AtomicBoolean(false);
+
+ private final Map<String, OperationStatistic> operationStatistics = new ConcurrentHashMap<>();
+
+ private QueryStatistics() {
+ ScheduledExecutorService scheduledExecutor =
+ IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
+ ScheduledExecutorUtil.safelyScheduleAtFixedRate(
+ scheduledExecutor,
+ this::printQueryStatistics,
+ 0,
+ QUERY_STATISTICS_PRINT_INTERVAL_IN_MS,
+ TimeUnit.MILLISECONDS);
+ }
+
+ private void printQueryStatistics() {
+ if (tracing.get()) {
+ operationStatistics.forEach(
+ (k, v) -> {
+ QUERY_STATISTICS_LOGGER.info("Operation: {}, Statistics: {}", k, v);
+ });
+ }
+ }
+
+ public void addCost(String key, long costTimeInNanos) {
+ if (tracing.get()) {
+ operationStatistics
+ .computeIfAbsent(key, k -> new OperationStatistic())
+ .addTimeCost(costTimeInNanos);
+ }
+ }
+
+ public void traceOff() {
+ tracing.set(false);
+ operationStatistics.clear();
+ }
+
+ public void tradeOn() {
+ tracing.set(true);
+ operationStatistics.clear();
+ }
+
+ private static class OperationStatistic {
+ // accumulated operation time in ns
+ private final AtomicLong totalTime;
+ private final AtomicLong totalCount;
+
+ public OperationStatistic() {
+ this.totalTime = new AtomicLong(0);
+ this.totalCount = new AtomicLong(0);
+ }
+
+ public void addTimeCost(long costTimeInNanos) {
+ totalTime.addAndGet(costTimeInNanos);
+ totalCount.incrementAndGet();
+ }
+
+ @Override
+ public String toString() {
+ long time = totalTime.get() / 1_000;
+ long count = totalCount.get();
+ return "{"
+ + "totalTime="
+ + time
+ + "us"
+ + ", totalCount="
+ + count
+ + ", avgOperationTime="
+ + (time / count)
+ + "us"
+ + '}';
+ }
+ }
+}