You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2019/09/26 12:42:00 UTC
[incubator-iotdb] branch cache_hit_display created (now 99848b5)
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a change to branch cache_hit_display
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 99848b5 display cache hit rate through jconsole
This branch includes the following new commits:
new 99848b5 display cache hit rate through jconsole
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: display cache hit rate through jconsole
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
suyue pushed a commit to branch cache_hit_display
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 99848b515bbbbbae1e4473f99dd14e62e8e2ad23
Author: suyue <23...@qq.com>
AuthorDate: Thu Sep 26 20:40:55 2019 +0800
display cache hit rate through jconsole
---
.../4-Performance Monitor.md | 9 +++
.../4-Performance Monitor.md | 11 +++
server/src/assembly/resources/conf/logback.xml | 2 +-
.../iotdb/db/engine/cache/CacheHitRateMonitor.java | 86 ++++++++++++++++++++++
.../db/engine/cache/CacheHitRateMonitorMXBean.java | 26 +++++++
.../iotdb/db/engine/cache/DeviceMetaDataCache.java | 41 ++++++++---
.../iotdb/db/engine/cache/TsFileMetaDataCache.java | 37 +++++++---
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +
.../org/apache/iotdb/db/service/ServiceType.java | 1 +
9 files changed, 191 insertions(+), 24 deletions(-)
diff --git a/docs/Documentation-CHN/UserGuide/4-Deployment and Management/4-Performance Monitor.md b/docs/Documentation-CHN/UserGuide/4-Deployment and Management/4-Performance Monitor.md
index 880564c..7c6a9d1 100644
--- a/docs/Documentation-CHN/UserGuide/4-Deployment and Management/4-Performance Monitor.md
+++ b/docs/Documentation-CHN/UserGuide/4-Deployment and Management/4-Performance Monitor.md
@@ -76,3 +76,12 @@
在监控结束区域增加记录代码:
Measurement.INSTANCE.addOperationLatency(Operation, t0);
+
+## cache命中率统计
+### 概述
+
+为了提高查询性能，IOTDB对ChunkMetaData和TsFileMetaData进行了缓存。用户可以通过debug级别的日志以及MXBean两种方式来查看缓存的命中率，并根据缓存命中率以及系统内存来调节缓存所使用的内存大小。使用MXBean查看缓存命中率的方法为：
+1. 通过端口31999连接jconsole，并在上方菜单项中选择‘MBean’.
+2. 展开侧边框并选择 'org.apache.iotdb.db.service'. 将会得到如下图所示结果：
+
+<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://user-images.githubusercontent.com/13203019/65687623-404fc380-e09c-11e9-83c3-3c7c63a5b0be.jpeg">
\ No newline at end of file
diff --git a/docs/Documentation/UserGuide/4-Deployment and Management/4-Performance Monitor.md b/docs/Documentation/UserGuide/4-Deployment and Management/4-Performance Monitor.md
index 7128f05..1f22d15 100644
--- a/docs/Documentation/UserGuide/4-Deployment and Management/4-Performance Monitor.md
+++ b/docs/Documentation/UserGuide/4-Deployment and Management/4-Performance Monitor.md
@@ -77,3 +77,14 @@ Add timing code in the monitoring start area:
Add timing code in the monitoring stop area:
Measurement.INSTANCE.addOperationLatency(Operation, t0);
+
+
+## Cache Hit Rate Statistics
+### Overview
+
+To improve query performance, IOTDB caches ChunkMetaData and TsFileMetaData. Users can view the cache hit rate through debug level log and MXBean, and adjust the memory occupied by the cache according to the cache hit rate and system memory. The method of using MXBean to view cache hit ratio is as follows:
+1. Connect to jconsole with port 31999 and select 'MBean' in the menu item above.
+2. Expand the sidebar and select 'org.apache.iotdb.db.service'. You will get the results shown in the following figure:
+
+<img style="width:100%; max-width:800px; max-height:600px; margin-left:auto; margin-right:auto; display:block;" src="https://user-images.githubusercontent.com/13203019/65687623-404fc380-e09c-11e9-83c3-3c7c63a5b0be.jpeg">
+
diff --git a/server/src/assembly/resources/conf/logback.xml b/server/src/assembly/resources/conf/logback.xml
index ff7c459..4104b85 100644
--- a/server/src/assembly/resources/conf/logback.xml
+++ b/server/src/assembly/resources/conf/logback.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<configuration>
+<configuration scan="true" scanPeriod="100 seconds">
<jmxConfigurator/>
<!-- prevent logback from outputting its own status at the start of every log -->
<statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitor.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitor.java
new file mode 100644
index 0000000..c108f5f
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitor.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.cache;
+
+import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.exception.StartupException;
+import org.apache.iotdb.db.service.IService;
+import org.apache.iotdb.db.service.JMXService;
+import org.apache.iotdb.db.service.ServiceType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CacheHitRateMonitor implements CacheHitRateMonitorMXBean, IService {
+
+ double chunkMetaDataHitRate;
+ double tsfileMetaDataHitRate;
+
+ private static Logger logger = LoggerFactory.getLogger(CacheHitRateMonitor.class);
+ static final CacheHitRateMonitor instance = AsyncCacheHitRateHolder.DISPLAYER;
+ private final String mbeanName = String
+ .format("%s:%s=%s", IoTDBConstant.IOTDB_PACKAGE, IoTDBConstant.JMX_TYPE,
+ getID().getJmxName());
+
+ @Override
+ public void start() throws StartupException {
+ try {
+ JMXService.registerMBean(instance, mbeanName);
+ } catch (Exception e) {
+ String errorMessage = String
+ .format("Failed to start %s because of %s", this.getID().getName(),
+ e.getMessage());
+ throw new StartupException(errorMessage, e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ JMXService.deregisterMBean(mbeanName);
+ logger.info("{}: stop {}...", IoTDBConstant.GLOBAL_DB_NAME, this.getID().getName());
+ }
+
+ @Override
+ public ServiceType getID() {
+ return ServiceType.CACHE_HIT_RATE_DISPLAY_SERVICE;
+ }
+
+ @Override
+ public double getChunkMetaDataHitRate() {
+ chunkMetaDataHitRate = DeviceMetaDataCache.getInstance().calculateChunkMetaDataHitRate();
+ return chunkMetaDataHitRate;
+ }
+
+ @Override
+ public double getTsfileMetaDataHitRate() {
+ tsfileMetaDataHitRate = TsFileMetaDataCache.getInstance().calculateTsfileMetaDataHitRate();
+ return tsfileMetaDataHitRate;
+ }
+
+ public static CacheHitRateMonitor getInstance() {
+ return instance;
+ }
+
+ private static class AsyncCacheHitRateHolder {
+
+ private static final CacheHitRateMonitor DISPLAYER = new CacheHitRateMonitor();
+
+ private AsyncCacheHitRateHolder() {
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitorMXBean.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitorMXBean.java
new file mode 100644
index 0000000..7e7b536
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/CacheHitRateMonitorMXBean.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.cache;
+
+public interface CacheHitRateMonitorMXBean {
+
+ double getChunkMetaDataHitRate();
+
+ double getTsfileMetaDataHitRate();
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
index 334f630..2f1ad91 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/DeviceMetaDataCache.java
@@ -99,8 +99,9 @@ public class DeviceMetaDataCache {
return TsFileMetadataUtils.getChunkMetaDataList(seriesPath.getMeasurement(), deviceMetaData);
}
- StringBuilder builder = new StringBuilder(resource.getFile().getPath()).append(".").append(seriesPath
- .getDevice());
+ StringBuilder builder = new StringBuilder(resource.getFile().getPath()).append(".")
+ .append(seriesPath
+ .getDevice());
String pathDeviceStr = builder.toString();
String key = builder.append(".").append(seriesPath.getMeasurement()).toString();
Object devicePathObject = pathDeviceStr.intern();
@@ -109,26 +110,19 @@ public class DeviceMetaDataCache {
cacheRequestNum.incrementAndGet();
if (lruCache.containsKey(key)) {
cacheHitNum.incrementAndGet();
- if (logger.isDebugEnabled()) {
- logger.debug(
- "Cache hit: the number of requests for cache is {}, "
- + "the number of hints for cache is {}",
- cacheRequestNum.get(), cacheHitNum.get());
- }
+ printCacheLog(true);
return new ArrayList<>(lruCache.get(key));
}
}
synchronized (devicePathObject) {
synchronized (lruCache) {
if (lruCache.containsKey(key)) {
+ printCacheLog(true);
cacheHitNum.incrementAndGet();
return new ArrayList<>(lruCache.get(key));
}
}
- if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hit: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
+ printCacheLog(false);
TsFileMetaData fileMetaData = TsFileMetaDataCache.getInstance().get(resource);
TsDeviceMetadata deviceMetaData = TsFileMetadataUtils
.getTsDeviceMetaData(resource, seriesPath, fileMetaData);
@@ -153,6 +147,29 @@ public class DeviceMetaDataCache {
}
}
+ private void printCacheLog(boolean isHit) {
+ if (!logger.isDebugEnabled()) {
+ return;
+ }
+ if (isHit) {
+ logger.debug(
+ "[ChunkMetaData cache hit] The number of requests for cache is {}, hit rate is {}.",
+ cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ } else {
+ logger.debug(
+ "[ChunkMetaData cache didn't hit] The number of requests for cache is {}, hit rate is {}.",
+ cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ }
+ }
+
+ public double calculateChunkMetaDataHitRate() {
+ if (cacheRequestNum.get() != 0) {
+ return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
+ } else {
+ return 0;
+ }
+ }
+
/**
* calculate the most frequently query measurements set.
*
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
index 1721ff0..426bdf0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TsFileMetaDataCache.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
-import org.apache.iotdb.tsfile.read.common.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,12 +100,7 @@ public class TsFileMetaDataCache {
synchronized (cache) {
if (cache.containsKey(path)) {
cacheHitNum.incrementAndGet();
- if (logger.isDebugEnabled()) {
- logger.debug(
- "Cache hit: the number of requests for cache is {}, "
- + "the number of hints for cache is {}",
- cacheRequestNum.get(), cacheHitNum.get());
- }
+ printCacheLog(true);
return cache.get(path);
}
}
@@ -114,13 +108,11 @@ public class TsFileMetaDataCache {
synchronized (cache) {
if (cache.containsKey(path)) {
cacheHitNum.incrementAndGet();
+ printCacheLog(true);
return cache.get(path);
}
}
- if (logger.isDebugEnabled()) {
- logger.debug("Cache didn't hit: the number of requests for cache is {}",
- cacheRequestNum.get());
- }
+ printCacheLog(false);
TsFileMetaData fileMetaData = TsFileMetadataUtils.getTsFileMetaData(tsFileResource);
synchronized (cache) {
cache.put(tsFileResource, fileMetaData);
@@ -129,6 +121,29 @@ public class TsFileMetaDataCache {
}
}
+ private void printCacheLog(boolean isHit) {
+ if (!logger.isDebugEnabled()) {
+ return;
+ }
+ if (isHit) {
+ logger.debug(
+ "[TsFileMetaData cache hit] The number of requests for cache is {}, hit rate is {}.",
+ cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ } else {
+ logger.debug(
+ "[TsFileMetaData cache didn't hit] The number of requests for cache is {}, hit rate is {}.",
+ cacheRequestNum.get(), cacheHitNum.get() * 1.0 / cacheRequestNum.get());
+ }
+ }
+
+ public double calculateTsfileMetaDataHitRate() {
+ if (cacheRequestNum.get() != 0) {
+ return cacheHitNum.get() * 1.0 / cacheRequestNum.get();
+ } else {
+ return 0;
+ }
+ }
+
public void remove(TsFileResource resource) {
synchronized (cache) {
if (cache != null) {
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 8264fad..c78dde6 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.conf.adapter.ManageDynamicParameters;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
import org.apache.iotdb.db.cost.statistic.Measurement;
import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.CacheHitRateMonitor;
import org.apache.iotdb.db.engine.merge.manage.MergeManager;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.exception.StartupException;
@@ -101,6 +102,7 @@ public class IoTDB implements IoTDBMBean {
registerManager.register(TVListAllocator.getInstance());
registerManager.register(FlushManager.getInstance());
registerManager.register(MergeManager.getINSTANCE());
+ registerManager.register(CacheHitRateMonitor.getInstance());
JMXService.registerMBean(getInstance(), mbeanName);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
index 70b8b49..511f997 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/ServiceType.java
@@ -35,6 +35,7 @@ public enum ServiceType {
PERFORMANCE_STATISTIC_SERVICE("PERFORMANCE_STATISTIC_SERVICE","PERFORMANCE_STATISTIC_SERVICE"),
MANAGE_DYNAMIC_PARAMETERS_SERVICE("Manage Dynamic Parameters", "Manage Dynamic Parameters"),
TVLIST_ALLOCATOR_SERVICE("TVList Allocator", ""),
+ CACHE_HIT_RATE_DISPLAY_SERVICE("CACHE_HIT_RATE_DISPLAY_SERVICE","Cache Hit Rate"),
FLUSH_SERVICE("Flush ServerService",
generateJmxName("org.apache.iotdb.db.engine.pool", "Flush Manager"));