You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2022/12/20 12:52:09 UTC
[iotdb] branch rel/0.13 updated: [To rel/0.13][IOTDB-5232] Add schema memory metric (#8540)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new c3e2782295 [To rel/0.13][IOTDB-5232] Add schema memory metric (#8540)
c3e2782295 is described below
commit c3e278229519b9f8aa75f031480478552737ebdf
Author: Marcos_Zyk <38...@users.noreply.github.com>
AuthorDate: Tue Dec 20 20:52:02 2022 +0800
[To rel/0.13][IOTDB-5232] Add schema memory metric (#8540)
[To rel/0.13][IOTDB-5232] Add schema memory metric (#8540)
---
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 14 +++---
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 14 +++---
.../org/apache/iotdb/db/metadata/MManager.java | 21 +++++++-
.../apache/iotdb/db/metadata/MManagerMetrics.java | 57 ++++++++++++++++++++--
4 files changed, 87 insertions(+), 19 deletions(-)
diff --git a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
index 57d44bbad2..8699be63e0 100644
--- a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -100,9 +100,9 @@ Next, we will choose Prometheus format data as samples to describe each kind of
#### 1.3.3.3. Memory Usage
-| Metric | Tag | level | Description | Sample |
-| ------ | --------------------------------------- | --------- | --------------------------------------------------------------------- | --------------------------------- |
-| mem | name="chunkMetaData/storageGroup/mtree" | important | Current memory size of chunkMetaData/storageGroup/mtree data in bytes | mem{name="chunkMetaData",} 2050.0 |
+| Metric | Tag | level | Description | Sample |
+| ------ |---------------------------------------------------------------| --------- |-------------------------------------------------------------------------------------------------| --------------------------------- |
+| mem | name="chunkMetaData/storageGroup/schemaUsage/schemaRemaining" | important | Current memory size of chunkMetaData/storageGroup/schema usage/schema remaining memory in bytes | mem{name="chunkMetaData",} 2050.0 |
#### 1.3.3.4. Cache
@@ -112,10 +112,10 @@ Next, we will choose Prometheus format data as samples to describe each kind of
#### 1.3.3.5. Business Data
-| Metric | Tag | level | Description | Sample |
-|----------|---------------------------------------------------------------------|-----------|---------------------------------------------------------------|-----------------------------------------------|
-| quantity | name="timeSeries/storageGroup/device", type="total/normal/template" | important | The current count of timeSeries/storageGroup/devices in IoTDB | quantity{name="timeSeries",type="normal"} 1.0 |
-| points | sg="{{storageGroup}}", type="flush" | core | The point number of last flushed memtable | quantity{name="memtable",type="flush"} 1.0 |
+| Metric | Tag | level | Description | Sample |
+|----------|--------------------------------------------------------------------------------------------------|-----------|--------------------------------------------------------------------------------------|-----------------------------------------------|
+| quantity | name="timeSeries/storageGroup/device/deviceUsingTemplate", type="total/normal/template/template" | important | The current count of timeSeries/storageGroup/devices/devices using template in IoTDB | quantity{name="timeSeries",type="normal"} 1.0 |
+| points | sg="{{storageGroup}}", type="flush" | core | The point number of last flushed memtable | quantity{name="memtable",type="flush"} 1.0 |
#### 1.3.3.6. Cluster
diff --git a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
index 44560b68e8..3eaaffc1b2 100644
--- a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -99,9 +99,9 @@ IoTDB对外提供JMX和Prometheus格式的监控指标，对于JMX，可以通
#### 1.3.3.3. 内存占用
-| Metric | Tag | level | 说明 | 示例 |
-| ------ | --------------------------------------- | --------- | -------------------------------------------------- | --------------------------------- |
-| mem | name="chunkMetaData/storageGroup/mtree" | important | chunkMetaData/storageGroup/mtree占用的内存（byte） | mem{name="chunkMetaData",} 2050.0 |
+| Metric | Tag | level | 说明 | 示例 |
+| ------ |---------------------------------------------------------------| --------- |---------------------------------------------------------------| --------------------------------- |
+| mem | name="chunkMetaData/storageGroup/schemaUsage/schemaRemaining" | important | chunkMetaData 占用/storageGroup 占用/schema 占用/schema 剩余的内存（byte） | mem{name="chunkMetaData",} 2050.0 |
#### 1.3.3.4. 缓存
@@ -111,10 +111,10 @@ IoTDB对外提供JMX和Prometheus格式的监控指标，对于JMX，可以通
#### 1.3.3.5. 业务数据
-| Metric | Tag | level | 说明 | 示例 |
-|----------|---------------------------------------------------------------------| --------- |---------------------------------------| --------------------------------------------- |
-| quantity | name="timeSeries/storageGroup/device", type="total/normal/template" | important | 当前时间timeSeries/storageGroup/device的数量 | quantity{name="timeSeries",type="normal"} 1.0 |
-| points | sg="{{storageGroup}}", type="flush" | core | 最新一个刷盘的memtale的点数 | quantity{name="memtable",type="flush"} 1.0 |
+| Metric | Tag | level | 说明 | 示例 |
+|----------|-------------------------------------------------------------------------------------------| --------- |----------------------------------------------------| --------------------------------------------- |
+| quantity | name="timeSeries/storageGroup/device/deviceUsingTemplate", type="total/normal/template/template" | important | 当前时间timeSeries/storageGroup/device/激活了模板的device的数量 | quantity{name="timeSeries",type="normal"} 1.0 |
+| points | sg="{{storageGroup}}", type="flush" | core | 最新一个刷盘的memtale的点数 | quantity{name="memtable",type="flush"} 1.0 |
#### 1.3.3.6. 集群
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8c05c0f8dd..a4f9dadbb5 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -190,6 +190,7 @@ public class MManager {
private final AtomicLong totalNormalSeriesNumber = new AtomicLong();
private final AtomicLong totalTemplateSeriesNumber = new AtomicLong();
+ private final AtomicLong totalTemplateActivatedNumber = new AtomicLong();
private final int mtreeSnapshotInterval;
private final long mtreeSnapshotThresholdTime;
@@ -443,6 +444,7 @@ public class MManager {
}
this.totalNormalSeriesNumber.set(0);
this.totalTemplateSeriesNumber.set(0);
+ this.totalTemplateActivatedNumber.set(0);
this.templateManager.clear();
if (logWriter != null) {
logWriter.close();
@@ -559,7 +561,9 @@ public class MManager {
if (!allowToCreateNewSeries) {
throw new MetadataException(
"IoTDB system load is too large to create timeseries, "
- + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
+ + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart."
+ + "Current timeseries num is "
+ + totalNormalSeriesNumber.get());
}
if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) {
@@ -685,7 +689,9 @@ public class MManager {
if (!allowToCreateNewSeries) {
throw new MetadataException(
"IoTDB system load is too large to create timeseries, "
- + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
+ + "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart."
+ + "Current timeseries num is "
+ + totalNormalSeriesNumber.get());
}
int seriesCount = plan.getMeasurements().size();
@@ -1042,6 +1048,15 @@ public class MManager {
return totalTemplateSeriesNumber.get();
}
+ public long getTotalTemplateActivatedNumber() {
+ return totalTemplateActivatedNumber.get();
+ }
+
+ public long getTotalEstimatedMemoryUsage() {
+ return (totalNormalSeriesNumber.get() + totalTemplateActivatedNumber.get())
+ * config.getEstimatedSeriesSize();
+ }
+
/**
* To calculate the count of timeseries matching given path. The path could be a pattern of a full
* path, may contain wildcard. If using prefix match, the path pattern is used to match prefix
@@ -2587,6 +2602,7 @@ public class MManager {
node.setUseTemplate(false);
int seriesCount = node.getUpperTemplate().getMeasurementsCount();
+ totalTemplateActivatedNumber.addAndGet(-1);
totalTemplateSeriesNumber.addAndGet(-seriesCount);
if (seriesNumerMonitor != null) {
seriesNumerMonitor.deleteTimeSeries(seriesCount);
@@ -2649,6 +2665,7 @@ public class MManager {
throw t;
}
+ totalTemplateActivatedNumber.addAndGet(1);
totalTemplateSeriesNumber.addAndGet(template.getMeasurementsCount());
if (node != mountedMNode) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManagerMetrics.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManagerMetrics.java
index 999796df95..b8f3df8f51 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManagerMetrics.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManagerMetrics.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.metadata;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.MetricService;
import org.apache.iotdb.db.service.metrics.enums.Metric;
import org.apache.iotdb.db.service.metrics.enums.Tag;
@@ -80,14 +81,40 @@ public class MManagerMetrics implements IMetricSet {
Tag.TYPE.toString(),
"total");
+ MetricService.getInstance()
+ .getOrCreateAutoGauge(
+ Metric.QUANTITY.toString(),
+ MetricLevel.IMPORTANT,
+ mManager,
+ MManager::getTotalTemplateActivatedNumber,
+ Tag.NAME.toString(),
+ "deviceUsingTemplate",
+ Tag.TYPE.toString(),
+ "template");
+
MetricService.getInstance()
.getOrCreateAutoGauge(
Metric.MEM.toString(),
MetricLevel.IMPORTANT,
mManager,
- MManager::getMtreeSize,
+ MManager::getTotalEstimatedMemoryUsage,
Tag.NAME.toString(),
- "mtree");
+ "schemaUsage",
+ Tag.TYPE.toString(),
+ "total");
+
+ MetricService.getInstance()
+ .getOrCreateAutoGauge(
+ Metric.MEM.toString(),
+ MetricLevel.IMPORTANT,
+ mManager,
+ mManager ->
+ IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForSchema()
+ - mManager.getTotalEstimatedMemoryUsage(),
+ Tag.NAME.toString(),
+ "schemaRemaining",
+ Tag.TYPE.toString(),
+ "total");
}
@Override
@@ -129,6 +156,30 @@ public class MManagerMetrics implements IMetricSet {
"total");
MetricService.getInstance()
- .remove(MetricType.GAUGE, Metric.MEM.toString(), Tag.NAME.toString(), "mtree");
+ .remove(
+ MetricType.GAUGE,
+ Metric.QUANTITY.toString(),
+ Tag.NAME.toString(),
+ "deviceUsingTemplate",
+ Tag.TYPE.toString(),
+ "template");
+
+ MetricService.getInstance()
+ .remove(
+ MetricType.GAUGE,
+ Metric.MEM.toString(),
+ Tag.NAME.toString(),
+ "schemaUsage",
+ Tag.TYPE.toString(),
+ "total");
+
+ MetricService.getInstance()
+ .remove(
+ MetricType.GAUGE,
+ Metric.MEM.toString(),
+ Tag.NAME.toString(),
+ "schemaRemaining",
+ Tag.TYPE.toString(),
+ "total");
}
}