You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2024/03/20 13:07:35 UTC
(iotdb) 01/01: Add Query Metric for TsFileResourceManager
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ty/ResourceMemory
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1e84048e4645037426ef94caedb76b82d458aa0f
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Mar 20 21:06:48 2024 +0800
Add Query Metric for TsFileResourceManager
---
.../metric/QueryRelatedResourceMetricSet.java | 88 ++++++++++++++++++++++
.../rescon/memory/TsFileResourceManager.java | 35 ++++++---
.../rescon/memory/ResourceManagerTest.java | 10 +--
3 files changed, 118 insertions(+), 15 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
index a5f5d9ed4e0..496a2437bf8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeService
import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool;
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
+import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager;
import org.apache.iotdb.metrics.AbstractMetricService;
import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
import org.apache.iotdb.metrics.metricsets.IMetricSet;
@@ -57,6 +58,19 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
private static final String FRAGMENT_INSTANCE_EXECUTION_SIZE = "fragment_instance_execution_size";
private static final String FRAGMENT_INSTANCE_DRIVER_SIZE = "fragment_instance_driver_size";
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ // FragmentInstanceManager
+ /////////////////////////////////////////////////////////////////////////////////////////////////
+ private static final TsFileResourceManager TS_FILE_RESOURCE_MANAGER =
+ TsFileResourceManager.getInstance();
+ private static final String RESOURCE_INDEX = "resource_index";
+ private static final String RESOURCE_INDEX_NUM_TYPE = "number";
+ private static final String TOTAL_RESOURCE_NUM = "total_resource_num";
+ private static final String DEGRADED_RESOURCE_NUM = "degraded_resource_num";
+ private static final String RESOURCE_MEMORY_TYPE = "memory";
+ private static final String RESOURCE_INDEX_MAX_MEMORY = "max_memory";
+ private static final String RESOURCE_INDEX_USED_MEMORY = "used_memory";
+
private Timer fragmentInstanceExecutionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Histogram fragmentInstanceContextSizeHistogram =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
@@ -191,6 +205,47 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
MetricLevel.IMPORTANT,
Tag.NAME.toString(),
ESTIMATED_MEMORY_SIZE);
+
+ // resource index
+ metricService.createAutoGauge(
+ RESOURCE_INDEX,
+ MetricLevel.IMPORTANT,
+ TS_FILE_RESOURCE_MANAGER,
+ TsFileResourceManager::getDegradedTimeIndexNum,
+ Tag.TYPE.toString(),
+ RESOURCE_INDEX_NUM_TYPE,
+ Tag.NAME.toString(),
+ DEGRADED_RESOURCE_NUM);
+
+ metricService.createAutoGauge(
+ RESOURCE_INDEX,
+ MetricLevel.IMPORTANT,
+ TS_FILE_RESOURCE_MANAGER,
+ TsFileResourceManager::getPriorityQueueSize,
+ Tag.TYPE.toString(),
+ RESOURCE_INDEX_NUM_TYPE,
+ Tag.NAME.toString(),
+ TOTAL_RESOURCE_NUM);
+
+ metricService.createAutoGauge(
+ RESOURCE_INDEX,
+ MetricLevel.IMPORTANT,
+ TS_FILE_RESOURCE_MANAGER,
+ TsFileResourceManager::getTotalTimeIndexMemCost,
+ Tag.TYPE.toString(),
+ RESOURCE_MEMORY_TYPE,
+ Tag.NAME.toString(),
+ RESOURCE_INDEX_USED_MEMORY);
+
+ metricService.createAutoGauge(
+ RESOURCE_INDEX,
+ MetricLevel.IMPORTANT,
+ TS_FILE_RESOURCE_MANAGER,
+ TsFileResourceManager::getTimeIndexMemoryThreshold,
+ Tag.TYPE.toString(),
+ RESOURCE_MEMORY_TYPE,
+ Tag.NAME.toString(),
+ RESOURCE_INDEX_MAX_MEMORY);
}
@Override
@@ -245,6 +300,39 @@ public class QueryRelatedResourceMetricSet implements IMetricSet {
FREE_MEMORY_FOR_OPERATORS);
metricService.remove(
MetricType.HISTOGRAM, LOCAL_EXECUTION_PLANNER, Tag.NAME.toString(), ESTIMATED_MEMORY_SIZE);
+
+ // resource index
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ RESOURCE_INDEX,
+ Tag.TYPE.toString(),
+ RESOURCE_INDEX_NUM_TYPE,
+ Tag.NAME.toString(),
+ DEGRADED_RESOURCE_NUM);
+
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ RESOURCE_INDEX,
+ Tag.TYPE.toString(),
+ RESOURCE_INDEX_NUM_TYPE,
+ Tag.NAME.toString(),
+ TOTAL_RESOURCE_NUM);
+
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ RESOURCE_INDEX,
+ Tag.TYPE.toString(),
+ RESOURCE_MEMORY_TYPE,
+ Tag.NAME.toString(),
+ RESOURCE_INDEX_USED_MEMORY);
+
+ metricService.remove(
+ MetricType.AUTO_GAUGE,
+ RESOURCE_INDEX,
+ Tag.TYPE.toString(),
+ RESOURCE_MEMORY_TYPE,
+ Tag.NAME.toString(),
+ RESOURCE_INDEX_MAX_MEMORY);
}
private QueryRelatedResourceMetricSet() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
index 1480d779004..9da89447c72 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
@@ -35,7 +35,7 @@ public class TsFileResourceManager {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
/** threshold total memory for all TimeIndex */
- private double TIME_INDEX_MEMORY_THRESHOLD = CONFIG.getAllocateMemoryForTimeIndex();
+ private long timeIndexMemoryThreshold = CONFIG.getAllocateMemoryForTimeIndex();
/** store the sealed TsFileResource, sorted by priority of TimeIndex */
private final TreeSet<TsFileResource> sealedTsFileResources =
@@ -44,12 +44,14 @@ public class TsFileResourceManager {
/** total used memory for TimeIndex */
private long totalTimeIndexMemCost;
+ // degraded time index number
+ private long degradedTimeIndexNum = 0;
+
@TestOnly
- public void setTimeIndexMemoryThreshold(double timeIndexMemoryThreshold) {
- TIME_INDEX_MEMORY_THRESHOLD = timeIndexMemoryThreshold;
+ public void setTimeIndexMemoryThreshold(long timeIndexMemoryThreshold) {
+ this.timeIndexMemoryThreshold = timeIndexMemoryThreshold;
}
- @TestOnly
public long getPriorityQueueSize() {
return sealedTsFileResources.size();
}
@@ -73,6 +75,7 @@ public class TsFileResourceManager {
if (TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())
== TimeIndexLevel.FILE_TIME_INDEX) {
totalTimeIndexMemCost -= tsFileResource.calculateRamSize();
+ degradedTimeIndexNum--;
} else {
totalTimeIndexMemCost -= tsFileResource.getRamSize();
}
@@ -89,7 +92,7 @@ public class TsFileResourceManager {
* threshold.
*/
private void chooseTsFileResourceToDegrade() {
- while (totalTimeIndexMemCost > TIME_INDEX_MEMORY_THRESHOLD) {
+ while (totalTimeIndexMemCost > timeIndexMemoryThreshold) {
TsFileResource tsFileResource = sealedTsFileResources.pollFirst();
if (tsFileResource == null
|| TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())
@@ -100,27 +103,39 @@ public class TsFileResourceManager {
}
long memoryReduce = tsFileResource.degradeTimeIndex();
logger.debug("Degrade tsfile resource {}", tsFileResource.getTsFilePath());
+ degradedTimeIndexNum++;
releaseTimeIndexMemCost(memoryReduce);
// add the polled tsFileResource to the priority queue
sealedTsFileResources.add(tsFileResource);
}
}
+ public long getDegradedTimeIndexNum() {
+ return degradedTimeIndexNum;
+ }
+
+ public long getTimeIndexMemoryThreshold() {
+ return timeIndexMemoryThreshold;
+ }
+
+ public long getTotalTimeIndexMemCost() {
+ return totalTimeIndexMemCost;
+ }
+
/** function for clearing TsFileManager */
public synchronized void clear() {
- if (this.sealedTsFileResources != null) {
- this.sealedTsFileResources.clear();
- }
+ this.sealedTsFileResources.clear();
this.totalTimeIndexMemCost = 0;
+ this.degradedTimeIndexNum = 0;
}
public static TsFileResourceManager getInstance() {
- return TsFileResourceManager.InstanceHolder.instance;
+ return TsFileResourceManager.InstanceHolder.INSTANCE;
}
private static class InstanceHolder {
private InstanceHolder() {}
- private static TsFileResourceManager instance = new TsFileResourceManager();
+ private static final TsFileResourceManager INSTANCE = new TsFileResourceManager();
}
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
index 2717c6e8d40..8ed1dc60e99 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java
@@ -209,7 +209,7 @@ public class ResourceManagerTest {
assertEquals(
TimeIndexLevel.DEVICE_TIME_INDEX,
TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()));
- double curTimeIndexMemoryThreshold = 322;
+ long curTimeIndexMemoryThreshold = 322;
tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold);
tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
assertEquals(
@@ -237,7 +237,7 @@ public class ResourceManagerTest {
TimeIndexLevel.DEVICE_TIME_INDEX,
TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()));
long previousRamSize = tsFileResource.calculateRamSize();
- double curTimeIndexMemoryThreshold = 3221;
+ long curTimeIndexMemoryThreshold = 3221;
tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold);
tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
assertEquals(0, previousRamSize - tsFileResource.calculateRamSize());
@@ -266,7 +266,7 @@ public class ResourceManagerTest {
assertEquals(
TimeIndexLevel.DEVICE_TIME_INDEX,
TimeIndexLevel.valueOf(tsFileResource1.getTimeIndexType()));
- double curTimeIndexMemoryThreshold = 3221;
+ long curTimeIndexMemoryThreshold = 3221;
tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold);
tsFileResourceManager.registerSealedTsFileResource(tsFileResource1);
assertEquals(
@@ -300,7 +300,7 @@ public class ResourceManagerTest {
@Test
public void testMultiDeviceTimeIndexDegrade() throws IOException, WriteProcessException {
- double curTimeIndexMemoryThreshold = 9663.7;
+ long curTimeIndexMemoryThreshold = 9663;
tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold);
for (int i = 0; i < seqFileNum; i++) {
File file =
@@ -340,7 +340,7 @@ public class ResourceManagerTest {
@Test
public void testAllFileTimeIndexDegrade() throws IOException, WriteProcessException {
- double curTimeIndexMemoryThreshold = 322;
+ long curTimeIndexMemoryThreshold = 322;
tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold);
for (int i = 0; i < seqFileNum; i++) {
File file =