You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2023/09/21 05:30:44 UTC
[iotdb] branch feature/disk-metric created (now 289b02cc071)
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/disk-metric
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 289b02cc071 add database and region tags into file metric
This branch includes the following new commits:
new 289b02cc071 add database and region tags into file metric
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: add database and region tags into file metric
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/disk-metric
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 289b02cc0713eab21ae0ed2333c8438c2550a2a2
Author: spricoder <sp...@qq.com>
AuthorDate: Thu Sep 21 13:30:23 2023 +0800
add database and region tags into file metric
---
.../iotdb/db/service/metrics/FileMetrics.java | 162 +++++++++++++++------
.../db/storageengine/dataregion/DataRegion.java | 47 +++---
.../execute/task/CrossSpaceCompactionTask.java | 6 +-
.../execute/task/InnerSpaceCompactionTask.java | 2 +
.../compaction/execute/utils/CompactionUtils.java | 14 +-
5 files changed, 151 insertions(+), 80 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index c57a3a8526e..8b7d627ea37 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.Abst
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.metrics.AbstractMetricService;
@@ -66,6 +67,10 @@ public class FileMetrics implements IMetricSet {
private static final String SEQUENCE = "sequence";
private static final String UNSEQUENCE = "unsequence";
private static final String LEVEL = "level";
+ private final Map<String, Map<String, Long>> seqFileSizeGaugeMap = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, Long>> unseqFileSizeGaugeMap = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, Integer>> seqFileNumGaugeMap = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, Integer>> unseqFileNumGaugeMap = new ConcurrentHashMap<>();
private final AtomicLong seqFileSize = new AtomicLong(0);
private final AtomicLong unseqFileSize = new AtomicLong(0);
private final AtomicInteger seqFileNum = new AtomicInteger(0);
@@ -114,20 +119,6 @@ public class FileMetrics implements IMetricSet {
}
private void bindTsFileMetrics(AbstractMetricService metricService) {
- metricService.createAutoGauge(
- Metric.FILE_SIZE.toString(),
- MetricLevel.CORE,
- this,
- o -> o.getFileSize(true),
- Tag.NAME.toString(),
- "seq");
- metricService.createAutoGauge(
- Metric.FILE_SIZE.toString(),
- MetricLevel.CORE,
- this,
- o -> o.getFileSize(false),
- Tag.NAME.toString(),
- "unseq");
metricService.createAutoGauge(
Metric.FILE_SIZE.toString(),
MetricLevel.CORE,
@@ -135,20 +126,6 @@ public class FileMetrics implements IMetricSet {
FileMetrics::getModFileSize,
Tag.NAME.toString(),
"mods");
- metricService.createAutoGauge(
- Metric.FILE_COUNT.toString(),
- MetricLevel.CORE,
- this,
- o -> o.getFileNum(true),
- Tag.NAME.toString(),
- "seq");
- metricService.createAutoGauge(
- Metric.FILE_COUNT.toString(),
- MetricLevel.CORE,
- this,
- o -> o.getFileNum(false),
- Tag.NAME.toString(),
- "unseq");
metricService.createAutoGauge(
Metric.FILE_COUNT.toString(),
MetricLevel.CORE,
@@ -333,8 +310,8 @@ public class FileMetrics implements IMetricSet {
}
// following are update functions for tsfile metrics
- public void addFile(long size, boolean seq, String name) {
- updateGlobalCountAndSize(size, 1, seq);
+ public void addFile(String database, String regionId, long size, boolean seq, String name) {
+ updateGlobalCountAndSize(database, regionId, size, 1, seq);
try {
TsFileNameGenerator.TsFileName tsFileName = TsFileNameGenerator.getTsFileName(name);
int level = tsFileName.getInnerCompactionCnt();
@@ -344,13 +321,110 @@ public class FileMetrics implements IMetricSet {
}
}
- private void updateGlobalCountAndSize(long sizeDelta, int countDelta, boolean seq) {
+ private void updateGlobalCountAndSize(
+ String database, String regionId, long sizeDelta, int countDelta, boolean seq) {
if (seq) {
- seqFileSize.getAndAdd(sizeDelta);
- seqFileNum.getAndAdd(countDelta);
+ // update sequence file size
+ seqFileSizeGaugeMap.compute(
+ database,
+ (k, v) -> {
+ long size = 0;
+ if (v == null) {
+ v = new ConcurrentHashMap<>();
+ } else if (v.containsKey(regionId)) {
+ size = v.get(regionId);
+ }
+ v.put(regionId, size + sizeDelta);
+ return v;
+ });
+ // update sequence file number
+ seqFileNumGaugeMap.compute(
+ database,
+ (k, v) -> {
+ int count = 0;
+ if (v == null) {
+ v = new ConcurrentHashMap<>();
+ } else if (v.containsKey(regionId)) {
+ count += countDelta;
+ }
+ v.put(regionId, count);
+ return v;
+ });
+ // update sequence file size metric
+ metricService
+ .getOrCreateGauge(
+ Metric.FILE_SIZE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "seq",
+ Tag.DATABASE.toString(),
+ database,
+ Tag.REGION.toString(),
+ regionId)
+ .set(seqFileSizeGaugeMap.get(database).get(regionId));
+ // update sequence file number metric
+ metricService
+ .getOrCreateGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "seq",
+ Tag.DATABASE.toString(),
+ database,
+ Tag.REGION.toString(),
+ regionId)
+ .set(seqFileNumGaugeMap.get(database).get(regionId));
} else {
- unseqFileSize.getAndAdd(sizeDelta);
- unseqFileNum.getAndAdd(countDelta);
+ // update unsequence file size
+ unseqFileSizeGaugeMap.compute(
+ database,
+ (k, v) -> {
+ long size = 0;
+ if (v == null) {
+ v = new ConcurrentHashMap<>();
+ } else if (v.containsKey(regionId)) {
+ size = v.get(regionId);
+ }
+ v.put(regionId, size + sizeDelta);
+ return v;
+ });
+ // update unsequence file number
+ unseqFileNumGaugeMap.compute(
+ database,
+ (k, v) -> {
+ int count = 0;
+ if (v == null) {
+ v = new ConcurrentHashMap<>();
+ } else if (v.containsKey(regionId)) {
+ count = v.get(regionId);
+ }
+ v.put(regionId, count + countDelta);
+ return v;
+ });
+ // update unsequence file size metric
+ metricService
+ .getOrCreateGauge(
+ Metric.FILE_SIZE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "unseq",
+ Tag.DATABASE.toString(),
+ database,
+ Tag.REGION.toString(),
+ regionId)
+ .set(unseqFileSizeGaugeMap.get(database).get(regionId));
+ // update unsequence file number metric
+ metricService
+ .getOrCreateGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "unseq",
+ Tag.DATABASE.toString(),
+ database,
+ Tag.REGION.toString(),
+ regionId)
+ .set(unseqFileNumGaugeMap.get(database).get(regionId));
}
}
@@ -435,19 +509,15 @@ public class FileMetrics implements IMetricSet {
.set(size);
}
- public void deleteFile(long[] sizeList, boolean seq, List<String> names) {
- long totalSize = 0;
- for (long size : sizeList) {
- totalSize += size;
- }
- updateGlobalCountAndSize(-totalSize, -sizeList.length, seq);
- for (int i = 0, length = names.size(); i < length; ++i) {
- int level = -1;
- String name = names.get(i);
- long size = sizeList[i];
+ public void deleteFile(boolean seq, List<TsFileResource> tsFileResourceList) {
+ for (TsFileResource tsFileResource : tsFileResourceList) {
+ String name = tsFileResource.getTsFile().getName();
+ long size = tsFileResource.getTsFileSize();
+ updateGlobalCountAndSize(
+ tsFileResource.getDatabaseName(), tsFileResource.getDataRegionId(), -size, -1, seq);
try {
TsFileNameGenerator.TsFileName tsFileName = TsFileNameGenerator.getTsFileName(name);
- level = tsFileName.getInnerCompactionCnt();
+ int level = tsFileName.getInnerCompactionCnt();
updateLevelCountAndSize(-size, -1, seq, level);
} catch (IOException e) {
log.warn("Unexpected error occurred when getting tsfile name", e);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 0794f469073..61f01a7c02a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -443,7 +443,12 @@ public class DataRegion implements IDataRegionForQuery {
for (TsFileResource resource : value) {
if (resource.resourceFileExists()) {
FileMetrics.getInstance()
- .addFile(resource.getTsFile().length(), true, resource.getTsFile().getName());
+ .addFile(
+ resource.getDatabaseName(),
+ resource.getDataRegionId(),
+ resource.getTsFile().length(),
+ true,
+ resource.getTsFile().getName());
if (resource.getModFile().exists()) {
FileMetrics.getInstance().increaseModFileNum(1);
FileMetrics.getInstance().increaseModFileSize(resource.getModFile().getSize());
@@ -469,7 +474,12 @@ public class DataRegion implements IDataRegionForQuery {
for (TsFileResource resource : value) {
if (resource.resourceFileExists()) {
FileMetrics.getInstance()
- .addFile(resource.getTsFile().length(), false, resource.getTsFile().getName());
+ .addFile(
+ resource.getDatabaseName(),
+ resource.getDataRegionId(),
+ resource.getTsFile().length(),
+ false,
+ resource.getTsFile().getName());
}
if (resource.getModFile().exists()) {
FileMetrics.getInstance().increaseModFileNum(1);
@@ -703,6 +713,8 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceManager.registerSealedTsFileResource(tsFileResource);
FileMetrics.getInstance()
.addFile(
+ tsFileResource.getDatabaseName(),
+ tsFileResource.getDataRegionId(),
tsFileResource.getTsFile().length(),
recoverPerformer.isSequence(),
tsFileResource.getTsFile().getName());
@@ -1445,11 +1457,7 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceList.addAll(tsFileManager.getTsFileList(false));
tsFileResourceList.forEach(
x -> {
- FileMetrics.getInstance()
- .deleteFile(
- new long[] {x.getTsFileSize()},
- x.isSeq(),
- Collections.singletonList(x.getTsFile().getName()));
+ FileMetrics.getInstance().deleteFile(x.isSeq(), Collections.singletonList(x));
if (x.getModFile().exists()) {
FileMetrics.getInstance().decreaseModFileNum(1);
FileMetrics.getInstance().decreaseModFileSize(x.getModFile().getSize());
@@ -1524,11 +1532,7 @@ public class DataRegion implements IDataRegionForQuery {
try {
// try to delete physical data file
resource.remove();
- FileMetrics.getInstance()
- .deleteFile(
- new long[] {resource.getTsFileSize()},
- isSeq,
- Collections.singletonList(resource.getTsFile().getName()));
+ FileMetrics.getInstance().deleteFile(isSeq, Collections.singletonList(resource));
logger.info(
"Removed a file {} before {} by ttl ({} {})",
resource.getTsFilePath(),
@@ -2074,11 +2078,14 @@ public class DataRegion implements IDataRegionForQuery {
synchronized (closeStorageGroupCondition) {
closeStorageGroupCondition.notifyAll();
}
+ TsFileResource tsFileResource = tsFileProcessor.getTsFileResource();
FileMetrics.getInstance()
.addFile(
- tsFileProcessor.getTsFileResource().getTsFileSize(),
+ tsFileResource.getDatabaseName(),
+ tsFileResource.getDataRegionId(),
+ tsFileResource.getTsFileSize(),
tsFileProcessor.isSequence(),
- tsFileProcessor.getTsFileResource().getTsFile().getName());
+ tsFileResource.getTsFile().getName());
logger.info("signal closing database condition in {}", databaseName + "-" + dataRegionId);
}
@@ -2203,6 +2210,8 @@ public class DataRegion implements IDataRegionForQuery {
FileMetrics.getInstance()
.addFile(
+ newTsFileResource.getDatabaseName(),
+ newTsFileResource.getDataRegionId(),
newTsFileResource.getTsFile().length(),
false,
newTsFileResource.getTsFile().getName());
@@ -2492,10 +2501,7 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceToBeMoved = sequenceResource;
tsFileManager.remove(tsFileResourceToBeMoved, true);
FileMetrics.getInstance()
- .deleteFile(
- new long[] {tsFileResourceToBeMoved.getTsFileSize()},
- true,
- Collections.singletonList(tsFileResourceToBeMoved.getTsFile().getName()));
+ .deleteFile(true, Collections.singletonList(tsFileResourceToBeMoved));
break;
}
}
@@ -2507,10 +2513,7 @@ public class DataRegion implements IDataRegionForQuery {
tsFileResourceToBeMoved = unsequenceResource;
tsFileManager.remove(tsFileResourceToBeMoved, false);
FileMetrics.getInstance()
- .deleteFile(
- new long[] {tsFileResourceToBeMoved.getTsFileSize()},
- false,
- Collections.singletonList(tsFileResourceToBeMoved.getTsFile().getName()));
+ .deleteFile(false, Collections.singletonList(tsFileResourceToBeMoved));
break;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index af9c49eb5b9..03ac69e3c0d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -220,7 +220,11 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
if (!targetResource.isDeleted()) {
FileMetrics.getInstance()
.addFile(
- targetResource.getTsFileSize(), true, targetResource.getTsFile().getName());
+ targetResource.getDatabaseName(),
+ targetResource.getDataRegionId(),
+ targetResource.getTsFileSize(),
+ true,
+ targetResource.getTsFile().getName());
// set target resources to CLOSED, so that they can be selected to compact
targetResource.setStatus(TsFileResourceStatus.NORMAL);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 301f7b16634..8fa7481654e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -258,6 +258,8 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
if (!targetTsFileResource.isDeleted()) {
FileMetrics.getInstance()
.addFile(
+ targetTsFileResource.getDatabaseName(),
+ targetTsFileResource.getDataRegionId(),
targetTsFileResource.getTsFile().length(),
sequence,
targetTsFileResource.getTsFile().getName());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index ae76fc1a0cf..a44730501ad 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -54,7 +54,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -446,9 +445,7 @@ public class CompactionUtils {
public static void deleteSourceTsFileAndUpdateFileMetrics(
List<TsFileResource> resources, boolean seq) {
- long[] fileSizes = new long[resources.size()];
- List<String> fileNames = new ArrayList<>(resources.size());
- int removeSuccessFileNum = 0;
+ List<TsFileResource> removeResources = new ArrayList<>();
for (TsFileResource resource : resources) {
if (!resource.remove()) {
logger.warn(
@@ -456,14 +453,9 @@ public class CompactionUtils {
resource.getTsFile().getAbsolutePath());
} else {
logger.info("[Compaction] delete file: {}", resource.getTsFile().getAbsolutePath());
- fileSizes[removeSuccessFileNum] = resource.getTsFileSize();
- fileNames.add(resource.getTsFile().getName());
- removeSuccessFileNum++;
+ removeResources.add(resource);
}
}
- if (removeSuccessFileNum != 0) {
- fileSizes = Arrays.copyOfRange(fileSizes, 0, removeSuccessFileNum);
- FileMetrics.getInstance().deleteFile(fileSizes, seq, fileNames);
- }
+ FileMetrics.getInstance().deleteFile(seq, resources);
}
}