You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2023/01/30 09:02:30 UTC
[iotdb] 01/01: add metric for mods file
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-5418
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a30220ff4c2876fcd787e71dde44f7e7874a5fef
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Mon Jan 30 16:59:46 2023 +0800
add metric for mods file
---
.../org/apache/iotdb/db/engine/TsFileMetricManager.java | 14 ++++++++++++++
.../compaction/execute/task/CrossSpaceCompactionTask.java | 12 ++++++++++++
.../engine/compaction/execute/utils/CompactionUtils.java | 8 ++++++++
.../apache/iotdb/db/engine/storagegroup/DataRegion.java | 12 ++++++++++++
.../org/apache/iotdb/db/service/metrics/FileMetrics.java | 9 +++++++++
5 files changed, 55 insertions(+)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java b/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
index d5ccfb78ee..22f08139fc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/TsFileMetricManager.java
@@ -32,6 +32,8 @@ public class TsFileMetricManager {
private final AtomicInteger seqFileNum = new AtomicInteger(0);
private final AtomicInteger unseqFileNum = new AtomicInteger(0);
+ private final AtomicInteger modFileNum = new AtomicInteger(0);
+
// compaction temporal files
private final AtomicLong innerSeqCompactionTempFileSize = new AtomicLong(0);
private final AtomicLong innerUnseqCompactionTempFileSize = new AtomicLong(0);
@@ -74,6 +76,18 @@ public class TsFileMetricManager {
return seq ? seqFileNum.get() : unseqFileNum.get();
}
+ public int getModFileNum() {
+ return modFileNum.get();
+ }
+
+ public void increaseModFileNum(int num) {
+ modFileNum.addAndGet(num);
+ }
+
+ public void decreaseModFileNum(int num) {
+ modFileNum.addAndGet(-num);
+ }
+
public void addCompactionTempFileSize(boolean innerSpace, boolean seq, long delta) {
if (innerSpace) {
long unused =
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 8ae67744ba..dc4a9842d4 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -184,6 +184,18 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
releaseReadAndLockWrite(selectedSequenceFiles);
releaseReadAndLockWrite(selectedUnsequenceFiles);
+ for (TsFileResource sequenceResource : selectedSequenceFiles) {
+ if (sequenceResource.getModFile().exists()) {
+ TsFileMetricManager.getInstance().decreaseModFileNum(1);
+ }
+ }
+
+ for (TsFileResource unsequenceResource : selectedUnsequenceFiles) {
+ if (unsequenceResource.getModFile().exists()) {
+ TsFileMetricManager.getInstance().decreaseModFileNum(1);
+ }
+ }
+
long sequenceFileSize = deleteOldFiles(selectedSequenceFiles);
long unsequenceFileSize = deleteOldFiles(selectedUnsequenceFiles);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
index a4bf898447..81d2337d3c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/CompactionUtils.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.compaction.execute.utils;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.engine.TsFileMetricManager;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.modification.ModificationFile;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
@@ -118,6 +119,9 @@ public class CompactionUtils {
Set<Modification> seqModifications =
new HashSet<>(ModificationFile.getCompactionMods(seqResources.get(i)).getModifications());
modifications.addAll(seqModifications);
+ if (modifications.size() > 0) {
+ TsFileMetricManager.getInstance().increaseModFileNum(1);
+ }
updateOneTargetMods(targetResource, modifications);
modifications.removeAll(seqModifications);
}
@@ -136,6 +140,9 @@ public class CompactionUtils {
modifications.addAll(sourceCompactionModificationFile.getModifications());
}
}
+ if (modifications.size() > 0) {
+ TsFileMetricManager.getInstance().increaseModFileNum(1);
+ }
updateOneTargetMods(targetTsFile, modifications);
}
@@ -199,6 +206,7 @@ public class CompactionUtils {
ModificationFile normalModification = ModificationFile.getNormalMods(tsFileResource);
if (normalModification.exists()) {
normalModification.remove();
+ TsFileMetricManager.getInstance().decreaseModFileNum(1);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 278edfd28a..d688d99cfa 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -1472,6 +1472,14 @@ public class DataRegion implements IDataRegionForQuery {
// normally, mergingModification is just need to be closed by after a merge task is finished.
// we close it here just for IT test.
closeAllResources();
+ List<TsFileResource> tsFileResourceList = tsFileManager.getTsFileList(true);
+ tsFileResourceList.addAll(tsFileManager.getTsFileList(false));
+ tsFileResourceList.forEach(
+ x -> {
+ if (x.getModFile().exists()) {
+ TsFileMetricManager.getInstance().decreaseModFileNum(1);
+ }
+ });
deleteAllSGFolders(DirectoryManager.getInstance().getAllFilesFolders());
this.workSequenceTsFileProcessors.clear();
@@ -2004,9 +2012,13 @@ public class DataRegion implements IDataRegionForQuery {
} else {
deletion.setFileOffset(tsFileResource.getTsFileSize());
// write deletion into modification file
+ boolean modFileExists = tsFileResource.getModFile().exists();
tsFileResource.getModFile().write(deletion);
// remember to close mod file
tsFileResource.getModFile().close();
+ if (!modFileExists) {
+ TsFileMetricManager.getInstance().increaseModFileNum(1);
+ }
}
logger.info(
"[Deletion] Deletion with path:{}, time:{}-{} written into mods file:{}.",
diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index 53573f7286..755e45e446 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -85,6 +85,13 @@ public class FileMetrics implements IMetricSet {
o -> o.getFileNum(false),
Tag.NAME.toString(),
"unseq");
+ metricService.createAutoGauge(
+ Metric.FILE_COUNT.toString(),
+ MetricLevel.IMPORTANT,
+ TS_FILE_METRIC_MANAGER,
+ o -> o.getModFileNum(),
+ Tag.NAME.toString(),
+ "mods");
}
private void bindWalFileMetrics(AbstractMetricService metricService) {
@@ -184,6 +191,8 @@ public class FileMetrics implements IMetricSet {
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "seq");
metricService.remove(
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "unseq");
+ metricService.remove(
+ MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), "mods");
}
private void unbindWalMetrics(AbstractMetricService metricService) {