You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/12/11 08:45:09 UTC
(iotdb) branch dev/1.3 updated: [to dev/1.3] When deleting data for TsFileResource in COMPACTING status, the mod file metrics is not modified (#14376)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new f6b0cf934eb [to dev/1.3] When deleting data for TsFileResource in COMPACTING status, the mod file metrics is not modified (#14376)
f6b0cf934eb is described below
commit f6b0cf934eb7a953b602c71807d41e5671bfade5
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Wed Dec 11 16:45:04 2024 +0800
[to dev/1.3] When deleting data for TsFileResource in COMPACTING status, the mod file metrics is not modified (#14376)
* When deleting data for TsFileResource in COMPACTING status, the mod file metrics is not modified
* add ut
---
.../iotdb/db/service/metrics/FileMetrics.java | 5 ++++
.../db/service/metrics/file/ModsFileMetrics.java | 2 +-
.../db/storageengine/dataregion/DataRegion.java | 33 ++++++++++-----------
.../storageengine/dataregion/DataRegionTest.java | 34 ++++++++++++++++++++++
4 files changed, 55 insertions(+), 19 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
index 34fae0f3bec..6ef16d2ab50 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java
@@ -109,4 +109,9 @@ public class FileMetrics implements IMetricSet {
public long getFileCount(boolean seq) {
return TS_FILE_METRICS.getFileCount(seq);
}
+
+ @TestOnly
+ public int getModFileNum() {
+ return MODS_FILE_METRICS.getModFileNum();
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
index 8ddefeddd24..dc86f6d7e9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ModsFileMetrics.java
@@ -60,7 +60,7 @@ public class ModsFileMetrics implements IMetricSet {
MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), MODS);
}
- private int getModFileNum() {
+ public int getModFileNum() {
return modFileNum.get();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 213f25cf000..1a899a8c521 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2392,6 +2392,7 @@ public class DataRegion implements IDataRegionForQuery {
synchronized (modFile) {
try {
originSize = modFile.getSize();
+ boolean modFileExists = modFile.exists();
// delete data in sealed file
if (tsFileResource.isCompacting()) {
// we have to set modification offset to MAX_VALUE, as the offset of source chunk may
@@ -2407,8 +2408,6 @@ public class DataRegion implements IDataRegionForQuery {
} else {
deletion.setFileOffset(tsFileResource.getTsFileSize());
// write deletion into modification file
- boolean modFileExists = modFile.exists();
-
modFile.write(deletion);
// remember to close mod file
@@ -2416,15 +2415,14 @@ public class DataRegion implements IDataRegionForQuery {
// if file length greater than 1M,execute compact.
modFile.compact();
+ }
- if (!modFileExists) {
- FileMetrics.getInstance().increaseModFileNum(1);
- }
-
- // The file size may be smaller than the original file, so the increment here may be
- // negative
- FileMetrics.getInstance().increaseModFileSize(modFile.getSize() - originSize);
+ if (!modFileExists) {
+ FileMetrics.getInstance().increaseModFileNum(1);
}
+ // The file size may be smaller than the original file, so the increment here may be
+ // negative
+ FileMetrics.getInstance().increaseModFileSize(modFile.getSize() - originSize);
} catch (Throwable t) {
if (originSize != -1) {
modFile.truncate(originSize);
@@ -2469,6 +2467,7 @@ public class DataRegion implements IDataRegionForQuery {
synchronized (modFile) {
try {
originSize = modFile.getSize();
+ boolean modFileExists = modFile.exists();
// delete data in sealed file
if (tsFileResource.isCompacting()) {
// we have to set modification offset to MAX_VALUE, as the offset of source chunk
@@ -2484,7 +2483,6 @@ public class DataRegion implements IDataRegionForQuery {
} else {
deletion.setFileOffset(tsFileResource.getTsFileSize());
// write deletion into modification file
- boolean modFileExists = modFile.exists();
modFile.write(deletion);
@@ -2493,15 +2491,14 @@ public class DataRegion implements IDataRegionForQuery {
// if file length greater than 1M,execute compact.
modFile.compact();
-
- if (!modFileExists) {
- FileMetrics.getInstance().increaseModFileNum(1);
- }
-
- // The file size may be smaller than the original file, so the increment here may be
- // negative
- FileMetrics.getInstance().increaseModFileSize(modFile.getSize() - originSize);
}
+ if (!modFileExists) {
+ FileMetrics.getInstance().increaseModFileNum(1);
+ }
+
+ // The file size may be smaller than the original file, so the increment here may be
+ // negative
+ FileMetrics.getInstance().increaseModFileSize(modFile.getSize() - originSize);
} catch (Throwable t) {
if (originSize != -1) {
modFile.truncate(originSize);
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index 155db11082c..7361f940b48 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
+import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
@@ -55,6 +56,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
@@ -1583,4 +1585,36 @@ public class DataRegionTest {
Assert.assertFalse(tsFileResourceSeq.getModFile().exists());
Assert.assertFalse(tsFileResourceUnSeq.getModFile().exists());
}
+
+ @Test
+ public void testDeleteDataWithMetricUpdate()
+ throws IllegalPathException, IOException, WriteProcessException {
+ int modFileNumWhenTestStart = FileMetrics.getInstance().getModFileNum();
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, deviceId);
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+
+ for (TsFileResource resource : dataRegion.getTsFileManager().getTsFileList(true)) {
+ resource.setStatusForTest(TsFileResourceStatus.COMPACTING);
+ }
+
+ dataRegion.deleteDataDirectly(new PartialPath("root.vehicle.d0.**"), 101, 200, 0);
+
+ for (int j = 200; j < 300; j++) {
+ TSRecord record = new TSRecord(j, deviceId);
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+
+ dataRegion.deleteDataDirectly(new PartialPath("root.vehicle.d0.**"), 201, 300, 0);
+
+ Assert.assertEquals(
+ dataRegion.getTsFileManager().getTsFileList(true).size() + modFileNumWhenTestStart,
+ FileMetrics.getInstance().getModFileNum());
+ }
}