You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/06/07 06:46:06 UTC
[iotdb] 01/01: Fix TsfileResource error after delete device in sequence working memtable
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch fix_delete_resource_error_master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5336c96650607a3671c77ae7486c28d004e3c99d
Author: Haonan <hh...@outlook.com>
AuthorDate: Tue Jun 6 01:05:08 2023 +0800
Fix TsfileResource error after delete device in sequence working memtable
---
.../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++-
.../db/engine/storagegroup/TsFileResource.java | 13 +++++++++--
.../db/engine/settle/SettleRequestHandlerTest.java | 1 -
.../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++
4 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 656e30b40f1..f4bdddea394 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -710,6 +710,11 @@ public class TsFileProcessor {
}
try {
if (workMemTable != null) {
+ logger.info(
+ "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable",
+ deletion.getPath(),
+ deletion.getStartTime(),
+ deletion.getEndTime());
for (PartialPath device : devicePaths) {
workMemTable.delete(
deletion.getPath(), device, deletion.getStartTime(), deletion.getEndTime());
@@ -987,7 +992,15 @@ public class TsFileProcessor {
Map<String, Long> lastTimeForEachDevice = new HashMap<>();
if (sequence) {
lastTimeForEachDevice = tobeFlushed.getMaxTime();
- tsFileResource.updateEndTime(lastTimeForEachDevice);
+ // If some devices have been removed in MemTable, the number of device in MemTable and
+ // tsFileResource will not be the same. And the endTime of these devices in resource will be
+ // Long.minValue.
+ // In the case, we need to delete the removed devices in tsFileResource.
+ if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) {
+ tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice);
+ } else {
+ tsFileResource.updateEndTime(lastTimeForEachDevice);
+ }
}
for (FlushListener flushListener : flushListeners) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index e6c430da223..bdb93bbd165 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -1192,8 +1192,17 @@ public class TsFileResource {
}
}
- public void updateEndTime(Map<String, Long> times) {
- for (Map.Entry<String, Long> entry : times.entrySet()) {
+ public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
+ newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey()));
+ newTimeIndex.updateEndTime(entry.getKey(), entry.getValue());
+ }
+ timeIndex = newTimeIndex;
+ }
+
+ public void updateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
timeIndex.updateEndTime(entry.getKey(), entry.getValue());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
index f849ac9f63d..e832df7b398 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
@@ -143,7 +143,6 @@ public class SettleRequestHandlerTest {
}
for (TsFileProcessor tsFileProcessor : dataRegion.getWorkSequenceTsFileProcessors()) {
paths.add(tsFileProcessor.getTsFileResource().getTsFilePath());
- tsFileProcessor.syncFlush();
}
dataRegion.syncCloseAllWorkingTsFileProcessors();
if (i != 2) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
index 89406f8f580..acf7e6eb672 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
@@ -1118,6 +1118,33 @@ public class DataRegionTest {
Assert.assertEquals(3, tsFileResource.getModFile().getModifications().size());
}
+ @Test
+ public void testDeleteDataInSeqWorkingMemtable()
+ throws IllegalPathException, WriteProcessException, IOException {
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d0");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d199");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0);
+
+ // delete data which is not in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null);
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null);
+
+ // delete data which is in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null);
+
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+ Assert.assertFalse(tsFileResource.getModFile().exists());
+ Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199"));
+ }
+
@Test
public void testFlushingEmptyMemtable()
throws IllegalPathException, WriteProcessException, IOException {