You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/06/05 10:46:41 UTC
[iotdb] branch fix_delete_device created (now 4c4dd6e4154)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch fix_delete_device
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 4c4dd6e4154 fix tefileResource error after delete device in sequence working memtable
This branch includes the following new commits:
new 4c4dd6e4154 fix tefileResource error after delete device in sequence working memtable
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: fix tefileResource error after delete device in sequence working memtable
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch fix_delete_device
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c4dd6e41545e98945ec128922ff1e131a587443
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Jun 5 18:46:25 2023 +0800
fix tefileResource error after delete device in sequence working memtable
---
.../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++-
.../db/engine/storagegroup/TsFileResource.java | 13 +++++++++--
.../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++
3 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 3d399ecdff6..b6806a99c86 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -680,6 +680,11 @@ public class TsFileProcessor {
}
try {
if (workMemTable != null) {
+ logger.info(
+ "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable",
+ deletion.getPath(),
+ deletion.getStartTime(),
+ deletion.getEndTime());
for (PartialPath device : devicePaths) {
workMemTable.delete(
deletion.getPath(), device, deletion.getStartTime(), deletion.getEndTime());
@@ -953,7 +958,15 @@ public class TsFileProcessor {
Map<String, Long> lastTimeForEachDevice = new HashMap<>();
if (sequence) {
lastTimeForEachDevice = tobeFlushed.getMaxTime();
- tsFileResource.updateEndTime(lastTimeForEachDevice);
+ // If some devices have been removed in MemTable, the number of device in MemTable and
+ // tsFileResource will not be the same. And the endTime of these devices in resource will be
+ // Long.minValue.
+ // In the case, we need to delete the removed devices in tsFileResource.
+ if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) {
+ tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice);
+ } else {
+ tsFileResource.updateEndTime(lastTimeForEachDevice);
+ }
}
for (FlushListener flushListener : flushListeners) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index e08ba2c9982..c3cea8210e5 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -1112,8 +1112,17 @@ public class TsFileResource {
}
}
- public void updateEndTime(Map<String, Long> times) {
- for (Map.Entry<String, Long> entry : times.entrySet()) {
+ public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
+ newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey()));
+ newTimeIndex.updateEndTime(entry.getKey(), entry.getValue());
+ }
+ timeIndex = newTimeIndex;
+ }
+
+ public void updateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
timeIndex.updateEndTime(entry.getKey(), entry.getValue());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
index 34b2655e6cc..066404ddb06 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
@@ -1118,6 +1118,33 @@ public class DataRegionTest {
Assert.assertEquals(3, tsFileResource.getModFile().getModifications().size());
}
+ @Test
+ public void testDeleteDataInSeqWorkingMemtable()
+ throws IllegalPathException, WriteProcessException, IOException {
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d0");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d199");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0);
+
+ // delete data which is not in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null);
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null);
+
+ // delete data which is in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null);
+
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+ Assert.assertFalse(tsFileResource.getModFile().exists());
+ Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199"));
+ }
+
@Test
public void testFlushingEmptyMemtable()
throws IllegalPathException, WriteProcessException, IOException {