You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/06/07 06:56:42 UTC
[iotdb] branch fix_delete_resource_error_1.2 created (now 9893884fa38)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch fix_delete_resource_error_1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 9893884fa38 Fix TsfileResource error after delete device in sequence working memtable
This branch includes the following new commits:
new 9893884fa38 Fix TsfileResource error after delete device in sequence working memtable
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Fix TsfileResource error after delete device in sequence working memtable
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch fix_delete_resource_error_1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9893884fa38e6540d2463a8de0e9e118cf063698
Author: Haonan <hh...@outlook.com>
AuthorDate: Tue Jun 6 01:05:08 2023 +0800
Fix TsfileResource error after delete device in sequence working memtable
---
.../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++-
.../db/engine/storagegroup/TsFileResource.java | 13 +++++++++--
.../db/engine/settle/SettleRequestHandlerTest.java | 1 -
.../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++
4 files changed, 52 insertions(+), 4 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 656e30b40f1..f4bdddea394 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -710,6 +710,11 @@ public class TsFileProcessor {
}
try {
if (workMemTable != null) {
+ logger.info(
+ "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable",
+ deletion.getPath(),
+ deletion.getStartTime(),
+ deletion.getEndTime());
for (PartialPath device : devicePaths) {
workMemTable.delete(
deletion.getPath(), device, deletion.getStartTime(), deletion.getEndTime());
@@ -987,7 +992,15 @@ public class TsFileProcessor {
Map<String, Long> lastTimeForEachDevice = new HashMap<>();
if (sequence) {
lastTimeForEachDevice = tobeFlushed.getMaxTime();
- tsFileResource.updateEndTime(lastTimeForEachDevice);
+ // If some devices have been removed in MemTable, the number of device in MemTable and
+ // tsFileResource will not be the same. And the endTime of these devices in resource will be
+ // Long.minValue.
+ // In the case, we need to delete the removed devices in tsFileResource.
+ if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) {
+ tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice);
+ } else {
+ tsFileResource.updateEndTime(lastTimeForEachDevice);
+ }
}
for (FlushListener flushListener : flushListeners) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index e6c430da223..bdb93bbd165 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -1192,8 +1192,17 @@ public class TsFileResource {
}
}
- public void updateEndTime(Map<String, Long> times) {
- for (Map.Entry<String, Long> entry : times.entrySet()) {
+ public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
+ newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey()));
+ newTimeIndex.updateEndTime(entry.getKey(), entry.getValue());
+ }
+ timeIndex = newTimeIndex;
+ }
+
+ public void updateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
timeIndex.updateEndTime(entry.getKey(), entry.getValue());
}
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
index f849ac9f63d..e832df7b398 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
@@ -143,7 +143,6 @@ public class SettleRequestHandlerTest {
}
for (TsFileProcessor tsFileProcessor : dataRegion.getWorkSequenceTsFileProcessors()) {
paths.add(tsFileProcessor.getTsFileResource().getTsFilePath());
- tsFileProcessor.syncFlush();
}
dataRegion.syncCloseAllWorkingTsFileProcessors();
if (i != 2) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
index 89406f8f580..acf7e6eb672 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
@@ -1118,6 +1118,33 @@ public class DataRegionTest {
Assert.assertEquals(3, tsFileResource.getModFile().getModifications().size());
}
+ @Test
+ public void testDeleteDataInSeqWorkingMemtable()
+ throws IllegalPathException, WriteProcessException, IOException {
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d0");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d199");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0);
+
+ // delete data which is not in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null);
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null);
+
+ // delete data which is in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null);
+
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+ Assert.assertFalse(tsFileResource.getModFile().exists());
+ Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199"));
+ }
+
@Test
public void testFlushingEmptyMemtable()
throws IllegalPathException, WriteProcessException, IOException {