You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/11/28 14:34:39 UTC
(iotdb) branch fix_deletion_merge created (now 28832d1c3cd)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch fix_deletion_merge
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 28832d1c3cd Fix that table deletion will be mistakenly merged together
This branch includes the following new commits:
new 28832d1c3cd Fix that table deletion will be mistakenly merged together
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Fix that table deletion will be mistakenly merged together
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch fix_deletion_merge
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 28832d1c3cdf07d966b3e85c89dbbe505199f40c
Author: Tian Jiang <jt...@163.com>
AuthorDate: Thu Nov 28 22:36:58 2024 +0800
Fix that table deletion will be mistakenly merged together
---
.../relational/it/db/it/IoTDBDeletionTableIT.java | 37 ++++++++++++++++++++++
.../modification/TableDeletionEntry.java | 4 +++
.../apache/iotdb/db/utils/ModificationUtils.java | 21 +++++++++++-
3 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
index 7c3c3ceab1c..5e5a0932fe7 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
@@ -671,6 +671,43 @@ public class IoTDBDeletionTableIT {
}
}
+ @Test
+ public void testMergeDeletion() throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database db1");
+ statement.execute("use db1");
+ statement.execute(
+ "create table t1(country id,region id, city id, device id, ab1 ATTRIBUTE, s1 int32, s2 float, s3 boolean, s4 string)");
+ statement.execute("INSERT INTO t1(time,country,region,city,device,ab1,s1,s2,s3,s4) values (100,'china','hebei','shijiazhuang','d1','ab1',1,1,1,1),(200,null,'hebei','shijiazhuang','d2','ab2',1,1,1,1),(300,'china','beijing','beijing','d1','ab3',1,1,1,1),(400,'china','tianjin','tianjin','d1','ab4',1,1,1,1),(500,'china','sichuan','chengdu','d1',null,1,1,1,1),(600,'china','zhejiang','hangzhou','d1','ab6',1,1,1,1),(700,'japan','dao','tokyo','d1','ab7',1,1,1,1),(800,'canada','tronto','shi [...]
+ statement.execute("flush");
+ int cnt = 0;
+ try (ResultSet set = statement.executeQuery("select time,country,region,city,device,ab1,s1,s2,s3,s4 from t1 order by time")) {
+ while (set.next()) {
+ cnt ++;
+ }
+ assertEquals(13, cnt);
+ }
+ cnt = 0;
+ statement.execute("delete from t1 where country='japan'");
+ try (ResultSet set = statement.executeQuery("select time,country,region,city,device,ab1,s1,s2,s3,s4 from t1 order by time")) {
+ while (set.next()) {
+ cnt ++;
+ }
+ assertEquals(12, cnt);
+ }
+ cnt = 0;
+ statement.execute("delete from t1 where country='china' and region='beijing'");
+ try (ResultSet set = statement.executeQuery("select time,country,region,city,device,ab1,s1,s2,s3,s4 from t1 order by time")) {
+ while (set.next()) {
+ cnt ++;
+ }
+ assertEquals(11, cnt);
+ }
+
+ }
+ }
+
@Ignore
@Test
public void testDeletionWritePerformance() throws SQLException, IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
index aec0823c3ca..9dfbe6102a4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntry.java
@@ -148,4 +148,8 @@ public class TableDeletionEntry extends ModEntry {
public int hashCode() {
return Objects.hash(predicate, timeRange);
}
+
+ public DeletionPredicate getPredicate() {
+ return predicate;
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
index 984e620ea8b..99d2bc137ab 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
@@ -22,6 +22,8 @@ package org.apache.iotdb.db.utils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SettleSelectorImpl;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry;
+import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
@@ -34,6 +36,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Objects;
public class ModificationUtils {
@@ -360,6 +363,22 @@ public class ModificationUtils {
return extendedRightMin <= left.getMax();
}
+ public static boolean canMerge(ModEntry left, ModEntry right) {
+ if (!Objects.equals(left.getClass(), right.getClass())) {
+ return false;
+ }
+
+ if (left instanceof TreeDeletionEntry) {
+ return Objects.equals(
+ ((TreeDeletionEntry) left).getPathPattern(),
+ ((TreeDeletionEntry) right).getPathPattern());
+ } else if (left instanceof TableDeletionEntry) {
+ return Objects.equals(
+ ((TableDeletionEntry) left).getPredicate(), ((TableDeletionEntry) right).getPredicate());
+ }
+ return false;
+ }
+
public static List<ModEntry> sortAndMerge(List<ModEntry> modifications) {
modifications.sort(Comparator.comparing(ModEntry::getTimeRange));
List<ModEntry> result = new ArrayList<>();
@@ -367,7 +386,7 @@ public class ModificationUtils {
ModEntry current = modifications.get(0).clone();
for (int i = 1; i < modifications.size(); i++) {
ModEntry next = modifications.get(i);
- if (canMerge(current.getTimeRange(), next.getTimeRange())) {
+ if (canMerge(current, next)) {
current.getTimeRange().merge(next.getTimeRange());
} else {
result.add(current);