You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/11/23 13:18:20 UTC
[iotdb] 01/04: init
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch delete_aligned
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2706b61b30cd89faba6e9f11365073e8f97d3497
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue Nov 23 16:33:59 2021 +0800
init
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 19 +-------------
.../engine/memtable/AlignedWritableMemChunk.java | 8 ++++--
.../memtable/AlignedWritableMemChunkGroup.java | 29 ++++++++++++++++++++++
.../db/engine/memtable/IWritableMemChunk.java | 2 +-
.../db/engine/memtable/IWritableMemChunkGroup.java | 3 +++
.../db/engine/memtable/WritableMemChunkGroup.java | 25 +++++++++++++++++++
.../db/utils/datastructure/AlignedTVList.java | 13 ++++++++--
...IoTDBRawQueryWithValueFilterWithDeletionIT.java | 2 +-
8 files changed, 77 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index c5e8ae2..bbb6db0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -348,24 +348,7 @@ public abstract class AbstractMemTable implements IMemTable {
if (memChunkGroup == null) {
return;
}
-
- Iterator<Entry<String, IWritableMemChunk>> iter =
- memChunkGroup.getMemChunkMap().entrySet().iterator();
- while (iter.hasNext()) {
- Entry<String, IWritableMemChunk> entry = iter.next();
- IWritableMemChunk chunk = entry.getValue();
- // the key is measurement rather than component of multiMeasurement
- PartialPath fullPath = devicePath.concatNode(entry.getKey());
- if (originalPath.matchFullPath(fullPath)) {
- // matchFullPath ensures this branch could work on delete data of unary or multi measurement
- // and delete timeseries or aligned timeseries
- if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) {
- iter.remove();
- }
- int deletedPointsNumber = chunk.delete(startTimestamp, endTimestamp);
- totalPointsNum -= deletedPointsNumber;
- }
- }
+ totalPointsNum -= memChunkGroup.delete(originalPath, devicePath, startTimestamp, endTimestamp);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index 001a3c2..caaa7fc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.db.engine.memtable;
+import java.util.Set;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
import org.apache.iotdb.db.utils.datastructure.TVList;
@@ -56,6 +57,10 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
this.list = TVListAllocator.getInstance().allocate(dataTypeList);
}
+ public Set<String> getAllMeasurements() {
+ return measurementIndexMap.keySet();
+ }
+
public boolean containsMeasurement(String measurementId) {
return measurementIndexMap.containsKey(measurementId);
}
@@ -242,9 +247,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
@Override
- // TODO: THIS METHOLD IS FOR DELETING ONE COLUMN OF A VECTOR
public int delete(long lowerBound, long upperBound, String measurementId) {
- return 0;
+ return list.delete(lowerBound, upperBound, measurementIndexMap.get(measurementId));
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
index 0a62bd6..e56abf1b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.db.engine.memtable;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
@@ -71,6 +75,31 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
+ public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp,
+ long endTimestamp) {
+ int deletedPointsNumber = 0;
+ boolean shouldDeleteAll = true;
+ Set<String> measurements = memChunk.getAllMeasurements();
+ for (String measurement : measurements) {
+ PartialPath fullPath = devicePath.concatNode(measurement);
+ if (!originalPath.matchFullPath(fullPath)) {
+ shouldDeleteAll = false;
+ }
+ }
+ if (shouldDeleteAll) {
+ deletedPointsNumber += memChunk.delete(startTimestamp, endTimestamp);
+ return deletedPointsNumber;
+ }
+ for (String measurement : measurements) {
+ PartialPath fullPath = devicePath.concatNode(measurement);
+ if (originalPath.matchFullPath(fullPath)) {
+ deletedPointsNumber += memChunk.delete(startTimestamp, endTimestamp, measurement);
+ }
+ }
+ return deletedPointsNumber;
+ }
+
+ @Override
public long getCurrentChunkPointNum(String measurement) {
return memChunk.count();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
index c2c1781..e17980b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java
@@ -124,7 +124,7 @@ public interface IWritableMemChunk {
/** @return how many points are deleted */
int delete(long lowerBound, long upperBound);
- // For delete one column in the vector
+ /** @return how many points are deleted */
int delete(long lowerBound, long upperBound, String measurementId);
IChunkWriter createIChunkWriter();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
index 85586aa..62b933e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.memtable;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
@@ -45,5 +46,7 @@ public interface IWritableMemChunkGroup {
Map<String, IWritableMemChunk> getMemChunkMap();
+ int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp);
+
long getCurrentChunkPointNum(String measurement);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
index 2ca32ce..2f97021 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
@@ -19,6 +19,9 @@
package org.apache.iotdb.db.engine.memtable;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.tsfile.utils.BitMap;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
@@ -112,6 +115,28 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
+ public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp,
+ long endTimestamp) {
+ int deletedPointsNumber = 0;
+ Iterator<Entry<String, IWritableMemChunk>> iter = memChunkMap.entrySet().iterator();
+ while (iter.hasNext()) {
+ Entry<String, IWritableMemChunk> entry = iter.next();
+ IWritableMemChunk chunk = entry.getValue();
+ // the key is measurement rather than component of multiMeasurement
+ PartialPath fullPath = devicePath.concatNode(entry.getKey());
+ if (originalPath.matchFullPath(fullPath)) {
+ // matchFullPath ensures this branch could work on delete data of unary or multi measurement
+ // and delete timeseries
+ if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) {
+ iter.remove();
+ }
+ deletedPointsNumber += chunk.delete(startTimestamp, endTimestamp);
+ }
+ }
+ return deletedPointsNumber;
+ }
+
+ @Override
public long getCurrentChunkPointNum(String measurement) {
return memChunkMap.get(measurement).count();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
index 1c9f6ed..9ce81f0 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java
@@ -453,9 +453,18 @@ public class AlignedTVList extends TVList {
return deletedNumber * getTsDataTypes().size();
}
- // TODO: THIS METHOLD IS FOR DELETING ONE COLUMN OF A VECTOR
public int delete(long lowerBound, long upperBound, int columnIndex) {
- throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT);
+ int deletedNumber = 0;
+ for (int i = 0; i < size; i++) {
+ long time = getTime(i);
+ if (time >= lowerBound && time <= upperBound) {
+ int arrayIndex = i / ARRAY_SIZE;
+ int elementIndex = i % ARRAY_SIZE;
+ markNullValue(columnIndex, arrayIndex, elementIndex);
+ deletedNumber++;
+ }
+ }
+ return deletedNumber;
}
protected void set(int index, long timestamp, int value) {
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java
index a1ce64c..b61bcf1 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java
@@ -68,7 +68,7 @@ public class IoTDBRawQueryWithValueFilterWithDeletionIT {
Statement statement = connection.createStatement()) {
// TODO currently aligned data in memory doesn't support deletion, so we flush all data to
// disk before doing deletion
- statement.execute("flush");
+ // statement.execute("flush");
statement.execute("delete timeseries root.sg1.d1.s2");
statement.execute("delete from root.sg1.d1.s1 where time <= 21");
} catch (Exception e) {