You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2021/11/25 08:02:23 UTC
[iotdb] branch master updated: [IOTDB-2056] Fix totalPointsNumThreshold calculation error causes too many tsfile flushed (#4461)
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a3cd374 [IOTDB-2056] Fix totalPointsNumThreshold calculation error causes too many tsfile flushed (#4461)
a3cd374 is described below
commit a3cd3743c375858c43567c5d43b191f428595709
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Nov 25 16:01:49 2021 +0800
[IOTDB-2056] Fix totalPointsNumThreshold calculation error causes too many tsfile flushed (#4461)
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 12 ++----
.../db/engine/memtable/WritableMemChunkGroup.java | 10 +----
.../db/engine/memtable/PrimitiveMemTableTest.java | 44 ++++++++++++++++++++++
3 files changed, 49 insertions(+), 17 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index 7844383..e834529 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -92,13 +92,7 @@ public abstract class AbstractMemTable implements IMemTable {
private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet(
String deviceId, List<IMeasurementSchema> schemaList) {
IWritableMemChunkGroup memChunkGroup =
- memTableMap.computeIfAbsent(
- deviceId,
- k -> {
- seriesNumber++;
- totalPointsNumThreshold += avgSeriesPointNumThreshold;
- return new WritableMemChunkGroup(schemaList);
- });
+ memTableMap.computeIfAbsent(deviceId, k -> new WritableMemChunkGroup());
for (IMeasurementSchema schema : schemaList) {
if (!memChunkGroup.contains(schema.getMeasurementId())) {
seriesNumber++;
@@ -114,8 +108,8 @@ public abstract class AbstractMemTable implements IMemTable {
memTableMap.computeIfAbsent(
deviceId,
k -> {
- seriesNumber++;
- totalPointsNumThreshold += avgSeriesPointNumThreshold;
+ seriesNumber += schemaList.size();
+ totalPointsNumThreshold += avgSeriesPointNumThreshold * schemaList.size();
return new AlignedWritableMemChunkGroup(schemaList);
});
for (IMeasurementSchema schema : schemaList) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
index 2ca32ce..fd6c408 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
@@ -30,11 +30,8 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
private Map<String, IWritableMemChunk> memChunkMap;
- public WritableMemChunkGroup(List<IMeasurementSchema> schemaList) {
+ public WritableMemChunkGroup() {
memChunkMap = new HashMap<>();
- for (IMeasurementSchema schema : schemaList) {
- createMemChunkIfNotExistAndGet(schema);
- }
}
@Override
@@ -65,10 +62,7 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
private IWritableMemChunk createMemChunkIfNotExistAndGet(IMeasurementSchema schema) {
return memChunkMap.computeIfAbsent(
- schema.getMeasurementId(),
- k -> {
- return new WritableMemChunk(schema);
- });
+ schema.getMeasurementId(), k -> new WritableMemChunk(schema));
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
index 6be17a0..a6f61e8 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
@@ -152,6 +152,50 @@ public class PrimitiveMemTableTest {
}
}
+ @Test
+ public void totalSeriesNumberTest() throws IOException, QueryProcessException, MetadataException {
+ IMemTable memTable = new PrimitiveMemTable();
+ int count = 10;
+ String deviceId = "d1";
+ String[] measurementId = new String[count];
+ for (int i = 0; i < measurementId.length; i++) {
+ measurementId[i] = "s" + i;
+ }
+ List<IMeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(
+ new UnaryMeasurementSchema(measurementId[0], TSDataType.INT32, TSEncoding.PLAIN));
+ schemaList.add(
+ new UnaryMeasurementSchema(measurementId[1], TSDataType.INT32, TSEncoding.PLAIN));
+ int dataSize = 10000;
+ for (int i = 0; i < dataSize; i++) {
+ memTable.write(
+ deviceId,
+ Collections.singletonList(
+ new UnaryMeasurementSchema(measurementId[0], TSDataType.INT32, TSEncoding.PLAIN)),
+ i,
+ new Object[] {i});
+ }
+ deviceId = "d2";
+ for (int i = 0; i < dataSize; i++) {
+ memTable.write(deviceId, schemaList, i, new Object[] {i, i});
+ }
+ Assert.assertEquals(3, memTable.getSeriesNumber());
+ // aligned
+ deviceId = "d3";
+
+ for (int i = 0; i < dataSize; i++) {
+ memTable.writeAlignedRow(deviceId, schemaList, i, new Object[] {i, i});
+ }
+ Assert.assertEquals(5, memTable.getSeriesNumber());
+ memTable.writeAlignedRow(
+ deviceId,
+ Collections.singletonList(
+ new UnaryMeasurementSchema(measurementId[2], TSDataType.INT32, TSEncoding.PLAIN)),
+ 0,
+ new Object[] {0});
+ Assert.assertEquals(6, memTable.getSeriesNumber());
+ }
+
private void write(
IMemTable memTable,
String deviceId,