You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/01/20 02:59:29 UTC
[iotdb] 01/01: [IOTDB-2433] Fix aligned timeseries mem control bug 2
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch aligned_prof
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 717293ffb8efaa0653e7a593513d25cdf366e33a
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jan 20 10:58:26 2022 +0800
[IOTDB-2433] Fix aligned timeseries mem control bug 2
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedWritableMemChunkGroup.java | 4 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 2 +-
.../db/engine/memtable/IWritableMemChunkGroup.java | 2 +-
.../db/engine/memtable/WritableMemChunkGroup.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 8 +-
.../engine/storagegroup/TsFileProcessorTest.java | 97 ++++++++++++++--------
7 files changed, 73 insertions(+), 48 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index e491a87..57116e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -349,9 +349,9 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public long getCurrentChunkPointNum(IDeviceID deviceId, String measurement) {
+ public long getCurrentTVListSize(IDeviceID deviceId, String measurement) {
IWritableMemChunkGroup memChunkGroup = memTableMap.get(deviceId);
- return memChunkGroup.getCurrentChunkPointNum(measurement);
+ return memChunkGroup.getCurrentTVListSize(measurement);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
index 795f9c0..7b6589b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
@@ -110,8 +110,8 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
- public long getCurrentChunkPointNum(String measurement) {
- return memChunk.count();
+ public long getCurrentTVListSize(String measurement) {
+ return memChunk.getTVList().size();
}
public AlignedWritableMemChunk getAlignedMemChunk() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index 3e2a515..69aeb47 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -155,7 +155,7 @@ public interface IMemTable {
boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
- long getCurrentChunkPointNum(IDeviceID deviceId, String measurement);
+ long getCurrentTVListSize(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
void addTextDataSize(long textDataIncrement);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
index 00bcf7c..0ee7075 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
@@ -49,5 +49,5 @@ public interface IWritableMemChunkGroup {
int delete(
PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp);
- long getCurrentChunkPointNum(String measurement);
+ long getCurrentTVListSize(String measurement);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
index 578a346..743eaa2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
@@ -131,7 +131,7 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
- public long getCurrentChunkPointNum(String measurement) {
- return memChunkMap.get(measurement).count();
+ public long getCurrentTVListSize(String measurement) {
+ return memChunkMap.get(measurement).getTVList().size();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 0f0d827..71eaef1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -369,7 +369,7 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceID, insertRowPlan.getMeasurements()[i]);
+ workMemTable.getCurrentTVListSize(deviceID, insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i])
@@ -409,7 +409,7 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
+ workMemTable.getCurrentTVListSize(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes())
@@ -518,7 +518,7 @@ public class TsFileProcessor {
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
* TVList.tvListArrayMemCost(dataType);
} else {
- long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, measurement);
+ long currentChunkPointNum = workMemTable.getCurrentTVListSize(deviceId, measurement);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
@@ -558,7 +558,7 @@ public class TsFileProcessor {
* AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int currentChunkPointNum =
- (int) workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
+ (int) workMemTable.getCurrentTVListSize(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index 444de28..2341b47 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -302,11 +302,41 @@ public class TsFileProcessorTest {
processor.setTsFileProcessorInfo(tsFileProcessorInfo);
this.sgInfo.initTsFileProcessorInfo(processor);
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
- processor.insertTablet(genInsertTablePlan(0), 0, 100, new TSStatus[100]);
+ processor.insertTablet(genInsertTablePlan(0, true), 0, 10, new TSStatus[10]);
IMemTable memTable = processor.getWorkMemTable();
- Assert.assertEquals(3008, memTable.getTVListsRamCost());
- processor.insertTablet(genInsertTablePlan(100), 0, 100, new TSStatus[100]);
- Assert.assertEquals(5264, memTable.getTVListsRamCost());
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(100, true), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(200, true), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ Assert.assertEquals(90000, memTable.getTotalPointsNum());
+ Assert.assertEquals(720360, memTable.memSize());
+ }
+
+ @Test
+ public void nonAlignedTvListRamCostTest()
+ throws MetadataException, WriteProcessException, IOException {
+ processor =
+ new TsFileProcessor(
+ storageGroup,
+ SystemFileFactory.INSTANCE.getFile(filePath),
+ sgInfo,
+ this::closeTsFileProcessor,
+ (tsFileProcessor) -> true,
+ true);
+ TsFileProcessorInfo tsFileProcessorInfo = new TsFileProcessorInfo(sgInfo);
+ processor.setTsFileProcessorInfo(tsFileProcessorInfo);
+ this.sgInfo.initTsFileProcessorInfo(processor);
+ SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
+ processor.insertTablet(genInsertTablePlan(0, false), 0, 10, new TSStatus[10]);
+ IMemTable memTable = processor.getWorkMemTable();
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(100, false), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(200, false), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ Assert.assertEquals(90000, memTable.getTotalPointsNum());
+ Assert.assertEquals(1440000, memTable.memSize());
}
@Test
@@ -384,50 +414,45 @@ public class TsFileProcessorTest {
}
}
- private InsertTabletPlan genInsertTablePlan(long startTime) throws IllegalPathException {
- String[] measurements = new String[2];
- measurements[0] = "sensor0";
- measurements[1] = "sensor1";
-
- List<Integer> dataTypesList = new ArrayList<>();
- TSDataType[] dataTypes = new TSDataType[2];
- dataTypesList.add(TSDataType.BOOLEAN.ordinal());
- dataTypesList.add(TSDataType.INT64.ordinal());
- dataTypes[0] = TSDataType.BOOLEAN;
- dataTypes[1] = TSDataType.INT64;
-
- TSEncoding[] encodings = new TSEncoding[2];
- encodings[0] = TSEncoding.PLAIN;
- encodings[1] = TSEncoding.GORILLA;
-
+ private InsertTabletPlan genInsertTablePlan(long startTime, boolean isAligned)
+ throws IllegalPathException {
String deviceId = "root.sg.device5";
-
- IMeasurementMNode[] mNodes = new IMeasurementMNode[2];
- IMeasurementSchema schema0 = new MeasurementSchema(measurements[0], dataTypes[0], encodings[0]);
- IMeasurementSchema schema1 = new MeasurementSchema(measurements[1], dataTypes[1], encodings[1]);
- mNodes[0] = MeasurementMNode.getMeasurementMNode(null, "sensor0", schema0, null);
- mNodes[1] = MeasurementMNode.getMeasurementMNode(null, "sensor1", schema1, null);
-
+ String[] measurements = new String[3000];
+ List<Integer> dataTypesList = new ArrayList<>();
+ TSDataType[] dataTypes = new TSDataType[3000];
+ TSEncoding[] encodings = new TSEncoding[3000];
+ IMeasurementMNode[] mNodes = new IMeasurementMNode[3000];
+ for (int i = 0; i < 3000; i++) {
+ measurements[i] = "s" + i;
+ dataTypesList.add(TSDataType.INT64.ordinal());
+ dataTypes[i] = TSDataType.INT64;
+ encodings[i] = TSEncoding.PLAIN;
+ IMeasurementSchema schema =
+ new MeasurementSchema(measurements[i], dataTypes[i], encodings[i]);
+ mNodes[i] = MeasurementMNode.getMeasurementMNode(null, measurements[i], schema, null);
+ }
InsertTabletPlan insertTabletPlan =
new InsertTabletPlan(new PartialPath(deviceId), measurements, dataTypesList);
- long[] times = new long[100];
- Object[] columns = new Object[2];
- columns[0] = new boolean[100];
- columns[1] = new long[100];
+ long[] times = new long[10];
+ Object[] columns = new Object[3000];
+ for (int i = 0; i < 3000; i++) {
+ columns[i] = new long[10];
+ }
- for (long r = 0; r < 100; r++) {
+ for (long r = 0; r < 10; r++) {
times[(int) r] = r + startTime;
- ((boolean[]) columns[0])[(int) r] = false;
- ((long[]) columns[1])[(int) r] = r;
+ for (int i = 0; i < 3000; i++) {
+ ((long[]) columns[i])[(int) r] = r;
+ }
}
insertTabletPlan.setTimes(times);
insertTabletPlan.setColumns(columns);
insertTabletPlan.setRowCount(times.length);
insertTabletPlan.setMeasurementMNodes(mNodes);
insertTabletPlan.setStart(0);
- insertTabletPlan.setEnd(100);
- insertTabletPlan.setAligned(true);
+ insertTabletPlan.setEnd(10);
+ insertTabletPlan.setAligned(isAligned);
return insertTabletPlan;
}