You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/01/20 02:59:28 UTC
[iotdb] branch aligned_prof created (now 717293f)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch aligned_prof
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 717293f [IOTDB-2433] Fix aligned timeseries mem control bug 2
This branch includes the following new commits:
new 717293f [IOTDB-2433] Fix aligned timeseries mem control bug 2
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-2433] Fix aligned timeseries mem control bug 2
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch aligned_prof
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 717293ffb8efaa0653e7a593513d25cdf366e33a
Author: HTHou <hh...@outlook.com>
AuthorDate: Thu Jan 20 10:58:26 2022 +0800
[IOTDB-2433] Fix aligned timeseries mem control bug 2
---
.../iotdb/db/engine/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedWritableMemChunkGroup.java | 4 +-
.../apache/iotdb/db/engine/memtable/IMemTable.java | 2 +-
.../db/engine/memtable/IWritableMemChunkGroup.java | 2 +-
.../db/engine/memtable/WritableMemChunkGroup.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 8 +-
.../engine/storagegroup/TsFileProcessorTest.java | 97 ++++++++++++++--------
7 files changed, 73 insertions(+), 48 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
index e491a87..57116e9 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
@@ -349,9 +349,9 @@ public abstract class AbstractMemTable implements IMemTable {
}
@Override
- public long getCurrentChunkPointNum(IDeviceID deviceId, String measurement) {
+ public long getCurrentTVListSize(IDeviceID deviceId, String measurement) {
IWritableMemChunkGroup memChunkGroup = memTableMap.get(deviceId);
- return memChunkGroup.getCurrentChunkPointNum(measurement);
+ return memChunkGroup.getCurrentTVListSize(measurement);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
index 795f9c0..7b6589b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java
@@ -110,8 +110,8 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
- public long getCurrentChunkPointNum(String measurement) {
- return memChunk.count();
+ public long getCurrentTVListSize(String measurement) {
+ return memChunk.getTVList().size();
}
public AlignedWritableMemChunk getAlignedMemChunk() {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
index 3e2a515..69aeb47 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
@@ -155,7 +155,7 @@ public interface IMemTable {
boolean checkIfChunkDoesNotExist(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
- long getCurrentChunkPointNum(IDeviceID deviceId, String measurement);
+ long getCurrentTVListSize(IDeviceID deviceId, String measurement);
/** only used when mem control enabled */
void addTextDataSize(long textDataIncrement);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
index 00bcf7c..0ee7075 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java
@@ -49,5 +49,5 @@ public interface IWritableMemChunkGroup {
int delete(
PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp);
- long getCurrentChunkPointNum(String measurement);
+ long getCurrentTVListSize(String measurement);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
index 578a346..743eaa2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java
@@ -131,7 +131,7 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup {
}
@Override
- public long getCurrentChunkPointNum(String measurement) {
- return memChunkMap.get(measurement).count();
+ public long getCurrentTVListSize(String measurement) {
+ return memChunkMap.get(measurement).getTVList().size();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 0f0d827..71eaef1 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -369,7 +369,7 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceID, insertRowPlan.getMeasurements()[i]);
+ workMemTable.getCurrentTVListSize(deviceID, insertRowPlan.getMeasurements()[i]);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? TVList.tvListArrayMemCost(insertRowPlan.getDataTypes()[i])
@@ -409,7 +409,7 @@ public class TsFileProcessor {
} else {
// here currentChunkPointNum >= 1
long currentChunkPointNum =
- workMemTable.getCurrentChunkPointNum(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
+ workMemTable.getCurrentTVListSize(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
memTableIncrement +=
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? AlignedTVList.alignedTvListArrayMemCost(insertRowPlan.getDataTypes())
@@ -518,7 +518,7 @@ public class TsFileProcessor {
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
* TVList.tvListArrayMemCost(dataType);
} else {
- long currentChunkPointNum = workMemTable.getCurrentChunkPointNum(deviceId, measurement);
+ long currentChunkPointNum = workMemTable.getCurrentTVListSize(deviceId, measurement);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
@@ -558,7 +558,7 @@ public class TsFileProcessor {
* AlignedTVList.alignedTvListArrayMemCost(dataTypes);
} else {
int currentChunkPointNum =
- (int) workMemTable.getCurrentChunkPointNum(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
+ (int) workMemTable.getCurrentTVListSize(deviceId, AlignedPath.VECTOR_PLACEHOLDER);
if (currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE == 0) {
memIncrements[0] +=
((end - start) / PrimitiveArrayManager.ARRAY_SIZE + 1)
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
index 444de28..2341b47 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorTest.java
@@ -302,11 +302,41 @@ public class TsFileProcessorTest {
processor.setTsFileProcessorInfo(tsFileProcessorInfo);
this.sgInfo.initTsFileProcessorInfo(processor);
SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
- processor.insertTablet(genInsertTablePlan(0), 0, 100, new TSStatus[100]);
+ processor.insertTablet(genInsertTablePlan(0, true), 0, 10, new TSStatus[10]);
IMemTable memTable = processor.getWorkMemTable();
- Assert.assertEquals(3008, memTable.getTVListsRamCost());
- processor.insertTablet(genInsertTablePlan(100), 0, 100, new TSStatus[100]);
- Assert.assertEquals(5264, memTable.getTVListsRamCost());
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(100, true), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(200, true), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(828424, memTable.getTVListsRamCost());
+ Assert.assertEquals(90000, memTable.getTotalPointsNum());
+ Assert.assertEquals(720360, memTable.memSize());
+ }
+
+ @Test
+ public void nonAlignedTvListRamCostTest()
+ throws MetadataException, WriteProcessException, IOException {
+ processor =
+ new TsFileProcessor(
+ storageGroup,
+ SystemFileFactory.INSTANCE.getFile(filePath),
+ sgInfo,
+ this::closeTsFileProcessor,
+ (tsFileProcessor) -> true,
+ true);
+ TsFileProcessorInfo tsFileProcessorInfo = new TsFileProcessorInfo(sgInfo);
+ processor.setTsFileProcessorInfo(tsFileProcessorInfo);
+ this.sgInfo.initTsFileProcessorInfo(processor);
+ SystemInfo.getInstance().reportStorageGroupStatus(sgInfo, processor);
+ processor.insertTablet(genInsertTablePlan(0, false), 0, 10, new TSStatus[10]);
+ IMemTable memTable = processor.getWorkMemTable();
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(100, false), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ processor.insertTablet(genInsertTablePlan(200, false), 0, 10, new TSStatus[10]);
+ Assert.assertEquals(1656000, memTable.getTVListsRamCost());
+ Assert.assertEquals(90000, memTable.getTotalPointsNum());
+ Assert.assertEquals(1440000, memTable.memSize());
}
@Test
@@ -384,50 +414,45 @@ public class TsFileProcessorTest {
}
}
- private InsertTabletPlan genInsertTablePlan(long startTime) throws IllegalPathException {
- String[] measurements = new String[2];
- measurements[0] = "sensor0";
- measurements[1] = "sensor1";
-
- List<Integer> dataTypesList = new ArrayList<>();
- TSDataType[] dataTypes = new TSDataType[2];
- dataTypesList.add(TSDataType.BOOLEAN.ordinal());
- dataTypesList.add(TSDataType.INT64.ordinal());
- dataTypes[0] = TSDataType.BOOLEAN;
- dataTypes[1] = TSDataType.INT64;
-
- TSEncoding[] encodings = new TSEncoding[2];
- encodings[0] = TSEncoding.PLAIN;
- encodings[1] = TSEncoding.GORILLA;
-
+ private InsertTabletPlan genInsertTablePlan(long startTime, boolean isAligned)
+ throws IllegalPathException {
String deviceId = "root.sg.device5";
-
- IMeasurementMNode[] mNodes = new IMeasurementMNode[2];
- IMeasurementSchema schema0 = new MeasurementSchema(measurements[0], dataTypes[0], encodings[0]);
- IMeasurementSchema schema1 = new MeasurementSchema(measurements[1], dataTypes[1], encodings[1]);
- mNodes[0] = MeasurementMNode.getMeasurementMNode(null, "sensor0", schema0, null);
- mNodes[1] = MeasurementMNode.getMeasurementMNode(null, "sensor1", schema1, null);
-
+ String[] measurements = new String[3000];
+ List<Integer> dataTypesList = new ArrayList<>();
+ TSDataType[] dataTypes = new TSDataType[3000];
+ TSEncoding[] encodings = new TSEncoding[3000];
+ IMeasurementMNode[] mNodes = new IMeasurementMNode[3000];
+ for (int i = 0; i < 3000; i++) {
+ measurements[i] = "s" + i;
+ dataTypesList.add(TSDataType.INT64.ordinal());
+ dataTypes[i] = TSDataType.INT64;
+ encodings[i] = TSEncoding.PLAIN;
+ IMeasurementSchema schema =
+ new MeasurementSchema(measurements[i], dataTypes[i], encodings[i]);
+ mNodes[i] = MeasurementMNode.getMeasurementMNode(null, measurements[i], schema, null);
+ }
InsertTabletPlan insertTabletPlan =
new InsertTabletPlan(new PartialPath(deviceId), measurements, dataTypesList);
- long[] times = new long[100];
- Object[] columns = new Object[2];
- columns[0] = new boolean[100];
- columns[1] = new long[100];
+ long[] times = new long[10];
+ Object[] columns = new Object[3000];
+ for (int i = 0; i < 3000; i++) {
+ columns[i] = new long[10];
+ }
- for (long r = 0; r < 100; r++) {
+ for (long r = 0; r < 10; r++) {
times[(int) r] = r + startTime;
- ((boolean[]) columns[0])[(int) r] = false;
- ((long[]) columns[1])[(int) r] = r;
+ for (int i = 0; i < 3000; i++) {
+ ((long[]) columns[i])[(int) r] = r;
+ }
}
insertTabletPlan.setTimes(times);
insertTabletPlan.setColumns(columns);
insertTabletPlan.setRowCount(times.length);
insertTabletPlan.setMeasurementMNodes(mNodes);
insertTabletPlan.setStart(0);
- insertTabletPlan.setEnd(100);
- insertTabletPlan.setAligned(true);
+ insertTabletPlan.setEnd(10);
+ insertTabletPlan.setAligned(isAligned);
return insertTabletPlan;
}