You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/05/16 07:26:58 UTC
[iotdb] 01/01: Optimize memory check for insertAligned_records
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch opti_mem_check_for_aligned_records
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 16c8cce7ea18d5aa963d6e90beff4a2f2a3b0680
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue May 16 15:26:29 2023 +0800
Optimize memory check for insertAligned_records
---
.../engine/memtable/AlignedWritableMemChunk.java | 5 +--
.../db/engine/storagegroup/TsFileProcessor.java | 48 +++++++++++++---------
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index dabd3b472d..d947b6a034 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -41,7 +41,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -60,7 +59,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
private static final Logger LOGGER = LoggerFactory.getLogger(AlignedWritableMemChunk.class);
public AlignedWritableMemChunk(List<IMeasurementSchema> schemaList) {
- this.measurementIndexMap = new LinkedHashMap<>();
+ this.measurementIndexMap = new HashMap<>();
List<TSDataType> dataTypeList = new ArrayList<>();
this.schemaList = schemaList;
for (int i = 0; i < schemaList.size(); i++) {
@@ -71,7 +70,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
private AlignedWritableMemChunk(List<IMeasurementSchema> schemaList, AlignedTVList list) {
- this.measurementIndexMap = new LinkedHashMap<>();
+ this.measurementIndexMap = new HashMap<>();
this.schemaList = schemaList;
for (int i = 0; i < schemaList.size(); i++) {
measurementIndexMap.put(schemaList.get(i).getMeasurementId(), i);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index f4f33c1047..21634f782d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunkGroup;
import org.apache.iotdb.db.engine.memtable.IMemTable;
+import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -453,14 +454,26 @@ public class TsFileProcessor {
AlignedWritableMemChunk alignedMemChunk = null;
// get device id
IDeviceID deviceID = getDeviceID(deviceId);
+ IWritableMemChunkGroup memChunkGroup = workMemTable.getMemTableMap().get(deviceID);
- if (workMemTable.checkIfChunkDoesNotExist(deviceID, AlignedPath.VECTOR_PLACEHOLDER)) {
+ if (memChunkGroup == null) {
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* dataTypes.length;
memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes);
+ for (int i = 0; i < dataTypes.length; i++) {
+ // skip failed Measurements
+ if (dataTypes[i] == null || measurements[i] == null) {
+ continue;
+ }
+ // TEXT data mem size
+ if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
+ }
} else {
+ alignedMemChunk = ((AlignedWritableMemChunkGroup) memChunkGroup).getAlignedMemChunk();
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentTVListSize(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
@@ -468,24 +481,21 @@ public class TsFileProcessor {
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? AlignedTVList.alignedTvListArrayMemCost(dataTypes)
: 0;
- alignedMemChunk =
- ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceID))
- .getAlignedMemChunk();
- }
- for (int i = 0; i < dataTypes.length; i++) {
- // skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
- continue;
- }
- // extending the column of aligned mem chunk
- if (alignedMemChunk != null && !alignedMemChunk.containsMeasurement(measurements[i])) {
- memTableIncrement +=
- (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * dataTypes[i].getDataTypeSize();
- }
- // TEXT data mem size
- if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ for (int i = 0; i < dataTypes.length; i++) {
+ // skip failed Measurements
+ if (dataTypes[i] == null || measurements[i] == null) {
+ continue;
+ }
+ // extending the column of aligned mem chunk
+ if (!alignedMemChunk.containsMeasurement(measurements[i])) {
+ memTableIncrement +=
+ (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
+ * dataTypes[i].getDataTypeSize();
+ }
+ // TEXT data mem size
+ if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
}
}
updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement);