You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/05/16 07:26:57 UTC
[iotdb] branch opti_mem_check_for_aligned_records created (now 16c8cce7ea)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch opti_mem_check_for_aligned_records
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 16c8cce7ea Optimize memory check for insertAligned_records
This branch includes the following new commits:
new 16c8cce7ea Optimize memory check for insertAligned_records
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Optimize memory check for insertAligned_records
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch opti_mem_check_for_aligned_records
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 16c8cce7ea18d5aa963d6e90beff4a2f2a3b0680
Author: HTHou <hh...@outlook.com>
AuthorDate: Tue May 16 15:26:29 2023 +0800
Optimize memory check for insertAligned_records
---
.../engine/memtable/AlignedWritableMemChunk.java | 5 +--
.../db/engine/storagegroup/TsFileProcessor.java | 48 +++++++++++++---------
2 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
index dabd3b472d..d947b6a034 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java
@@ -41,7 +41,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -60,7 +59,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
private static final Logger LOGGER = LoggerFactory.getLogger(AlignedWritableMemChunk.class);
public AlignedWritableMemChunk(List<IMeasurementSchema> schemaList) {
- this.measurementIndexMap = new LinkedHashMap<>();
+ this.measurementIndexMap = new HashMap<>();
List<TSDataType> dataTypeList = new ArrayList<>();
this.schemaList = schemaList;
for (int i = 0; i < schemaList.size(); i++) {
@@ -71,7 +70,7 @@ public class AlignedWritableMemChunk implements IWritableMemChunk {
}
private AlignedWritableMemChunk(List<IMeasurementSchema> schemaList, AlignedTVList list) {
- this.measurementIndexMap = new LinkedHashMap<>();
+ this.measurementIndexMap = new HashMap<>();
this.schemaList = schemaList;
for (int i = 0; i < schemaList.size(); i++) {
measurementIndexMap.put(schemaList.get(i).getMeasurementId(), i);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index f4f33c1047..21634f782d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -37,6 +37,7 @@ import org.apache.iotdb.db.engine.flush.NotifyFlushMemTable;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk;
import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunkGroup;
import org.apache.iotdb.db.engine.memtable.IMemTable;
+import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
@@ -453,14 +454,26 @@ public class TsFileProcessor {
AlignedWritableMemChunk alignedMemChunk = null;
// get device id
IDeviceID deviceID = getDeviceID(deviceId);
+ IWritableMemChunkGroup memChunkGroup = workMemTable.getMemTableMap().get(deviceID);
- if (workMemTable.checkIfChunkDoesNotExist(deviceID, AlignedPath.VECTOR_PLACEHOLDER)) {
+ if (memChunkGroup == null) {
// ChunkMetadataIncrement
chunkMetadataIncrement +=
ChunkMetadata.calculateRamSize(AlignedPath.VECTOR_PLACEHOLDER, TSDataType.VECTOR)
* dataTypes.length;
memTableIncrement += AlignedTVList.alignedTvListArrayMemCost(dataTypes);
+ for (int i = 0; i < dataTypes.length; i++) {
+ // skip failed Measurements
+ if (dataTypes[i] == null || measurements[i] == null) {
+ continue;
+ }
+ // TEXT data mem size
+ if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
+ }
} else {
+ alignedMemChunk = ((AlignedWritableMemChunkGroup) memChunkGroup).getAlignedMemChunk();
// here currentChunkPointNum >= 1
long currentChunkPointNum =
workMemTable.getCurrentTVListSize(deviceID, AlignedPath.VECTOR_PLACEHOLDER);
@@ -468,24 +481,21 @@ public class TsFileProcessor {
(currentChunkPointNum % PrimitiveArrayManager.ARRAY_SIZE) == 0
? AlignedTVList.alignedTvListArrayMemCost(dataTypes)
: 0;
- alignedMemChunk =
- ((AlignedWritableMemChunkGroup) workMemTable.getMemTableMap().get(deviceID))
- .getAlignedMemChunk();
- }
- for (int i = 0; i < dataTypes.length; i++) {
- // skip failed Measurements
- if (dataTypes[i] == null || measurements[i] == null) {
- continue;
- }
- // extending the column of aligned mem chunk
- if (alignedMemChunk != null && !alignedMemChunk.containsMeasurement(measurements[i])) {
- memTableIncrement +=
- (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
- * dataTypes[i].getDataTypeSize();
- }
- // TEXT data mem size
- if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
- textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ for (int i = 0; i < dataTypes.length; i++) {
+ // skip failed Measurements
+ if (dataTypes[i] == null || measurements[i] == null) {
+ continue;
+ }
+ // extending the column of aligned mem chunk
+ if (!alignedMemChunk.containsMeasurement(measurements[i])) {
+ memTableIncrement +=
+ (alignedMemChunk.alignedListSize() / PrimitiveArrayManager.ARRAY_SIZE + 1)
+ * dataTypes[i].getDataTypeSize();
+ }
+ // TEXT data mem size
+ if (dataTypes[i] == TSDataType.TEXT && values[i] != null) {
+ textDataIncrement += MemUtils.getBinarySize((Binary) values[i]);
+ }
}
}
updateMemoryInfo(memTableIncrement, chunkMetadataIncrement, textDataIncrement);