You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2025/08/12 08:33:39 UTC
(iotdb) 02/02: optimize performance
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch addMemoryControlForModEntriesInQuery
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93c9afc531e4feb821ebee0c2c5c5055d7632555
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Tue Aug 12 16:33:22 2025 +0800
optimize performance
---
.../queryengine/execution/MemoryEstimationHelper.java | 12 ++++++++++++
.../storageengine/dataregion/modification/ModEntry.java | 6 +++++-
.../dataregion/modification/ModificationFile.java | 4 ----
.../dataregion/modification/TreeDeletionEntry.java | 17 ++++++++++++++---
4 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
index 7c548877a78..fd33bb767e2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
@@ -102,6 +102,18 @@ public class MemoryEstimationHelper {
return totalSize;
}
+ public static long getEstimatedSizeOfPartialPathNodes(@Nullable final PartialPath partialPath) {
+ if (partialPath == null) {
+ return 0;
+ }
+ long totalSize = 0;
+ String[] nodes = partialPath.getNodes();
+ if (nodes != null && nodes.length > 0) {
+ totalSize += Arrays.stream(nodes).mapToLong(RamUsageEstimator::sizeOf).sum();
+ }
+ return totalSize;
+ }
+
// This method should only be called if the content in the current PartialPath comes from other
// structures whose memory cost have already been calculated.
public static long getEstimatedSizeOfCopiedPartialPath(@Nullable final PartialPath partialPath) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
index 349f1b43ec2..462a0a21c39 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
@@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.Accountable;
import org.apache.tsfile.utils.ReadWriteIOUtils;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -178,7 +179,10 @@ public abstract class ModEntry
}
public static ModType deserialize(InputStream stream) throws IOException {
- byte typeNum = ReadWriteIOUtils.readByte(stream);
+ int typeNum = stream.read();
+ if (typeNum == -1) {
+ throw new EOFException();
+ }
switch (typeNum) {
case 0x00:
return TABLE_DELETION;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
index e72b06b9a1a..bc0e86a18c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
@@ -304,10 +304,6 @@ public class ModificationFile implements AutoCloseable {
}
if (nextEntry == null) {
try {
- if (inputStream.available() == 0) {
- close();
- return false;
- }
nextEntry = ModEntry.createFrom(inputStream);
} catch (EOFException e) {
close();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
index 3a755f0744f..c9fc8978f51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
@@ -23,12 +23,14 @@ import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternUtil;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.RamUsageEstimator;
@@ -108,7 +110,7 @@ public class TreeDeletionEntry extends ModEntry {
public void deserialize(InputStream stream) throws IOException {
super.deserialize(stream);
try {
- this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(stream));
+ this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(stream));
} catch (IllegalPathException e) {
throw new IOException(e);
}
@@ -118,12 +120,21 @@ public class TreeDeletionEntry extends ModEntry {
public void deserialize(ByteBuffer buffer) {
super.deserialize(buffer);
try {
- this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(buffer));
+ this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(buffer));
} catch (IllegalPathException e) {
throw new IllegalArgumentException(e);
}
}
+ private MeasurementPath getMeasurementPath(String path) throws IllegalPathException {
+ if (path.contains(TsFileConstant.BACK_QUOTE_STRING)) {
+ return new MeasurementPath(PathUtils.splitPathToDetachedNodes(path));
+ } else {
+ String[] nodes = path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
+ return new MeasurementPath(nodes);
+ }
+ }
+
@Override
public boolean matches(PartialPath path) {
return pathPattern.matchFullPath(path);
@@ -235,6 +246,6 @@ public class TreeDeletionEntry extends ModEntry {
public long ramBytesUsed() {
return SHALLOW_SIZE
+ MemoryEstimationHelper.TIME_RANGE_INSTANCE_SIZE
- + MemoryEstimationHelper.getEstimatedSizeOfPartialPath(pathPattern);
+ + MemoryEstimationHelper.getEstimatedSizeOfPartialPathNodes(pathPattern);
}
}