You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2025/08/12 08:33:37 UTC
(iotdb) branch addMemoryControlForModEntriesInQuery updated (52902afd59e -> 93c9afc531e)
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a change to branch addMemoryControlForModEntriesInQuery
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 52902afd59e modify compaction
new c7cedf051f5 fix bug
new 93c9afc531e optimize performance
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../queryengine/execution/MemoryEstimationHelper.java | 12 ++++++++++++
.../execution/fragment/FragmentInstanceContext.java | 9 +++++++--
.../db/queryengine/execution/fragment/QueryContext.java | 4 ++--
.../storageengine/dataregion/modification/ModEntry.java | 6 +++++-
.../dataregion/modification/ModificationFile.java | 4 ----
.../dataregion/modification/TreeDeletionEntry.java | 17 ++++++++++++++---
6 files changed, 40 insertions(+), 12 deletions(-)
(iotdb) 02/02: optimize performance
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch addMemoryControlForModEntriesInQuery
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93c9afc531e4feb821ebee0c2c5c5055d7632555
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Tue Aug 12 16:33:22 2025 +0800
optimize performance
---
.../queryengine/execution/MemoryEstimationHelper.java | 12 ++++++++++++
.../storageengine/dataregion/modification/ModEntry.java | 6 +++++-
.../dataregion/modification/ModificationFile.java | 4 ----
.../dataregion/modification/TreeDeletionEntry.java | 17 ++++++++++++++---
4 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
index 7c548877a78..fd33bb767e2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/MemoryEstimationHelper.java
@@ -102,6 +102,18 @@ public class MemoryEstimationHelper {
return totalSize;
}
+ public static long getEstimatedSizeOfPartialPathNodes(@Nullable final PartialPath partialPath) {
+ if (partialPath == null) {
+ return 0;
+ }
+ long totalSize = 0;
+ String[] nodes = partialPath.getNodes();
+ if (nodes != null && nodes.length > 0) {
+ totalSize += Arrays.stream(nodes).mapToLong(RamUsageEstimator::sizeOf).sum();
+ }
+ return totalSize;
+ }
+
// This method should only be called if the content in the current PartialPath comes from other
// structures whose memory cost have already been calculated.
public static long getEstimatedSizeOfCopiedPartialPath(@Nullable final PartialPath partialPath) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
index 349f1b43ec2..462a0a21c39 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModEntry.java
@@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.Accountable;
import org.apache.tsfile.utils.ReadWriteIOUtils;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -178,7 +179,10 @@ public abstract class ModEntry
}
public static ModType deserialize(InputStream stream) throws IOException {
- byte typeNum = ReadWriteIOUtils.readByte(stream);
+ int typeNum = stream.read();
+ if (typeNum == -1) {
+ throw new EOFException();
+ }
switch (typeNum) {
case 0x00:
return TABLE_DELETION;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
index e72b06b9a1a..bc0e86a18c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java
@@ -304,10 +304,6 @@ public class ModificationFile implements AutoCloseable {
}
if (nextEntry == null) {
try {
- if (inputStream.available() == 0) {
- close();
- return false;
- }
nextEntry = ModEntry.createFrom(inputStream);
} catch (EOFException e) {
close();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
index 3a755f0744f..c9fc8978f51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TreeDeletionEntry.java
@@ -23,12 +23,14 @@ import org.apache.iotdb.commons.path.AlignedPath;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternUtil;
+import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.tsfile.common.constant.TsFileConstant;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.common.TimeRange;
import org.apache.tsfile.utils.RamUsageEstimator;
@@ -108,7 +110,7 @@ public class TreeDeletionEntry extends ModEntry {
public void deserialize(InputStream stream) throws IOException {
super.deserialize(stream);
try {
- this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(stream));
+ this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(stream));
} catch (IllegalPathException e) {
throw new IOException(e);
}
@@ -118,12 +120,21 @@ public class TreeDeletionEntry extends ModEntry {
public void deserialize(ByteBuffer buffer) {
super.deserialize(buffer);
try {
- this.pathPattern = new MeasurementPath(ReadWriteIOUtils.readVarIntString(buffer));
+ this.pathPattern = getMeasurementPath(ReadWriteIOUtils.readVarIntString(buffer));
} catch (IllegalPathException e) {
throw new IllegalArgumentException(e);
}
}
+ private MeasurementPath getMeasurementPath(String path) throws IllegalPathException {
+ if (path.contains(TsFileConstant.BACK_QUOTE_STRING)) {
+ return new MeasurementPath(PathUtils.splitPathToDetachedNodes(path));
+ } else {
+ String[] nodes = path.split(TsFileConstant.PATH_SEPARATER_NO_REGEX);
+ return new MeasurementPath(nodes);
+ }
+ }
+
@Override
public boolean matches(PartialPath path) {
return pathPattern.matchFullPath(path);
@@ -235,6 +246,6 @@ public class TreeDeletionEntry extends ModEntry {
public long ramBytesUsed() {
return SHALLOW_SIZE
+ MemoryEstimationHelper.TIME_RANGE_INSTANCE_SIZE
- + MemoryEstimationHelper.getEstimatedSizeOfPartialPath(pathPattern);
+ + MemoryEstimationHelper.getEstimatedSizeOfPartialPathNodes(pathPattern);
}
}
(iotdb) 01/02: fix bug
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch addMemoryControlForModEntriesInQuery
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c7cedf051f581d036e5ac15cdf155b74b28c8931
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Tue Aug 12 12:33:00 2025 +0800
fix bug
---
.../queryengine/execution/fragment/FragmentInstanceContext.java | 9 +++++++--
.../iotdb/db/queryengine/execution/fragment/QueryContext.java | 4 ++--
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index e0944b17fc6..0126879d612 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -93,6 +93,8 @@ public class FragmentInstanceContext extends QueryContext {
// it will only be used once, after sharedQueryDataSource being inited, it will be set to null
private List<IFullPath> sourcePaths;
+ private boolean singleSourcePath = false;
+
// Used for region scan, relating methods are to be added.
private Map<IDeviceID, DeviceContext> devicePathsToContext;
@@ -485,6 +487,9 @@ public class FragmentInstanceContext extends QueryContext {
public void setSourcePaths(List<IFullPath> sourcePaths) {
this.sourcePaths = sourcePaths;
+ if (sourcePaths != null && sourcePaths.size() == 1) {
+ singleSourcePath = true;
+ }
}
public void setDevicePathsToContext(Map<IDeviceID, DeviceContext> devicePathsToContext) {
@@ -969,7 +974,7 @@ public class FragmentInstanceContext extends QueryContext {
return ignoreNotExistsDevice;
}
- public List<IFullPath> getSourcePaths() {
- return sourcePaths;
+ public boolean isSingleSourcePath() {
+ return singleSourcePath;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
index 52233a52789..4769ebed197 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java
@@ -108,7 +108,7 @@ public class QueryContext {
resource.getTsFileID(), k -> loadAllModificationsFromDisk(resource));
}
FragmentInstanceContext fragmentInstanceContext = (FragmentInstanceContext) this;
- if (fragmentInstanceContext.getSourcePaths().size() == 1) {
+ if (fragmentInstanceContext.isSingleSourcePath()) {
return loadAllModificationsFromDisk(resource);
}
@@ -199,7 +199,7 @@ public class QueryContext {
if (fileModEntries == null) {
return Collections.emptyList();
}
- List<ModEntry> modEntries = fileModEntries.getOverlapped(new PartialPath(deviceID));
+ List<ModEntry> modEntries = fileModEntries.getDeviceOverlapped(new PartialPath(deviceID));
if (deviceID.isTableModel()) {
// the pattern tree has false-positive for table model deletion, so we do a further
// filtering