You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/01/11 09:52:19 UTC
[iotdb] branch lmh/FixAggrMem created (now b02771471a)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch lmh/FixAggrMem
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at b02771471a modify mem calc
This branch includes the following new commits:
new b02771471a modify mem calc
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: modify mem calc
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/FixAggrMem
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b02771471a1c48f6a9a5fe224ecc922f62797fdb
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Jan 11 17:51:18 2023 +0800
modify mem calc
---
.../AbstractSeriesAggregationScanOperator.java | 8 ++---
.../mpp/execution/operator/OperatorMemoryTest.java | 35 ++++++++++------------
2 files changed, 19 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index ee64166a4b..122247c85b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -64,7 +64,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
protected boolean finished = false;
- private final long maxRetainedSize;
+ private final long cachedRawDataSize;
private final long maxReturnSize;
protected AbstractSeriesAggregationScanOperator(
@@ -92,14 +92,14 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
}
this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
- this.maxRetainedSize =
+ this.cachedRawDataSize =
(1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
this.maxReturnSize = maxReturnSize;
}
@Override
public long calculateMaxPeekMemory() {
- return maxRetainedSize + maxReturnSize;
+ return cachedRawDataSize + maxReturnSize;
}
@Override
@@ -109,7 +109,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return maxRetainedSize;
+ return isGroupByQuery ? cachedRawDataSize : 0;
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
index ac812e0b17..9e4dc825f0 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
@@ -1012,16 +1012,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION;
- long expectedMaxRetainSize =
- 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ long cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator1.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator1.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
// case2: without group by, step is PARTIAL
List<AggregationDescriptor> aggregationDescriptors2 =
@@ -1047,15 +1044,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ 2 * LongColumn.SIZE_IN_BYTES_PER_POSITION;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator2.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator2.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
long maxTsBlockLineNumber =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
@@ -1092,14 +1087,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION);
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator3.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator3.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator3.calculateRetainedSizeAfterCallingNext());
// case4: with group by, total window num > 1000
@@ -1130,14 +1125,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION));
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator4.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator4.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator4.calculateRetainedSizeAfterCallingNext());
// case5: over DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES
@@ -1166,14 +1161,14 @@ public class OperatorMemoryTest {
typeProvider);
expectedMaxReturnSize = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator5.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator5.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator5.calculateRetainedSizeAfterCallingNext());
} catch (IllegalPathException e) {
e.printStackTrace();