You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2023/01/11 09:52:20 UTC
[iotdb] 01/01: modify mem calc
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch lmh/FixAggrMem
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b02771471a1c48f6a9a5fe224ecc922f62797fdb
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Wed Jan 11 17:51:18 2023 +0800
modify mem calc
---
.../AbstractSeriesAggregationScanOperator.java | 8 ++---
.../mpp/execution/operator/OperatorMemoryTest.java | 35 ++++++++++------------
2 files changed, 19 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index ee64166a4b..122247c85b 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -64,7 +64,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
protected boolean finished = false;
- private final long maxRetainedSize;
+ private final long cachedRawDataSize;
private final long maxReturnSize;
protected AbstractSeriesAggregationScanOperator(
@@ -92,14 +92,14 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
}
this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
- this.maxRetainedSize =
+ this.cachedRawDataSize =
(1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
this.maxReturnSize = maxReturnSize;
}
@Override
public long calculateMaxPeekMemory() {
- return maxRetainedSize + maxReturnSize;
+ return cachedRawDataSize + maxReturnSize;
}
@Override
@@ -109,7 +109,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return maxRetainedSize;
+ return isGroupByQuery ? cachedRawDataSize : 0;
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
index ac812e0b17..9e4dc825f0 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
@@ -1012,16 +1012,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION;
- long expectedMaxRetainSize =
- 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ long cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator1.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator1.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
// case2: without group by, step is PARTIAL
List<AggregationDescriptor> aggregationDescriptors2 =
@@ -1047,15 +1044,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ 2 * LongColumn.SIZE_IN_BYTES_PER_POSITION;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator2.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator2.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
long maxTsBlockLineNumber =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
@@ -1092,14 +1087,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION);
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator3.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator3.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator3.calculateRetainedSizeAfterCallingNext());
// case4: with group by, total window num > 1000
@@ -1130,14 +1125,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION));
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator4.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator4.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator4.calculateRetainedSizeAfterCallingNext());
// case5: over DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES
@@ -1166,14 +1161,14 @@ public class OperatorMemoryTest {
typeProvider);
expectedMaxReturnSize = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator5.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator5.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator5.calculateRetainedSizeAfterCallingNext());
} catch (IllegalPathException e) {
e.printStackTrace();