You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/01/14 04:23:34 UTC
[iotdb] branch geely_1.0.1 updated: [IOTDB-5401] Reduce the estimated size of memory retained after calling next in SeriesAggregationScanOperator
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch geely_1.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/geely_1.0.1 by this push:
new 625ea5da01 [IOTDB-5401] Reduce the estimated size of memory retained after calling next in SeriesAggregationScanOperator
625ea5da01 is described below
commit 625ea5da014011856ded9d6e194abafddd033c39
Author: liuminghui233 <36...@users.noreply.github.com>
AuthorDate: Thu Jan 12 11:19:58 2023 +0800
[IOTDB-5401] Reduce the estimated size of memory retained after calling next in SeriesAggregationScanOperator
---
.../AbstractSeriesAggregationScanOperator.java | 8 ++---
.../mpp/execution/operator/OperatorMemoryTest.java | 35 ++++++++++------------
2 files changed, 19 insertions(+), 24 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
index 5f285de538..3566e1d834 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/AbstractSeriesAggregationScanOperator.java
@@ -64,7 +64,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
protected boolean finished = false;
- private final long maxRetainedSize;
+ private final long cachedRawDataSize;
private final long maxReturnSize;
public AbstractSeriesAggregationScanOperator(
@@ -92,14 +92,14 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
}
this.resultTsBlockBuilder = new TsBlockBuilder(dataTypes);
- this.maxRetainedSize =
+ this.cachedRawDataSize =
(1L + subSensorSize) * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
this.maxReturnSize = maxReturnSize;
}
@Override
public long calculateMaxPeekMemory() {
- return maxRetainedSize + maxReturnSize;
+ return cachedRawDataSize + maxReturnSize;
}
@Override
@@ -109,7 +109,7 @@ public abstract class AbstractSeriesAggregationScanOperator extends AbstractData
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return maxRetainedSize;
+ return isGroupByQuery ? cachedRawDataSize : 0;
}
@Override
diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
index 8bc9f944a4..447a91c955 100644
--- a/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/OperatorMemoryTest.java
@@ -1149,16 +1149,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION;
- long expectedMaxRetainSize =
- 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ long cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator1.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator1.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator1.calculateRetainedSizeAfterCallingNext());
// case2: without group by, step is PARTIAL
List<AggregationDescriptor> aggregationDescriptors2 =
@@ -1184,15 +1181,13 @@ public class OperatorMemoryTest {
TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ 2 * LongColumn.SIZE_IN_BYTES_PER_POSITION;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator2.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator2.calculateMaxReturnSize());
- assertEquals(
- expectedMaxRetainSize,
- seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
+ assertEquals(0, seriesAggregationScanOperator2.calculateRetainedSizeAfterCallingNext());
long maxTsBlockLineNumber =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
@@ -1229,14 +1224,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION);
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator3.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator3.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator3.calculateRetainedSizeAfterCallingNext());
// case4: with group by, total window num > 1000
@@ -1267,14 +1262,14 @@ public class OperatorMemoryTest {
* (TimeColumn.SIZE_IN_BYTES_PER_POSITION
+ 512 * Byte.BYTES
+ LongColumn.SIZE_IN_BYTES_PER_POSITION));
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator4.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator4.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator4.calculateRetainedSizeAfterCallingNext());
// case5: over DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES
@@ -1303,14 +1298,14 @@ public class OperatorMemoryTest {
typeProvider);
expectedMaxReturnSize = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
- expectedMaxRetainSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
+ cachedRawDataSize = 2L * TSFileDescriptor.getInstance().getConfig().getPageSizeInByte();
assertEquals(
- expectedMaxReturnSize + expectedMaxRetainSize,
+ expectedMaxReturnSize + cachedRawDataSize,
seriesAggregationScanOperator5.calculateMaxPeekMemory());
assertEquals(expectedMaxReturnSize, seriesAggregationScanOperator5.calculateMaxReturnSize());
assertEquals(
- expectedMaxRetainSize,
+ cachedRawDataSize,
seriesAggregationScanOperator5.calculateRetainedSizeAfterCallingNext());
} catch (IllegalPathException e) {
e.printStackTrace();