You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/06/20 02:03:47 UTC
Change in asterixdb[master]: [ASTERIXDB-3434][RT]: Fixed Group By Memmory Issue
From Gaurav Vaghasiya <gv...@gmail.com>:
Gaurav Vaghasiya has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18392 )
Change subject: [ASTERIXDB-3434][RT]: Fixed Group By Memmory Issue
......................................................................
[ASTERIXDB-3434][RT]: Fixed Group By Memmory Issue
Details:
Each Group By Operator will use half memory of total Group memory to accomdate both local and global Group By
Change-Id: I3973791350ea74bb695937f4937f60cc65592dd5
---
M asterixdb/asterix-app/src/main/resources/cc3.conf
M asterixdb/asterix-app/src/test/resources/cc-cbotest.conf
M hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
M asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf
M asterixdb/asterix-app/src/test/resources/cc-storage.conf
M asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
M asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate/min_max_arrays/min_max_arrays.35.query.sqlpp
M asterixdb/asterix-app/src/test/resources/cc-ssl.conf
M asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf
M asterixdb/asterix-app/src/main/resources/cc2.conf
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/min_max_arrays/min_max_arrays.32.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
M asterixdb/asterix-app/src/test/resources/cc.conf
M asterixdb/asterix-app/src/main/resources/cc4.conf
M asterixdb/asterix-app/src/test/resources/cc-single.conf
M asterixdb/asterix-app/src/main/resources/cc.conf
M asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
M asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
M asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf
M asterixdb/asterix-app/src/test/resources/cc-columnar.conf
M asterixdb/asterix-app/src/test/resources/cc-multipart.conf
22 files changed, 35 insertions(+), 22 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/92/18392/1
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index e0eb88d..dc322d3 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -53,7 +53,7 @@
log.level = DEBUG
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index 8868a54..6506c35 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -49,7 +49,7 @@
log.level = WARN
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index 71ad17b..83e711e 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -49,7 +49,7 @@
log.level = WARN
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf b/asterixdb/asterix-app/src/main/resources/cc4.conf
index bf37499..54975e2 100644
--- a/asterixdb/asterix-app/src/main/resources/cc4.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc4.conf
@@ -47,7 +47,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.windowmemory=192KB
messaging.frame.size=4096
diff --git a/asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf b/asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf
index ab028c8..982d8e1 100644
--- a/asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc_no_cbo.conf
@@ -55,7 +55,7 @@
compiler.cbotest=false
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf b/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf
index 9d28e55..999ee7d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc_static_partitioning.conf
@@ -53,7 +53,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cbotest.conf b/asterixdb/asterix-app/src/test/resources/cc-cbotest.conf
index 3a1ac4f..782f6de 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cbotest.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cbotest.conf
@@ -57,7 +57,7 @@
compiler.queryplanshape=zigzag
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
index 3c883a8..c9a4632 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage-gcs.conf
@@ -54,7 +54,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
index 6ea8308..3c2aa4a 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-cloud-storage.conf
@@ -54,7 +54,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-columnar.conf b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
index 11feecc..b2fe232 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-columnar.conf
@@ -53,7 +53,7 @@
log.level = DEBUG
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
index 6c67624..197840d 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-multipart.conf
@@ -48,7 +48,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.windowmemory=192KB
messaging.frame.size=4096
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf b/asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf
index d59d4d5..02658dd 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single-lookup.conf
@@ -54,7 +54,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
index dab765b..7eb6fba 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single-partition.conf
@@ -45,7 +45,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-single.conf b/asterixdb/asterix-app/src/test/resources/cc-single.conf
index a6f4e89..fcb50a4 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-single.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-single.conf
@@ -41,7 +41,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
index ae91f5e..48de257 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-small-txn-log-partition.conf
@@ -46,7 +46,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.windowmemory=192KB
messaging.frame.size=4096
diff --git a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
index 547beff..ff7281e 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-ssl.conf
@@ -60,7 +60,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc-storage.conf b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
index 68faa17..3c7bba8 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-storage.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-storage.conf
@@ -46,7 +46,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index 624455c..d2da581 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -54,7 +54,7 @@
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
-compiler.groupmemory=160KB
+compiler.groupmemory=320KB
compiler.joinmemory=256KB
compiler.textsearchmemory=160KB
compiler.windowmemory=192KB
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/min_max_arrays/min_max_arrays.32.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/min_max_arrays/min_max_arrays.32.query.sqlpp
index 05399e7..2495bca 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/min_max_arrays/min_max_arrays.32.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate-sql/min_max_arrays/min_max_arrays.32.query.sqlpp
@@ -22,6 +22,6 @@
*/
use test;
-set `compiler.groupmemory` "128KB";
+set `compiler.groupmemory` "256KB";
select group_id,min(array_f) from openDs3 group by group_id order by group_id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate/min_max_arrays/min_max_arrays.35.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate/min_max_arrays/min_max_arrays.35.query.sqlpp
index 49018eb..fdc0848 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate/min_max_arrays/min_max_arrays.35.query.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/aggregate/min_max_arrays/min_max_arrays.35.query.sqlpp
@@ -22,7 +22,7 @@
*/
use test;
-set `compiler.groupmemory` "128KB";
+set `compiler.groupmemory` "256KB";
from openDs3 as ds
group by ds.group_id group as g
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
index 0b6fdbf..d446edb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp
@@ -24,7 +24,7 @@
SET `compiler.parallelism` "1";
SET `compiler.min.memory.allocation` "false";
SET `compiler.sortmemory` "320KB";
-SET `compiler.groupmemory` "160KB";
+SET `compiler.groupmemory` "320KB";
SET `compiler.joinmemory` "256KB";
SET `compiler.textsearchmemory` "160KB";
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
index 4e859e4..fb0d266 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetMemoryRequirementsRule.java
@@ -166,7 +166,8 @@
@Override
public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
- setOperatorMemoryBudget(op, physConfig.getMaxFramesForGroupBy());
+ // each group by operator gets half of total group memory to accommodate both local and global operators
+ setOperatorMemoryBudget(op, physConfig.getMaxFramesForGroupBy()/2);
return null;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18392
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I3973791350ea74bb695937f4937f60cc65592dd5
Gerrit-Change-Number: 18392
Gerrit-PatchSet: 1
Gerrit-Owner: Gaurav Vaghasiya <gv...@gmail.com>
Gerrit-MessageType: newchange