You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ch...@apache.org on 2026/01/07 10:07:23 UTC
(incubator-gluten) branch main updated: [VL] Add config spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory (#11354)
This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ea9ba244f5 [VL] Add config spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory (#11354)
ea9ba244f5 is described below
commit ea9ba244f5511d0435b540f566df036371d75453
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Wed Jan 7 18:07:19 2026 +0800
[VL] Add config spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory (#11354)
---
.../main/scala/org/apache/gluten/config/VeloxConfig.scala | 12 ++++++++++++
cpp/velox/compute/WholeStageResultIterator.cc | 5 ++++-
cpp/velox/config/VeloxConfig.h | 2 ++
docs/velox-configuration.md | 1 +
4 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 6644f4edc2..03781d2fb5 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -380,6 +380,18 @@ object VeloxConfig extends ConfigRegistry {
.doubleConf
.createWithDefault(0.1)
+ val MAX_EXTENDED_PARTIAL_AGGREGATION_MEMORY =
+ buildConf("spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory")
+ .doc(
+ "Set the max extended memory of partial aggregation in bytes. When this option is set " +
+ "to a value greater than 0, it will override spark.gluten.sql.columnar.backend.velox." +
+ "maxExtendedPartialAggregationMemoryRatio. Note: this option only works when " +
+ "flushable partial aggregation is enabled. Ignored when " +
+ "spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false."
+ )
+ .bytesConf(ByteUnit.BYTE)
+ .createOptional
+
val MAX_EXTENDED_PARTIAL_AGGREGATION_MEMORY_RATIO =
buildConf("spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio")
.doc(
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
index 6b9761a154..e91e2ad69d 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -555,7 +555,10 @@ std::unordered_map<std::string, std::string> WholeStageResultIterator::getQueryC
: static_cast<int64_t>(veloxCfg_->get<double>(kMaxPartialAggregationMemoryRatio, 0.1) * offHeapMemory));
auto maxExtendedPartialAggregationMemory = std::max<int64_t>(
1 << 26,
- static_cast<long>(veloxCfg_->get<double>(kMaxExtendedPartialAggregationMemoryRatio, 0.15) * offHeapMemory));
+ veloxCfg_->get<int64_t>(kMaxExtendedPartialAggregationMemory).has_value()
+ ? veloxCfg_->get<int64_t>(kMaxExtendedPartialAggregationMemory).value()
+ : static_cast<int64_t>(
+ veloxCfg_->get<double>(kMaxExtendedPartialAggregationMemoryRatio, 0.15) * offHeapMemory));
configs[velox::core::QueryConfig::kMaxPartialAggregationMemory] = std::to_string(maxPartialAggregationMemory);
configs[velox::core::QueryConfig::kMaxExtendedPartialAggregationMemory] =
std::to_string(maxExtendedPartialAggregationMemory);
diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h
index 6db6fb0994..2cacee5369 100644
--- a/cpp/velox/config/VeloxConfig.h
+++ b/cpp/velox/config/VeloxConfig.h
@@ -55,6 +55,8 @@ const std::string kMaxPartialAggregationMemoryRatio =
const std::string kMaxPartialAggregationMemory = "spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemory";
const std::string kMaxExtendedPartialAggregationMemoryRatio =
"spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio";
+const std::string kMaxExtendedPartialAggregationMemory =
+ "spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory";
const std::string kAbandonPartialAggregationMinPct =
"spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct";
const std::string kAbandonPartialAggregationMinRows =
diff --git a/docs/velox-configuration.md b/docs/velox-configuration.md
index 9738f283ee..bd838f357c 100644
--- a/docs/velox-configuration.md
+++ b/docs/velox-configuration.md
@@ -37,6 +37,7 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.maxCoalescedBytes | 64MB | Set the max coalesced bytes for velox file scan [...]
| spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance | 512KB | Set the max coalesced distance bytes for velox file scan [...]
| spark.gluten.sql.columnar.backend.velox.maxCompiledRegexes | 100 | Controls maximum number of compiled regular expression patterns per function instance per thread of execution. [...]
+| spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemory | <undefined> | Set the max extended memory of partial aggregation in bytes. When this option is set to a value greater than 0, it will override spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio. Note: this option only works when flushable partial aggregation is enabled. Ignored when spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false. [...]
| spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio | 0.15 | Set the max extended memory of partial aggregation as maxExtendedPartialAggregationMemoryRatio of offheap size. Note: this option only works when flushable partial aggregation is enabled. Ignored when spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false. [...]
| spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemory | <undefined> | Set the max memory of partial aggregation in bytes. When this option is set to a value greater than 0, it will override spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio. Note: this option only works when flushable partial aggregation is enabled. Ignored when spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false. [...]
| spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio | 0.1 | Set the max memory of partial aggregation as maxPartialAggregationMemoryRatio of offheap size. Note: this option only works when flushable partial aggregation is enabled. Ignored when spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false. [...]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org