You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2025/01/08 00:34:05 UTC
(incubator-gluten) branch main updated: [VL] Fix wrong warning of "Memory overhead is set to ..." under default Spark config settings (#8448)
This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d6457b77c9 [VL] Fix wrong warning of "Memory overhead is set to ..." under default Spark config settings (#8448)
d6457b77c9 is described below
commit d6457b77c9741a52560c4960afffd403dca900bf
Author: Hongze Zhang <ho...@intel.com>
AuthorDate: Wed Jan 8 08:34:00 2025 +0800
[VL] Fix wrong warning of "Memory overhead is set to ..." under default Spark config settings (#8448)
---
.../org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 45a30f8cf3..5d75521b84 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -67,10 +67,11 @@ class VeloxListenerApi extends ListenerApi with Logging {
ByteUnit.BYTE.toMiB(desiredOverheadSize).toString)
}
val overheadSize: Long = SparkResourceUtil.getMemoryOverheadSize(conf)
- if (overheadSize < desiredOverheadSize) {
+ if (ByteUnit.BYTE.toMiB(overheadSize) < ByteUnit.BYTE.toMiB(desiredOverheadSize)) {
logWarning(
- s"Memory overhead is set to $overheadSize which is smaller than the recommended size" +
- s" $desiredOverheadSize. This may cause OOM.")
+ s"Memory overhead is set to ${ByteUnit.BYTE.toMiB(overheadSize)}MiB which is smaller than" +
+ s" the recommended size ${ByteUnit.BYTE.toMiB(desiredOverheadSize)}MiB." +
+ s" This may cause OOM.")
}
conf.set(GlutenConfig.GLUTEN_OVERHEAD_SIZE_IN_BYTES_KEY, overheadSize.toString)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org