You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by li...@apache.org on 2025/07/04 13:46:26 UTC

(incubator-gluten) branch main updated: [CORE] Add more log for spill (#10117)

This is an automated email from the ASF dual-hosted git repository.

liujiayi771 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 59665e93ab [CORE] Add more log for spill (#10117)
59665e93ab is described below

commit 59665e93ab39cbf9a768932e49ff2833c70a0dac
Author: Joey <jo...@alibaba-inc.com>
AuthorDate: Fri Jul 4 21:46:21 2025 +0800

    [CORE] Add more log for spill (#10117)
---
 .../org/apache/gluten/vectorized/NativePlanEvaluator.java     | 11 ++++++++++-
 .../scala/org/apache/gluten/memory/NativeMemoryManager.scala  |  4 +++-
 .../org/apache/gluten/memory/memtarget/MemoryTargets.java     |  2 +-
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
index 96643dd711..44d4107c56 100644
--- a/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
+++ b/gluten-arrow/src/main/java/org/apache/gluten/vectorized/NativePlanEvaluator.java
@@ -25,12 +25,15 @@ import org.apache.gluten.utils.DebugUtil;
 import org.apache.gluten.validate.NativePlanValidationInfo;
 
 import org.apache.spark.TaskContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
 public class NativePlanEvaluator {
+  private static final Logger LOGGER = LoggerFactory.getLogger(NativePlanEvaluator.class);
   private static final AtomicInteger id = new AtomicInteger(0);
 
   private final Runtime runtime;
@@ -86,7 +89,13 @@ public class NativePlanEvaluator {
                 if (!Spillers.PHASE_SET_SPILL_ONLY.contains(phase)) {
                   return 0L;
                 }
-                return out.spill(size);
+                long spilled = out.spill(size);
+                LOGGER.info(
+                    "NativePlanEvaluator-{}: Spilled {} / {} bytes of data.",
+                    id.get(),
+                    spilled,
+                    size);
+                return spilled;
               }
             });
     return out;
diff --git a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
index f2db33d8e1..5aae7f6c57 100644
--- a/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
+++ b/gluten-arrow/src/main/scala/org/apache/gluten/memory/NativeMemoryManager.scala
@@ -61,7 +61,9 @@ object NativeMemoryManager {
           // Only respond for shrinking.
           return 0L
         }
-        NativeMemoryManagerJniWrapper.shrink(handle, size)
+        val shrunk = NativeMemoryManagerJniWrapper.shrink(handle, size)
+        LOGGER.info(s"NativeMemoryManager: Shrunk $shrunk / $size bytes of data.")
+        shrunk
       }
     })
     mutableStats += "single" -> new MemoryUsageStatsBuilder {
diff --git a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
index 135a82fa58..5f8013c62e 100644
--- a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
+++ b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/MemoryTargets.java
@@ -90,7 +90,7 @@ public final class MemoryTargets {
           LOGGER.info("Request for spilling on consumer {}...", consumer.name());
           // Note: Spill from root node so other consumers also get spilled.
           long spilled = TreeMemoryTargets.spillTree(root, Long.MAX_VALUE);
-          LOGGER.info("Consumer {} spilled {} bytes.", consumer.name(), spilled);
+          LOGGER.info("Consumer {} gets {} bytes from spilling.", consumer.name(), spilled);
         });
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org