[asterixdb] 06/07: [NO ISSUE][OTH] Add some logging when join build fails to insert tuple

[mb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit a31f0658a2a698b7b491c224c59a02e5caa1fddf
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Mon Feb 6 12:48:29 2023 -0800

    [NO ISSUE][OTH] Add some logging when join build fails to insert tuple
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Change-Id: I5e342019d4831bfef0a10d6154a080c3cb524256
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17363
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
 .../PreferToSpillFullyOccupiedFramePolicy.java             | 10 ++++++++++
 .../hyracks/dataflow/std/join/OptimizedHybridHashJoin.java | 14 ++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
index 12985c0bec..613a396831 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/buffermanager/PreferToSpillFullyOccupiedFramePolicy.java
@@ -98,4 +98,14 @@ public class PreferToSpillFullyOccupiedFramePolicy {
             }
         };
     }
+
+    public String partitionsStatus() {
+        StringBuilder sb = new StringBuilder();
+        int numPartitions = bufferManager.getNumPartitions();
+        for (int p = 0; p < numPartitions; p++) {
+            sb.append("p:").append(p).append(",#t:").append(bufferManager.getNumTuples(p)).append(",s:")
+                    .append(spilledStatus.get(p)).append(",s:").append(bufferManager.getPhysicalSize(p)).append('\n');
+        }
+        return sb.toString();
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
index 7a9bb25ff3..02a381de14 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoin.java
@@ -49,6 +49,8 @@ import org.apache.hyracks.dataflow.std.buffermanager.VPartitionTupleBufferManage
 import org.apache.hyracks.dataflow.std.structures.ISerializableTable;
 import org.apache.hyracks.dataflow.std.structures.SerializableHashTable;
 import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * This class mainly applies one level of HHJ on a pair of
@@ -56,6 +58,7 @@ import org.apache.hyracks.dataflow.std.structures.TuplePointer;
  */
 public class OptimizedHybridHashJoin {
 
+    private static final Logger LOGGER = LogManager.getLogger();
     // Used for special probe BigObject which can not be held into the Join memory
     private FrameTupleAppender bigFrameAppender;
 
@@ -155,12 +158,19 @@ public class OptimizedHybridHashJoin {
             int recordSize = VPartitionTupleBufferManager.calculateActualSize(null, accessorBuild.getTupleLength(tid));
             double numFrames = (double) recordSize / (double) jobletCtx.getInitialFrameSize();
             int victimPartition;
-            if (numFrames > bufferManager.getConstrain().frameLimit(pid)
-                    || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) {
+            int partitionFrameLimit = bufferManager.getConstrain().frameLimit(pid);
+            if (numFrames > partitionFrameLimit || (victimPartition = spillPolicy.selectVictimPartition(pid)) < 0) {
                 // insert request can never be satisfied
                 if (numFrames > memSizeInFrames || recordSize < jobletCtx.getInitialFrameSize()) {
                     // the tuple is greater than the memory budget or although the record is small we could not find
                     // a frame for it (possibly due to a bug)
+                    String details = String.format(
+                            "partition %s, tuple size %s, needed # frames %s, partition frame limit %s, join "
+                                    + "memory in frames %s, initial frame size %s",
+                            pid, recordSize, numFrames, partitionFrameLimit, memSizeInFrames,
+                            jobletCtx.getInitialFrameSize());
+                    LOGGER.debug("can't insert tuple in join memory. {}", details);
+                    LOGGER.debug("partitions status:\n{}", spillPolicy.partitionsStatus());
                     throw HyracksDataException.create(ErrorCode.INSUFFICIENT_MEMORY);
                 }
                 // Record is large but insertion failed either 1) we could not satisfy the request because of the