You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ze...@apache.org on 2023/04/18 19:45:48 UTC

[arrow] branch main updated: GH-35212: [Go] Add ability to show full call stack with ARROW_CHECKED_MAX_RETAINED_FRAMES (#35215)

This is an automated email from the ASF dual-hosted git repository.

zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/main by this push:
     new 7a7b414d1f GH-35212: [Go] Add ability to show full call stack with ARROW_CHECKED_MAX_RETAINED_FRAMES (#35215)
7a7b414d1f is described below

commit 7a7b414d1fd0d648a6d248fb1bbe1984e9585480
Author: Herman Schaaf <he...@gmail.com>
AuthorDate: Tue Apr 18 20:45:41 2023 +0100

    GH-35212: [Go] Add ability to show full call stack with ARROW_CHECKED_MAX_RETAINED_FRAMES (#35215)
    
    This adds support for printing the full call stack when a leak is reported by the checked memory allocator. An `ARROW_CHECKED_MAX_RETAINED_FRAMES` environment variable controls how many frames are retained. To keep this completely backwards-compatible, the default right now is zero. In this case, the reported error is exactly the same as before. When a higher value is given though, a longer call stack is printed. For example:
    
    Before (same as `ARROW_CHECKED_MAX_RETAINED_FRAMES=0`):
    
    ```
    checked_allocator.go:160: LEAK of 64 bytes FROM github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).newData line 2396
    ```
    
    After (with `ARROW_CHECKED_MAX_RETAINED_FRAMES=100`):
    
    ```
    checked_allocator.go:160: LEAK of 64 bytes FROM github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).newData line 2396
         github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).NewTimestampArray line 2386
         github.com/apache/arrow/go/v12/arrow/array.(*TimestampBuilder).NewArray line 2380
         github.com/apache/arrow/go/v12/arrow/array.(*RecordBuilder).NewRecord line 346
         github.com/cloudquery/plugin-sdk/v2/testdata.GenTestData line 243
         github.com/cloudquery/plugin-sdk/v2/plugins/destination.testMigration line 53
         github.com/cloudquery/plugin-sdk/v2/plugins/destination.(*PluginTestSuite).destinationPluginTestMigrate.func5 line 239
         testing.tRunner line 1576
         runtime.goexit line 1172
    ```
    * Closes: #35212
    
    Authored-by: Herman Schaaf <he...@gmail.com>
    Signed-off-by: Matt Topol <zo...@gmail.com>
---
 go/arrow/memory/checked_allocator.go | 55 ++++++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 11 deletions(-)

diff --git a/go/arrow/memory/checked_allocator.go b/go/arrow/memory/checked_allocator.go
index 06be9bda30..12f92912bb 100644
--- a/go/arrow/memory/checked_allocator.go
+++ b/go/arrow/memory/checked_allocator.go
@@ -17,9 +17,11 @@
 package memory
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"unsafe"
@@ -46,8 +48,11 @@ func (a *CheckedAllocator) Allocate(size int) []byte {
 	}
 
 	ptr := uintptr(unsafe.Pointer(&out[0]))
+	pcs := make([]uintptr, maxRetainedFrames)
+	runtime.Callers(allocFrames, pcs)
+	callersFrames := runtime.CallersFrames(pcs)
 	if pc, _, l, ok := runtime.Caller(allocFrames); ok {
-		a.allocs.Store(ptr, &dalloc{pc: pc, line: l, sz: size})
+		a.allocs.Store(ptr, &dalloc{pc: pc, line: l, sz: size, callersFrames: callersFrames})
 	}
 	return out
 }
@@ -63,9 +68,13 @@ func (a *CheckedAllocator) Reallocate(size int, b []byte) []byte {
 
 	newptr := uintptr(unsafe.Pointer(&out[0]))
 	a.allocs.Delete(oldptr)
+	pcs := make([]uintptr, maxRetainedFrames)
+	runtime.Callers(reallocFrames, pcs)
+	callersFrames := runtime.CallersFrames(pcs)
 	if pc, _, l, ok := runtime.Caller(reallocFrames); ok {
-		a.allocs.Store(newptr, &dalloc{pc: pc, line: l, sz: size})
+		a.allocs.Store(newptr, &dalloc{pc: pc, line: l, sz: size, callersFrames: callersFrames})
 	}
+
 	return out
 }
 
@@ -86,14 +95,16 @@ func (a *CheckedAllocator) Free(b []byte) {
 // of the inner workings of Buffer in order to find the caller that actually triggered
 // the allocation via a call to Resize/Reserve/etc.
 const (
-	defAllocFrames   = 4
-	defReallocFrames = 3
+	defAllocFrames       = 4
+	defReallocFrames     = 3
+	defMaxRetainedFrames = 0
 )
 
 // Use the environment variables ARROW_CHECKED_ALLOC_FRAMES and ARROW_CHECKED_REALLOC_FRAMES
-// to control how many frames up it checks when storing the caller for allocations/reallocs
-// when using this to find memory leaks.
-var allocFrames, reallocFrames int = defAllocFrames, defReallocFrames
+// to control how many frames it skips when storing the caller for allocations/reallocs
+// when using this to find memory leaks. Use ARROW_CHECKED_MAX_RETAINED_FRAMES to control how
+// many frames are retained for printing the stack trace of a leak.
+var allocFrames, reallocFrames, maxRetainedFrames int = defAllocFrames, defReallocFrames, defMaxRetainedFrames
 
 func init() {
 	if val, ok := os.LookupEnv("ARROW_CHECKED_ALLOC_FRAMES"); ok {
@@ -107,12 +118,19 @@ func init() {
 			reallocFrames = f
 		}
 	}
+
+	if val, ok := os.LookupEnv("ARROW_CHECKED_MAX_RETAINED_FRAMES"); ok {
+		if f, err := strconv.Atoi(val); err == nil {
+			maxRetainedFrames = f
+		}
+	}
 }
 
 type dalloc struct {
-	pc   uintptr
-	line int
-	sz   int
+	pc            uintptr
+	line          int
+	sz            int
+	callersFrames *runtime.Frames
 }
 
 type TestingT interface {
@@ -124,7 +142,22 @@ func (a *CheckedAllocator) AssertSize(t TestingT, sz int) {
 	a.allocs.Range(func(_, value interface{}) bool {
 		info := value.(*dalloc)
 		f := runtime.FuncForPC(info.pc)
-		t.Errorf("LEAK of %d bytes FROM %s line %d\n", info.sz, f.Name(), info.line)
+		frames := info.callersFrames
+		var callersMsg strings.Builder
+		for {
+			frame, more := frames.Next()
+			if frame.Line == 0 {
+				break
+			}
+			callersMsg.WriteString("\t")
+			callersMsg.WriteString(frame.Function)
+			callersMsg.WriteString(fmt.Sprintf(" line %d", frame.Line))
+			callersMsg.WriteString("\n")
+			if !more {
+				break
+			}
+		}
+		t.Errorf("LEAK of %d bytes FROM %s line %d\n%v", info.sz, f.Name(), info.line, callersMsg.String())
 		return true
 	})