[skywalking-rover] branch main updated: Improve the stability of Off CPU Profiling (#103)

[wu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-rover.git


The following commit(s) were added to refs/heads/main by this push:
     new 97857df  Improve the stability of Off CPU Profiling (#103)
97857df is described below

commit 97857dfe8f9a677336d2307c820b51c69c5a7a74
Author: mrproliu <74...@qq.com>
AuthorDate: Sat Oct 21 09:55:04 2023 +0800

    Improve the stability of Off CPU Profiling (#103)
---
 CHANGES.md                          |  1 +
 pkg/profiling/task/offcpu/runner.go | 24 ++++++++++++++++--------
 pkg/tools/profiling/api.go          | 19 ++++++++++++++++---
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 97baef8..e392441 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,6 +8,7 @@ Release Notes.
 * Enhance compatibility when profiling with SSL.
 * Update `LabelValue` obtain pod information function to add default value parameter.
 * Add `HasOwnerName` to judgement pod has owner name.
+* Improve the stability of Off CPU Profiling.
 
 #### Bug Fixes
 
diff --git a/pkg/profiling/task/offcpu/runner.go b/pkg/profiling/task/offcpu/runner.go
index 83d25ab..492d996 100644
--- a/pkg/profiling/task/offcpu/runner.go
+++ b/pkg/profiling/task/offcpu/runner.go
@@ -26,6 +26,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 
+	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/link"
 
 	"github.com/apache/skywalking-rover/pkg/logger"
@@ -65,7 +66,7 @@ type Runner struct {
 	// runtime
 	previousStacks  map[ProcessStack]StackCounter
 	bpf             *bpfObjects
-	kprobe          link.Link
+	kprobe          *btf.Linker
 	stopChan        chan bool
 	flushDataNotify context.CancelFunc
 }
@@ -117,25 +118,32 @@ func (r *Runner) Run(ctx context.Context, notify base.ProfilingRunningSuccessNot
 	}
 	r.bpf = &objs
 
-	kprobe, err := link.Kprobe(r.findMatchesSymbol(), objs.DoFinishTaskSwitch, nil)
-	if err != nil {
+	symbols := r.findMatchesSymbol()
+	linker := btf.NewLinker()
+	switchers := make(map[string]*ebpf.Program)
+	for _, symbol := range symbols {
+		switchers[symbol] = objs.DoFinishTaskSwitch
+	}
+
+	linker.AddLink(link.Kprobe, switchers)
+	if err := linker.HasError(); err != nil {
 		return fmt.Errorf("link to finish task swtich failure: %v", err)
 	}
-	r.kprobe = kprobe
+	r.kprobe = linker
 
 	notify()
 	<-r.stopChan
 	return nil
 }
 
-func (r *Runner) findMatchesSymbol() string {
+func (r *Runner) findMatchesSymbol() []string {
 	if r.kernelProfiling == nil {
-		return defaultKernelSymbol
+		return []string{defaultKernelSymbol}
 	}
-	res, err := r.kernelProfiling.FindSymbolByRegex(`finish_task_switch(\.\w+\.\d+)?`)
+	res, err := r.kernelProfiling.FindMultipleSymbolByRegex(`finish_task_switch(\.\w+\.\d+)?`)
 	if err != nil {
 		log.Warnf("found symbol error: %v", err)
-		return defaultKernelSymbol
+		return []string{defaultKernelSymbol}
 	}
 	return res
 }
diff --git a/pkg/tools/profiling/api.go b/pkg/tools/profiling/api.go
index cf57adb..518da05 100644
--- a/pkg/tools/profiling/api.go
+++ b/pkg/tools/profiling/api.go
@@ -142,18 +142,31 @@ func (i *Info) FindSymbolAddress(name string) uint64 {
 }
 
 func (i *Info) FindSymbolByRegex(rep string) (string, error) {
-	compile, err := regexp.Compile(rep)
+	vals, err := i.FindMultipleSymbolByRegex(rep)
 	if err != nil {
 		return "", err
 	}
+	return vals[0], nil
+}
+
+func (i *Info) FindMultipleSymbolByRegex(rep string) ([]string, error) {
+	compile, err := regexp.Compile(rep)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]string, 0)
 	for _, m := range i.Modules {
 		for _, sym := range m.Symbols {
 			if compile.MatchString(sym.Name) {
-				return sym.Name, nil
+				result = append(result, sym.Name)
 			}
 		}
 	}
-	return "", fmt.Errorf("cannot found any matches symbol: %s", rep)
+
+	if len(result) == 0 {
+		return nil, fmt.Errorf("cannot found any matches symbol: %s", rep)
+	}
+	return result, nil
 }
 
 func (m *Module) contains(addr uint64) (uint64, bool) {