You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@yunikorn.apache.org by ti...@apache.org on 2026/09/29 16:12:17 UTC

(yunikorn-core) branch master updated: [YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)

This is an automated email from the ASF dual-hosted git repository.

HuangTing-Yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git


The following commit(s) were added to refs/heads/master by this push:
     new d13826c7 [YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)
d13826c7 is described below

commit d13826c76b96f55929ddf3d4c494f8bcef530fca
Author: andyhuangdev <ky...@gmail.com>
AuthorDate: Wed Sep 30 00:12:01 2026 +0800

    [YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)
    
    - Buffer RM reply channels with capacity one.
    - Complete configuration updates before replying.
    - Add regression tests for abandoned replies and lock release.
    
    Tests: scheduler and rmproxy suites pass with race and deadlock detection; lint passes with GOTOOLCHAIN=go1.26.0.
    
    Closes: #1167
    
    Signed-off-by: HuangTing-Yao <ti...@apache.org>
---
 pkg/rmproxy/rmproxy.go        |   8 +--
 pkg/rmproxy/rmproxy_test.go   |  60 +++++++++++++++++++++++
 pkg/scheduler/context.go      |  57 +++++++++++++---------
 pkg/scheduler/context_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 209 insertions(+), 26 deletions(-)

diff --git a/pkg/rmproxy/rmproxy.go b/pkg/rmproxy/rmproxy.go
index 80a2ba09..70be0cc7 100644
--- a/pkg/rmproxy/rmproxy.go
+++ b/pkg/rmproxy/rmproxy.go
@@ -240,7 +240,8 @@ func drainReplyChannel(ch chan *rmevent.Result) {
 func (rmp *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error) {
 	rmp.Lock()
 	defer rmp.Unlock()
-	c := make(chan *rmevent.Result)
+	// Each event replies once; buffering lets the scheduler finish without a receiver.
+	c := make(chan *rmevent.Result, 1)
 
 	// If this is a re-register we need to clean up first
 	if rmp.rmIDToCallback[request.RmID] != nil {
@@ -259,7 +260,7 @@ func (rmp *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerR
 		}
 	}
 
-	c = make(chan *rmevent.Result)
+	c = make(chan *rmevent.Result, 1)
 
 	// Add new RM.
 	go func() {
@@ -348,7 +349,8 @@ func (rmp *RMProxy) UpdateNode(request *si.NodeRequest) error {
 
 // Triggers scheduler to reload configuration and apply the changes on-the-fly to the scheduler itself.
 func (rmp *RMProxy) UpdateConfiguration(request *si.UpdateConfigurationRequest) error {
-	c := make(chan *rmevent.Result)
+	// Each event replies once; buffering lets the scheduler finish without a receiver.
+	c := make(chan *rmevent.Result, 1)
 	go func() {
 		rmp.schedulerEventHandler.HandleEvent(&rmevent.RMConfigUpdateEvent{
 			RmID:        request.RmID,
diff --git a/pkg/rmproxy/rmproxy_test.go b/pkg/rmproxy/rmproxy_test.go
index 9c98526f..efff00b7 100644
--- a/pkg/rmproxy/rmproxy_test.go
+++ b/pkg/rmproxy/rmproxy_test.go
@@ -19,15 +19,25 @@
 package rmproxy
 
 import (
+	"os"
 	"testing"
+	"testing/synctest"
 	"time"
 
+	godeadlock "github.com/sasha-s/go-deadlock"
 	"gotest.tools/v3/assert"
 
+	"github.com/apache/yunikorn-core/pkg/mock"
 	"github.com/apache/yunikorn-core/pkg/rmproxy/rmevent"
 	"github.com/apache/yunikorn-scheduler-interface/lib/go/si"
 )
 
+func TestMain(m *testing.M) {
+	// Pooled deadlock timers cannot be reused across synctest bubbles.
+	godeadlock.Opts.TimerPool = godeadlock.TimerPoolDisabled
+	os.Exit(m.Run())
+}
+
 func TestRMProxy_StopUnblocksWaitingCaller(t *testing.T) {
 	rmp := NewRMProxy(nil)
 	rmp.StartService()
@@ -83,3 +93,53 @@ func assertDrainFailedResult(t *testing.T, ch <-chan *rmevent.Result, name strin
 		t.Fatalf("timed out waiting for response on %s", name)
 	}
 }
+
+type infraReplyHandler struct {
+	events chan interface{}
+}
+
+func (h *infraReplyHandler) HandleEvent(event interface{}) { h.events <- event }
+
+func TestRMProxyInfraRepliesBuffered(t *testing.T) {
+	for _, name := range []string{"registration", "reregistration", "config"} {
+		t.Run(name, func(t *testing.T) {
+			synctest.Test(t, func(t *testing.T) {
+				handler := &infraReplyHandler{events: make(chan interface{})}
+				rmp := NewRMProxy(handler)
+				if name == "reregistration" {
+					rmp.rmIDToCallback["test"] = &mock.ResourceManagerCallback{}
+				}
+				done := make(chan error, 1)
+				go func() {
+					if name == "config" {
+						done <- rmp.UpdateConfiguration(&si.UpdateConfigurationRequest{RmID: "test"})
+					} else {
+						_, err := rmp.RegisterResourceManager(&si.RegisterResourceManagerRequest{RmID: "test"}, nil)
+						done <- err
+					}
+				}()
+				replies := 1
+				if name == "reregistration" {
+					replies = 2
+				}
+				for i := 0; i < replies; i++ {
+					event := <-handler.events
+					var ch chan *rmevent.Result
+					switch ev := event.(type) {
+					case *rmevent.RMRegistrationEvent:
+						ch = ev.Channel
+					case *rmevent.RMPartitionsRemoveEvent:
+						ch = ev.Channel
+					case *rmevent.RMConfigUpdateEvent:
+						ch = ev.Channel
+					default:
+						t.Fatalf("unexpected event %T", event)
+					}
+					assert.Check(t, cap(ch) == 1, "reply channel must hold one result without a receiver")
+					ch <- &rmevent.Result{Succeeded: true}
+				}
+				assert.NilError(t, <-done)
+			})
+		})
+	}
+}
diff --git a/pkg/scheduler/context.go b/pkg/scheduler/context.go
index c5ef60e1..78825cdb 100644
--- a/pkg/scheduler/context.go
+++ b/pkg/scheduler/context.go
@@ -157,17 +157,23 @@ func (cc *ClusterContext) schedule() bool {
 }
 
 func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrationEvent) {
+	// Complete all state changes and release the cluster lock before replying.
+	result := cc.processRMRegistrationEventResult(event)
+	event.Channel <- result
+}
+
+// processRMRegistrationEventResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) processRMRegistrationEventResult(event *rmevent.RMRegistrationEvent) *rmevent.Result {
 	cc.Lock()
 	defer cc.Unlock()
 	rmID := event.Registration.RmID
 
 	// we should not have any partitions set at this point
 	if len(cc.partitions) > 0 {
-		event.Channel <- &rmevent.Result{
+		return &rmevent.Result{
 			Reason:    fmt.Sprintf("RM %s has been registered before, active partitions %d", rmID, len(cc.partitions)),
 			Succeeded: false,
 		}
-		return
 	}
 	policyGroup := event.Registration.PolicyGroup
 	config := event.Registration.Config
@@ -180,13 +186,11 @@ func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrati
 	}
 	conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
 	if err != nil {
-		event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
-		return
+		return &rmevent.Result{Succeeded: false, Reason: err.Error()}
 	}
 	err = cc.updateSchedulerConfig(conf, rmID)
 	if err != nil {
-		event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
-		return
+		return &rmevent.Result{Succeeded: false, Reason: err.Error()}
 	}
 
 	// update global scheduler configs, set the policyGroup for this cluster
@@ -196,23 +200,29 @@ func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrati
 	// store the build information of RM
 	cc.SetRMInfo(rmID, event.Registration.BuildInfo)
 
-	// Done, notify channel
-	event.Channel <- &rmevent.Result{
+	// Done, return the result to send after unlocking
+	return &rmevent.Result{
 		Succeeded: true,
 	}
 }
 
 func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpdateEvent) {
+	// Complete all state changes and release the cluster lock before replying.
+	result := cc.processRMConfigUpdateEventResult(event)
+	event.Channel <- result
+}
+
+// processRMConfigUpdateEventResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) processRMConfigUpdateEventResult(event *rmevent.RMConfigUpdateEvent) *rmevent.Result {
 	cc.Lock()
 	defer cc.Unlock()
 	rmID := event.RmID
 	// need to enhance the check to support multiple RMs
 	if len(cc.partitions) == 0 {
-		event.Channel <- &rmevent.Result{
+		return &rmevent.Result{
 			Reason:    fmt.Sprintf("RM %s has no active partitions, make sure it is registered", rmID),
 			Succeeded: false,
 		}
-		return
 	}
 
 	// set extra configuration
@@ -226,8 +236,7 @@ func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpda
 	}
 	conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
 	if err != nil {
-		event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
-		return
+		return &rmevent.Result{Succeeded: false, Reason: err.Error()}
 	}
 	// skip update if the config has not changed: the checksum is calculated over the config again which lets
 	// us detect a real change even if the config map was updated for an unrelated reason
@@ -235,23 +244,18 @@ func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpda
 	if oldConf != nil && conf.Checksum == oldConf.Checksum {
 		log.Log(log.SchedContext).Info("configuration checksum unchanged, skipping config update",
 			zap.String("rmID", rmID), zap.String("checksum", conf.Checksum))
-		event.Channel <- &rmevent.Result{
+		return &rmevent.Result{
 			Succeeded: true,
 		}
-		return
 	}
 	// update scheduler configuration
 	err = cc.updateSchedulerConfig(conf, rmID)
 	if err != nil {
-		event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
-		return
-	}
-	// Done, notify channel
-	event.Channel <- &rmevent.Result{
-		Succeeded: true,
+		return &rmevent.Result{Succeeded: false, Reason: err.Error()}
 	}
-	// update global scheduler configs
+	// Publish the configuration before releasing the lock and replying.
 	configs.ConfigContext.Set(cc.policyGroup, conf)
+	return &rmevent.Result{Succeeded: true}
 }
 
 func (cc *ClusterContext) handleRMUpdateNodeEvent(event *rmevent.RMUpdateNodeEvent) {
@@ -311,6 +315,13 @@ func (cc *ClusterContext) processNodes(request *si.NodeRequest) {
 // Called when a RM re-registers. This triggers a full clean up.
 // Registration expects everything to be clean.
 func (cc *ClusterContext) removePartitionsByRMID(event *rmevent.RMPartitionsRemoveEvent) {
+	// Complete all state changes and release the cluster lock before replying.
+	result := cc.removePartitionsByRMIDResult(event)
+	event.Channel <- result
+}
+
+// removePartitionsByRMIDResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) removePartitionsByRMIDResult(event *rmevent.RMPartitionsRemoveEvent) *rmevent.Result {
 	cc.Lock()
 	defer cc.Unlock()
 	partitionToRemove := make(map[string]bool)
@@ -326,8 +337,8 @@ func (cc *ClusterContext) removePartitionsByRMID(event *rmevent.RMPartitionsRemo
 	for partitionName := range partitionToRemove {
 		delete(cc.partitions, partitionName)
 	}
-	// Done, notify channel
-	event.Channel <- &rmevent.Result{
+	// Done, return the result to send after unlocking
+	return &rmevent.Result{
 		Succeeded: true,
 	}
 }
diff --git a/pkg/scheduler/context_test.go b/pkg/scheduler/context_test.go
index e7ce92c5..a4fd6a3f 100644
--- a/pkg/scheduler/context_test.go
+++ b/pkg/scheduler/context_test.go
@@ -19,11 +19,15 @@
 package scheduler
 
 import (
+	"fmt"
+	"os"
 	"strings"
 	"testing"
+	"testing/synctest"
 
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
+	godeadlock "github.com/sasha-s/go-deadlock"
 	"gotest.tools/v3/assert"
 
 	"github.com/apache/yunikorn-core/pkg/common/configs"
@@ -43,6 +47,12 @@ type mockEventHandler struct {
 	newAllocHandler func(*rmevent.RMNewAllocationsEvent)
 }
 
+func TestMain(m *testing.M) {
+	// Pooled deadlock timers cannot be reused across synctest bubbles.
+	godeadlock.Opts.TimerPool = godeadlock.TimerPoolDisabled
+	os.Exit(m.Run())
+}
+
 func newMockEventHandler() *mockEventHandler {
 	return &mockEventHandler{
 		eventHandled:  false,
@@ -445,3 +455,103 @@ outer:
 
 	assert.Assert(t, checked, "Failed to find metric")
 }
+
+// An abandoned reply must not keep the cluster lock held. synctest.Wait
+// ensures the handler has reached its blocking send before probing the lock.
+func TestRMEventReplyReleasesClusterLock(t *testing.T) {
+	const registrationSuccess = "registration-success"
+	for _, name := range []string{"registration-invalid", "registration-duplicate", registrationSuccess, "config-unregistered", "config-invalid", "config-unchanged", "remove"} {
+		for _, capacity := range []int{0, 1} {
+			t.Run(fmt.Sprintf("%s/buffer-%d", name, capacity), func(t *testing.T) {
+				oldConfigMap := configs.GetConfigMap()
+				defer configs.SetConfigMap(oldConfigMap)
+				synctest.Test(t, func(t *testing.T) {
+					cc := &ClusterContext{partitions: make(map[string]*PartitionContext), policyGroup: t.Name()}
+					defer configs.ConfigContext.Set(cc.policyGroup, nil)
+					ch := make(chan *rmevent.Result, capacity)
+					var handle func()
+					succeeded := false
+					switch name {
+					case "registration-invalid", "registration-duplicate", registrationSuccess:
+						config := "invalid: ["
+						if name == "registration-duplicate" {
+							cc.partitions["existing"] = nil
+						}
+						if name == registrationSuccess {
+							config = "partitions: []"
+							succeeded = true
+						}
+						handle = func() {
+							cc.processRMRegistrationEvent(&rmevent.RMRegistrationEvent{
+								Registration: &si.RegisterResourceManagerRequest{RmID: "test", PolicyGroup: cc.policyGroup, Config: config}, Channel: ch,
+							})
+						}
+					case "config-unregistered", "config-invalid", "config-unchanged":
+						config := "invalid: ["
+						if name != "config-unregistered" {
+							cc.partitions["existing"] = nil
+						}
+						if name == "config-unchanged" {
+							config = configs.DefaultSchedulerConfig
+							conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
+							assert.NilError(t, err)
+							configs.ConfigContext.Set(cc.policyGroup, conf)
+							succeeded = true
+						}
+						handle = func() {
+							cc.processRMConfigUpdateEvent(&rmevent.RMConfigUpdateEvent{RmID: "test", Config: config, Channel: ch})
+						}
+					case "remove":
+						succeeded = true
+						handle = func() { cc.removePartitionsByRMID(&rmevent.RMPartitionsRemoveEvent{RmID: "test", Channel: ch}) }
+					}
+					done := make(chan struct{})
+					go func() { handle(); close(done) }()
+					synctest.Wait()
+					if cc.TryRLock() {
+						cc.RUnlock()
+					} else {
+						t.Error("reply without a receiver holds the cluster lock")
+					}
+					if capacity == 1 {
+						select {
+						case <-done:
+						default:
+							t.Error("buffered reply did not let the handler finish without a receiver")
+						}
+					}
+					// Drain even on failure so the handler does not leak.
+					result := <-ch
+					<-done
+					assert.Equal(t, result.Succeeded, succeeded)
+					if name == registrationSuccess {
+						assert.Assert(t, configs.ConfigContext.Get(cc.policyGroup) != nil)
+						assert.Equal(t, cc.GetRMInfoMapClone()["test"].RMBuildInformation["rmId"], "test")
+					}
+				})
+			})
+		}
+	}
+}
+
+func TestRMConfigPublishedBeforeReply(t *testing.T) {
+	cc := createTestContext(t, "[test]default")
+	defer cc.Stop()
+	cc.policyGroup = t.Name()
+	oldConfigMap := configs.GetConfigMap()
+	defer configs.SetConfigMap(oldConfigMap)
+	defer configs.ConfigContext.Set(cc.policyGroup, nil)
+	synctest.Test(t, func(t *testing.T) {
+		ch := make(chan *rmevent.Result)
+		done := make(chan struct{})
+		go func() {
+			cc.processRMConfigUpdateEvent(&rmevent.RMConfigUpdateEvent{RmID: "test", Config: configs.DefaultSchedulerConfig, Channel: ch})
+			close(done)
+		}()
+		synctest.Wait()
+		assert.Check(t, configs.ConfigContext.Get(cc.policyGroup) != nil, "configuration must be published before waiting for the reply receiver")
+		result := <-ch
+		<-done
+		assert.Assert(t, result.Succeeded, result.Reason)
+	})
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: issues-help@yunikorn.apache.org