You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@yunikorn.apache.org by ti...@apache.org on 2026/09/29 16:12:17 UTC
(yunikorn-core) branch master updated: [YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)
This is an automated email from the ASF dual-hosted git repository.
HuangTing-Yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new d13826c7 [YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)
d13826c7 is described below
commit d13826c76b96f55929ddf3d4c494f8bcef530fca
Author: andyhuangdev <ky...@gmail.com>
AuthorDate: Wed Sep 30 00:12:01 2026 +0800
[YUNIKORN-3417] Send RM event replies after releasing the cluster lock (#1167)
- Buffer RM reply channels with capacity one.
- Complete configuration updates before replying.
- Add regression tests for abandoned replies and lock release.
Tests: scheduler and rmproxy suites pass with race and deadlock detection; lint passes with GOTOOLCHAIN=go1.26.0.
Closes: #1167
Signed-off-by: HuangTing-Yao <ti...@apache.org>
---
pkg/rmproxy/rmproxy.go | 8 +--
pkg/rmproxy/rmproxy_test.go | 60 +++++++++++++++++++++++
pkg/scheduler/context.go | 57 +++++++++++++---------
pkg/scheduler/context_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 209 insertions(+), 26 deletions(-)
diff --git a/pkg/rmproxy/rmproxy.go b/pkg/rmproxy/rmproxy.go
index 80a2ba09..70be0cc7 100644
--- a/pkg/rmproxy/rmproxy.go
+++ b/pkg/rmproxy/rmproxy.go
@@ -240,7 +240,8 @@ func drainReplyChannel(ch chan *rmevent.Result) {
func (rmp *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerRequest, callback api.ResourceManagerCallback) (*si.RegisterResourceManagerResponse, error) {
rmp.Lock()
defer rmp.Unlock()
- c := make(chan *rmevent.Result)
+ // Each event replies once; buffering lets the scheduler finish without a receiver.
+ c := make(chan *rmevent.Result, 1)
// If this is a re-register we need to clean up first
if rmp.rmIDToCallback[request.RmID] != nil {
@@ -259,7 +260,7 @@ func (rmp *RMProxy) RegisterResourceManager(request *si.RegisterResourceManagerR
}
}
- c = make(chan *rmevent.Result)
+ c = make(chan *rmevent.Result, 1)
// Add new RM.
go func() {
@@ -348,7 +349,8 @@ func (rmp *RMProxy) UpdateNode(request *si.NodeRequest) error {
// Triggers scheduler to reload configuration and apply the changes on-the-fly to the scheduler itself.
func (rmp *RMProxy) UpdateConfiguration(request *si.UpdateConfigurationRequest) error {
- c := make(chan *rmevent.Result)
+ // Each event replies once; buffering lets the scheduler finish without a receiver.
+ c := make(chan *rmevent.Result, 1)
go func() {
rmp.schedulerEventHandler.HandleEvent(&rmevent.RMConfigUpdateEvent{
RmID: request.RmID,
diff --git a/pkg/rmproxy/rmproxy_test.go b/pkg/rmproxy/rmproxy_test.go
index 9c98526f..efff00b7 100644
--- a/pkg/rmproxy/rmproxy_test.go
+++ b/pkg/rmproxy/rmproxy_test.go
@@ -19,15 +19,25 @@
package rmproxy
import (
+ "os"
"testing"
+ "testing/synctest"
"time"
+ godeadlock "github.com/sasha-s/go-deadlock"
"gotest.tools/v3/assert"
+ "github.com/apache/yunikorn-core/pkg/mock"
"github.com/apache/yunikorn-core/pkg/rmproxy/rmevent"
"github.com/apache/yunikorn-scheduler-interface/lib/go/si"
)
+func TestMain(m *testing.M) {
+ // Pooled deadlock timers cannot be reused across synctest bubbles.
+ godeadlock.Opts.TimerPool = godeadlock.TimerPoolDisabled
+ os.Exit(m.Run())
+}
+
func TestRMProxy_StopUnblocksWaitingCaller(t *testing.T) {
rmp := NewRMProxy(nil)
rmp.StartService()
@@ -83,3 +93,53 @@ func assertDrainFailedResult(t *testing.T, ch <-chan *rmevent.Result, name strin
t.Fatalf("timed out waiting for response on %s", name)
}
}
+
+type infraReplyHandler struct {
+ events chan interface{}
+}
+
+func (h *infraReplyHandler) HandleEvent(event interface{}) { h.events <- event }
+
+func TestRMProxyInfraRepliesBuffered(t *testing.T) {
+ for _, name := range []string{"registration", "reregistration", "config"} {
+ t.Run(name, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ handler := &infraReplyHandler{events: make(chan interface{})}
+ rmp := NewRMProxy(handler)
+ if name == "reregistration" {
+ rmp.rmIDToCallback["test"] = &mock.ResourceManagerCallback{}
+ }
+ done := make(chan error, 1)
+ go func() {
+ if name == "config" {
+ done <- rmp.UpdateConfiguration(&si.UpdateConfigurationRequest{RmID: "test"})
+ } else {
+ _, err := rmp.RegisterResourceManager(&si.RegisterResourceManagerRequest{RmID: "test"}, nil)
+ done <- err
+ }
+ }()
+ replies := 1
+ if name == "reregistration" {
+ replies = 2
+ }
+ for i := 0; i < replies; i++ {
+ event := <-handler.events
+ var ch chan *rmevent.Result
+ switch ev := event.(type) {
+ case *rmevent.RMRegistrationEvent:
+ ch = ev.Channel
+ case *rmevent.RMPartitionsRemoveEvent:
+ ch = ev.Channel
+ case *rmevent.RMConfigUpdateEvent:
+ ch = ev.Channel
+ default:
+ t.Fatalf("unexpected event %T", event)
+ }
+ assert.Check(t, cap(ch) == 1, "reply channel must hold one result without a receiver")
+ ch <- &rmevent.Result{Succeeded: true}
+ }
+ assert.NilError(t, <-done)
+ })
+ })
+ }
+}
diff --git a/pkg/scheduler/context.go b/pkg/scheduler/context.go
index c5ef60e1..78825cdb 100644
--- a/pkg/scheduler/context.go
+++ b/pkg/scheduler/context.go
@@ -157,17 +157,23 @@ func (cc *ClusterContext) schedule() bool {
}
func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrationEvent) {
+ // Complete all state changes and release the cluster lock before replying.
+ result := cc.processRMRegistrationEventResult(event)
+ event.Channel <- result
+}
+
+// processRMRegistrationEventResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) processRMRegistrationEventResult(event *rmevent.RMRegistrationEvent) *rmevent.Result {
cc.Lock()
defer cc.Unlock()
rmID := event.Registration.RmID
// we should not have any partitions set at this point
if len(cc.partitions) > 0 {
- event.Channel <- &rmevent.Result{
+ return &rmevent.Result{
Reason: fmt.Sprintf("RM %s has been registered before, active partitions %d", rmID, len(cc.partitions)),
Succeeded: false,
}
- return
}
policyGroup := event.Registration.PolicyGroup
config := event.Registration.Config
@@ -180,13 +186,11 @@ func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrati
}
conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
if err != nil {
- event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
- return
+ return &rmevent.Result{Succeeded: false, Reason: err.Error()}
}
err = cc.updateSchedulerConfig(conf, rmID)
if err != nil {
- event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
- return
+ return &rmevent.Result{Succeeded: false, Reason: err.Error()}
}
// update global scheduler configs, set the policyGroup for this cluster
@@ -196,23 +200,29 @@ func (cc *ClusterContext) processRMRegistrationEvent(event *rmevent.RMRegistrati
// store the build information of RM
cc.SetRMInfo(rmID, event.Registration.BuildInfo)
- // Done, notify channel
- event.Channel <- &rmevent.Result{
+ // Done, return the result to send after unlocking
+ return &rmevent.Result{
Succeeded: true,
}
}
func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpdateEvent) {
+ // Complete all state changes and release the cluster lock before replying.
+ result := cc.processRMConfigUpdateEventResult(event)
+ event.Channel <- result
+}
+
+// processRMConfigUpdateEventResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) processRMConfigUpdateEventResult(event *rmevent.RMConfigUpdateEvent) *rmevent.Result {
cc.Lock()
defer cc.Unlock()
rmID := event.RmID
// need to enhance the check to support multiple RMs
if len(cc.partitions) == 0 {
- event.Channel <- &rmevent.Result{
+ return &rmevent.Result{
Reason: fmt.Sprintf("RM %s has no active partitions, make sure it is registered", rmID),
Succeeded: false,
}
- return
}
// set extra configuration
@@ -226,8 +236,7 @@ func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpda
}
conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
if err != nil {
- event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
- return
+ return &rmevent.Result{Succeeded: false, Reason: err.Error()}
}
// skip update if the config has not changed: the checksum is calculated over the config again which lets
// us detect a real change even if the config map was updated for an unrelated reason
@@ -235,23 +244,18 @@ func (cc *ClusterContext) processRMConfigUpdateEvent(event *rmevent.RMConfigUpda
if oldConf != nil && conf.Checksum == oldConf.Checksum {
log.Log(log.SchedContext).Info("configuration checksum unchanged, skipping config update",
zap.String("rmID", rmID), zap.String("checksum", conf.Checksum))
- event.Channel <- &rmevent.Result{
+ return &rmevent.Result{
Succeeded: true,
}
- return
}
// update scheduler configuration
err = cc.updateSchedulerConfig(conf, rmID)
if err != nil {
- event.Channel <- &rmevent.Result{Succeeded: false, Reason: err.Error()}
- return
- }
- // Done, notify channel
- event.Channel <- &rmevent.Result{
- Succeeded: true,
+ return &rmevent.Result{Succeeded: false, Reason: err.Error()}
}
- // update global scheduler configs
+ // Publish the configuration before releasing the lock and replying.
configs.ConfigContext.Set(cc.policyGroup, conf)
+ return &rmevent.Result{Succeeded: true}
}
func (cc *ClusterContext) handleRMUpdateNodeEvent(event *rmevent.RMUpdateNodeEvent) {
@@ -311,6 +315,13 @@ func (cc *ClusterContext) processNodes(request *si.NodeRequest) {
// Called when a RM re-registers. This triggers a full clean up.
// Registration expects everything to be clean.
func (cc *ClusterContext) removePartitionsByRMID(event *rmevent.RMPartitionsRemoveEvent) {
+ // Complete all state changes and release the cluster lock before replying.
+ result := cc.removePartitionsByRMIDResult(event)
+ event.Channel <- result
+}
+
+// removePartitionsByRMIDResult updates cluster state under the lock without sending a reply.
+func (cc *ClusterContext) removePartitionsByRMIDResult(event *rmevent.RMPartitionsRemoveEvent) *rmevent.Result {
cc.Lock()
defer cc.Unlock()
partitionToRemove := make(map[string]bool)
@@ -326,8 +337,8 @@ func (cc *ClusterContext) removePartitionsByRMID(event *rmevent.RMPartitionsRemo
for partitionName := range partitionToRemove {
delete(cc.partitions, partitionName)
}
- // Done, notify channel
- event.Channel <- &rmevent.Result{
+ // Done, return the result to send after unlocking
+ return &rmevent.Result{
Succeeded: true,
}
}
diff --git a/pkg/scheduler/context_test.go b/pkg/scheduler/context_test.go
index e7ce92c5..a4fd6a3f 100644
--- a/pkg/scheduler/context_test.go
+++ b/pkg/scheduler/context_test.go
@@ -19,11 +19,15 @@
package scheduler
import (
+ "fmt"
+ "os"
"strings"
"testing"
+ "testing/synctest"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
+ godeadlock "github.com/sasha-s/go-deadlock"
"gotest.tools/v3/assert"
"github.com/apache/yunikorn-core/pkg/common/configs"
@@ -43,6 +47,12 @@ type mockEventHandler struct {
newAllocHandler func(*rmevent.RMNewAllocationsEvent)
}
+func TestMain(m *testing.M) {
+ // Pooled deadlock timers cannot be reused across synctest bubbles.
+ godeadlock.Opts.TimerPool = godeadlock.TimerPoolDisabled
+ os.Exit(m.Run())
+}
+
func newMockEventHandler() *mockEventHandler {
return &mockEventHandler{
eventHandled: false,
@@ -445,3 +455,103 @@ outer:
assert.Assert(t, checked, "Failed to find metric")
}
+
+// An abandoned reply must not keep the cluster lock held. synctest.Wait
+// ensures the handler has reached its blocking send before probing the lock.
+func TestRMEventReplyReleasesClusterLock(t *testing.T) {
+ const registrationSuccess = "registration-success"
+ for _, name := range []string{"registration-invalid", "registration-duplicate", registrationSuccess, "config-unregistered", "config-invalid", "config-unchanged", "remove"} {
+ for _, capacity := range []int{0, 1} {
+ t.Run(fmt.Sprintf("%s/buffer-%d", name, capacity), func(t *testing.T) {
+ oldConfigMap := configs.GetConfigMap()
+ defer configs.SetConfigMap(oldConfigMap)
+ synctest.Test(t, func(t *testing.T) {
+ cc := &ClusterContext{partitions: make(map[string]*PartitionContext), policyGroup: t.Name()}
+ defer configs.ConfigContext.Set(cc.policyGroup, nil)
+ ch := make(chan *rmevent.Result, capacity)
+ var handle func()
+ succeeded := false
+ switch name {
+ case "registration-invalid", "registration-duplicate", registrationSuccess:
+ config := "invalid: ["
+ if name == "registration-duplicate" {
+ cc.partitions["existing"] = nil
+ }
+ if name == registrationSuccess {
+ config = "partitions: []"
+ succeeded = true
+ }
+ handle = func() {
+ cc.processRMRegistrationEvent(&rmevent.RMRegistrationEvent{
+ Registration: &si.RegisterResourceManagerRequest{RmID: "test", PolicyGroup: cc.policyGroup, Config: config}, Channel: ch,
+ })
+ }
+ case "config-unregistered", "config-invalid", "config-unchanged":
+ config := "invalid: ["
+ if name != "config-unregistered" {
+ cc.partitions["existing"] = nil
+ }
+ if name == "config-unchanged" {
+ config = configs.DefaultSchedulerConfig
+ conf, err := configs.LoadSchedulerConfigFromByteArray([]byte(config))
+ assert.NilError(t, err)
+ configs.ConfigContext.Set(cc.policyGroup, conf)
+ succeeded = true
+ }
+ handle = func() {
+ cc.processRMConfigUpdateEvent(&rmevent.RMConfigUpdateEvent{RmID: "test", Config: config, Channel: ch})
+ }
+ case "remove":
+ succeeded = true
+ handle = func() { cc.removePartitionsByRMID(&rmevent.RMPartitionsRemoveEvent{RmID: "test", Channel: ch}) }
+ }
+ done := make(chan struct{})
+ go func() { handle(); close(done) }()
+ synctest.Wait()
+ if cc.TryRLock() {
+ cc.RUnlock()
+ } else {
+ t.Error("reply without a receiver holds the cluster lock")
+ }
+ if capacity == 1 {
+ select {
+ case <-done:
+ default:
+ t.Error("buffered reply did not let the handler finish without a receiver")
+ }
+ }
+ // Drain even on failure so the handler does not leak.
+ result := <-ch
+ <-done
+ assert.Equal(t, result.Succeeded, succeeded)
+ if name == registrationSuccess {
+ assert.Assert(t, configs.ConfigContext.Get(cc.policyGroup) != nil)
+ assert.Equal(t, cc.GetRMInfoMapClone()["test"].RMBuildInformation["rmId"], "test")
+ }
+ })
+ })
+ }
+ }
+}
+
+func TestRMConfigPublishedBeforeReply(t *testing.T) {
+ cc := createTestContext(t, "[test]default")
+ defer cc.Stop()
+ cc.policyGroup = t.Name()
+ oldConfigMap := configs.GetConfigMap()
+ defer configs.SetConfigMap(oldConfigMap)
+ defer configs.ConfigContext.Set(cc.policyGroup, nil)
+ synctest.Test(t, func(t *testing.T) {
+ ch := make(chan *rmevent.Result)
+ done := make(chan struct{})
+ go func() {
+ cc.processRMConfigUpdateEvent(&rmevent.RMConfigUpdateEvent{RmID: "test", Config: configs.DefaultSchedulerConfig, Channel: ch})
+ close(done)
+ }()
+ synctest.Wait()
+ assert.Check(t, configs.ConfigContext.Get(cc.policyGroup) != nil, "configuration must be published before waiting for the reply receiver")
+ result := <-ch
+ <-done
+ assert.Assert(t, result.Succeeded, result.Reason)
+ })
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: issues-help@yunikorn.apache.org