You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@yunikorn.apache.org by "Peter Bacsko (Jira)" <ji...@apache.org> on 2024/01/02 17:17:00 UTC
[jira] [Created] (YUNIKORN-2303) Data race during queue sorting
Peter Bacsko created YUNIKORN-2303:
--------------------------------------
Summary: Data race during queue sorting
Key: YUNIKORN-2303
URL: https://issues.apache.org/jira/browse/YUNIKORN-2303
Project: Apache YuniKorn
Issue Type: Bug
Components: core - scheduler
Reporter: Peter Bacsko
The following data race has been uncovered while running a performance test:
{noformat}
==================
WARNING: DATA RACE
Read at 0x00c02179b5a8 by goroutine 51:
github.com/apache/yunikorn-core/pkg/common/resources.(*Resource).Clone()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/common/resources/resources.go:142 +0x304
github.com/apache/yunikorn-core/pkg/common/resources.Sub()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/common/resources/resources.go:349 +0xe0
github.com/apache/yunikorn-core/pkg/scheduler/objects.sortQueue.sortQueuesByPriorityAndFairness.func1()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/sorters.go:71 +0x17a
sort.insertionSort_func()
/snap/go/current/src/sort/zsortfunc.go:12 +0xc6
sort.stable_func()
/snap/go/current/src/sort/zsortfunc.go:343 +0x7e
sort.SliceStable()
/snap/go/current/src/sort/slice.go:38 +0x1b9
github.com/apache/yunikorn-core/pkg/scheduler/objects.sortQueuesByPriorityAndFairness()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/sorters.go:57 +0x104
github.com/apache/yunikorn-core/pkg/scheduler/objects.sortQueue()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/sorters.go:34 +0x7f
github.com/apache/yunikorn-core/pkg/scheduler/objects.(*Queue).sortQueues()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/queue.go:1174 +0x2e7
github.com/apache/yunikorn-core/pkg/scheduler/objects.(*Queue).TryAllocate()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/queue.go:1377 +0xdd
github.com/apache/yunikorn-core/pkg/scheduler.(*PartitionContext).tryAllocate()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/partition.go:825 +0x1f6
github.com/apache/yunikorn-core/pkg/scheduler.(*ClusterContext).schedule()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/context.go:140 +0x1a8
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).internalSchedule()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:75 +0xf6
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).StartService.func2()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:66 +0x33
Previous write at 0x00c02179b5a8 by goroutine 48:
github.com/apache/yunikorn-core/pkg/common/resources.(*Resource).Clone()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/common/resources/resources.go:143 +0x353
github.com/apache/yunikorn-core/pkg/common/resources.Add()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/common/resources/resources.go:329 +0xe0
github.com/apache/yunikorn-core/pkg/scheduler/objects.(*Queue).incPendingResource()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/queue.go:688 +0x11c
github.com/apache/yunikorn-core/pkg/scheduler/objects.(*Application).AddAllocationAsk()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/objects/application.go:658 +0x9ca
github.com/apache/yunikorn-core/pkg/scheduler.(*PartitionContext).addAllocationAsk()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/partition.go:1439 +0x91
github.com/apache/yunikorn-core/pkg/scheduler.(*ClusterContext).processAsks()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/context.go:826 +0x132
github.com/apache/yunikorn-core/pkg/scheduler.(*ClusterContext).handleRMUpdateAllocationEvent()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/context.go:743 +0x9d
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).handleRMEvent()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:111 +0x1fa
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).StartService.func1()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:55 +0x33
Goroutine 51 (running) created at:
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).StartService()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:66 +0x447
github.com/apache/yunikorn-core/pkg/entrypoint.startAllServicesWithParameters()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/entrypoint/entrypoint.go:92 +0x35b
github.com/apache/yunikorn-core/pkg/entrypoint.StartAllServices()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/entrypoint/entrypoint.go:43 +0x59
github.com/apache/yunikorn-k8shim/pkg/shim.(*MockScheduler).init()
/home/bacskop/repos/incubator-yunikorn-k8shim/pkg/shim/scheduler_mock_test.go:64 +0xad
github.com/apache/yunikorn-k8shim/pkg/shim.BenchmarkSchedulingThroughPut()
/home/bacskop/repos/incubator-yunikorn-k8shim/pkg/shim/scheduler_perf_test.go:82 +0x1c4
testing.(*B).runN()
/snap/go/current/src/testing/benchmark.go:193 +0x232
testing.(*B).run1.func1()
/snap/go/current/src/testing/benchmark.go:233 +0x7b
Goroutine 48 (running) created at:
github.com/apache/yunikorn-core/pkg/scheduler.(*Scheduler).StartService()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/scheduler/scheduler.go:55 +0x153
github.com/apache/yunikorn-core/pkg/entrypoint.startAllServicesWithParameters()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/entrypoint/entrypoint.go:92 +0x35b
github.com/apache/yunikorn-core/pkg/entrypoint.StartAllServices()
/home/bacskop/go/pkg/mod/github.com/apache/yunikorn-core@v0.0.0-20231203141034-3ae625bcfc07/pkg/entrypoint/entrypoint.go:43 +0x59
github.com/apache/yunikorn-k8shim/pkg/shim.(*MockScheduler).init()
/home/bacskop/repos/incubator-yunikorn-k8shim/pkg/shim/scheduler_mock_test.go:64 +0xad
github.com/apache/yunikorn-k8shim/pkg/shim.BenchmarkSchedulingThroughPut()
/home/bacskop/repos/incubator-yunikorn-k8shim/pkg/shim/scheduler_perf_test.go:82 +0x1c4
testing.(*B).runN()
/snap/go/current/src/testing/benchmark.go:193 +0x232
testing.(*B).run1.func1()
/snap/go/current/src/testing/benchmark.go:233 +0x7b
==================
{noformat}
We must not access {{Queue.pending}} directly:
{noformat}
if comp == 0 {
return resources.StrictlyGreaterThan(resources.Sub(l.pending, r.pending), resources.Zero)
}
...
if lPriority < rPriority {
return false
}
return resources.StrictlyGreaterThan(resources.Sub(l.pending, r.pending), resources.Zero)
{noformat}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: dev-help@yunikorn.apache.org