You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@yunikorn.apache.org by wi...@apache.org on 2024/02/23 02:48:56 UTC
(yunikorn-core) branch master updated: [YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)
This is an automated email from the ASF dual-hosted git repository.
wilfreds pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git
The following commit(s) were added to refs/heads/master by this push:
new 3148d2b8 [YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)
3148d2b8 is described below
commit 3148d2b8300c5d0b333a4d8f5d22bc05d4e33f4d
Author: Priyansh Choudhary <im...@gmail.com>
AuthorDate: Fri Feb 23 13:47:05 2024 +1100
[YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)
unifying allocationId to allocationID.
updating clusterId in PartitionNodesUtilDAOInfo
Closes: #806
Signed-off-by: Wilfred Spiegelenburg <wi...@apache.org>
---
pkg/scheduler/objects/node.go | 4 ++--
pkg/scheduler/partition.go | 42 ++++++++++++++++++++---------------------
pkg/scheduler/partition_test.go | 28 +++++++++++++--------------
pkg/webservice/dao/node_util.go | 2 +-
pkg/webservice/handlers.go | 2 +-
pkg/webservice/handlers_test.go | 6 +++---
6 files changed, 42 insertions(+), 42 deletions(-)
diff --git a/pkg/scheduler/objects/node.go b/pkg/scheduler/objects/node.go
index 7c5b8b92..edce1281 100644
--- a/pkg/scheduler/objects/node.go
+++ b/pkg/scheduler/objects/node.go
@@ -360,8 +360,8 @@ func (sn *Node) ReplaceAllocation(allocationID string, replace *Allocation, delt
sn.availableResource.SubFrom(delta)
if !before.FitIn(sn.allocatedResource) {
log.Log(log.SchedNode).Warn("unexpected increase in node usage after placeholder replacement",
- zap.String("placeholder allocationId", allocationID),
- zap.String("allocation allocationId", replace.GetAllocationID()),
+ zap.String("placeholder allocationID", allocationID),
+ zap.String("allocation allocationID", replace.GetAllocationID()),
zap.Stringer("delta", delta))
}
}
diff --git a/pkg/scheduler/partition.go b/pkg/scheduler/partition.go
index ab05972b..ec84c9f0 100644
--- a/pkg/scheduler/partition.go
+++ b/pkg/scheduler/partition.go
@@ -402,7 +402,7 @@ func (pc *PartitionContext) removeApplication(appID string) []*objects.Allocatio
if nodeAlloc := node.RemoveAllocation(currentAllocationID); nodeAlloc == nil {
log.Log(log.SchedPartition).Warn("unknown allocation: not found on the node",
zap.String("appID", appID),
- zap.String("allocationId", currentAllocationID),
+ zap.String("allocationID", currentAllocationID),
zap.String("nodeID", alloc.GetNodeID()))
}
}
@@ -748,9 +748,9 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
released = append(released, alloc)
log.Log(log.SchedPartition).Info("allocation removed from node and replacement confirmed",
zap.String("nodeID", node.NodeID),
- zap.String("allocationId", allocID),
+ zap.String("allocationID", allocID),
zap.String("replacement nodeID", release.GetNodeID()),
- zap.String("replacement allocationId", release.GetAllocationID()))
+ zap.String("replacement allocationID", release.GetAllocationID()))
continue
}
askAlloc = release
@@ -765,7 +765,7 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
zap.String("appID", askAlloc.GetApplicationID()),
zap.String("allocationKey", askAlloc.GetAsk().GetAllocationKey()),
zap.String("nodeID", node.NodeID),
- zap.String("replacement allocationId", askAlloc.GetAllocationID()))
+ zap.String("replacement allocationID", askAlloc.GetAllocationID()))
} else {
log.Log(log.SchedPartition).Error("node removal: repeat update failure for inflight replacement",
zap.String("appID", askAlloc.GetApplicationID()),
@@ -777,7 +777,7 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
// check allocations on the app
if app.RemoveAllocation(allocID, si.TerminationType_UNKNOWN_TERMINATION_TYPE) == nil {
log.Log(log.SchedPartition).Info("allocation is not found, skipping while removing the node",
- zap.String("allocationId", allocID),
+ zap.String("allocationID", allocID),
zap.String("appID", app.ApplicationID),
zap.String("nodeID", node.NodeID))
continue
@@ -867,8 +867,8 @@ func (pc *PartitionContext) tryPlaceholderAllocate() *objects.Allocation {
log.Log(log.SchedPartition).Info("scheduler replace placeholder processed",
zap.String("appID", alloc.GetApplicationID()),
zap.String("allocationKey", alloc.GetAllocationKey()),
- zap.String("allocationId", alloc.GetAllocationID()),
- zap.String("placeholder released allocationId", alloc.GetFirstRelease().GetAllocationID()))
+ zap.String("allocationID", alloc.GetAllocationID()),
+ zap.String("placeholder released allocationID", alloc.GetFirstRelease().GetAllocationID()))
// pass the release back to the RM via the cluster context
return alloc
}
@@ -931,7 +931,7 @@ func (pc *PartitionContext) allocate(alloc *objects.Allocation) *objects.Allocat
log.Log(log.SchedPartition).Info("scheduler allocation processed",
zap.String("appID", alloc.GetApplicationID()),
zap.String("allocationKey", alloc.GetAllocationKey()),
- zap.String("allocationId", alloc.GetAllocationID()),
+ zap.String("allocationID", alloc.GetAllocationID()),
zap.Stringer("allocatedResource", alloc.GetAllocatedResource()),
zap.Bool("placeholder", alloc.IsPlaceholder()),
zap.String("targetNode", alloc.GetNodeID()))
@@ -1137,11 +1137,11 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
return fmt.Errorf("partition %s is stopped cannot add new allocation %s", pc.Name, alloc.GetAllocationKey())
}
- // We must not generate a new allocationId for it, we directly use the allocationId reported by shim
- // to track this allocation, a missing allocationId is a broken allocation
+ // We must not generate a new allocationID for it, we directly use the allocationID reported by shim
+ // to track this allocation, a missing allocationID is a broken allocation
if alloc.GetAllocationID() == "" {
metrics.GetSchedulerMetrics().IncSchedulingError()
- return fmt.Errorf("failing to restore allocation %s for application %s: missing allocationId",
+ return fmt.Errorf("failing to restore allocation %s for application %s: missing allocationID",
alloc.GetAllocationKey(), alloc.GetApplicationID())
}
@@ -1149,7 +1149,7 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
zap.String("partitionName", pc.Name),
zap.String("appID", alloc.GetApplicationID()),
zap.String("allocKey", alloc.GetAllocationKey()),
- zap.String("allocationId", alloc.GetAllocationID()))
+ zap.String("allocationID", alloc.GetAllocationID()))
// Check if allocation violates any resource restriction, or allocate on a
// non-existent application or nodes.
@@ -1190,7 +1190,7 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
zap.String("partitionName", pc.Name),
zap.String("appID", alloc.GetApplicationID()),
zap.String("allocKey", alloc.GetAllocationKey()),
- zap.String("allocationId", alloc.GetAllocationID()),
+ zap.String("allocationID", alloc.GetAllocationID()),
zap.Bool("placeholder", alloc.IsPlaceholder()))
return nil
}
@@ -1257,7 +1257,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
if app == nil {
log.Log(log.SchedPartition).Info("Application not found while releasing allocation",
zap.String("appID", appID),
- zap.String("allocationId", allocationID),
+ zap.String("allocationID", allocationID),
zap.Stringer("terminationType", release.TerminationType))
return nil, nil
}
@@ -1270,7 +1270,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
// temp store for allocations manipulated
released := make([]*objects.Allocation, 0)
var confirmed *objects.Allocation
- // when allocationId is not specified, remove all allocations from the app
+ // when allocationID is not specified, remove all allocations from the app
if allocationID == "" {
log.Log(log.SchedPartition).Info("remove all allocations",
zap.String("appID", appID))
@@ -1280,14 +1280,14 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
if release.TerminationType == si.TerminationType_PLACEHOLDER_REPLACED {
log.Log(log.SchedPartition).Info("replacing placeholder allocation",
zap.String("appID", appID),
- zap.String("allocationId", allocationID))
+ zap.String("allocationID", allocationID))
if alloc := app.ReplaceAllocation(allocationID); alloc != nil {
released = append(released, alloc)
}
} else {
log.Log(log.SchedPartition).Info("removing allocation from application",
zap.String("appID", appID),
- zap.String("allocationId", allocationID),
+ zap.String("allocationID", allocationID),
zap.Stringer("terminationType", release.TerminationType))
if alloc := app.RemoveAllocation(allocationID, release.TerminationType); alloc != nil {
released = append(released, alloc)
@@ -1315,7 +1315,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
if node == nil {
log.Log(log.SchedPartition).Warn("node not found while releasing allocation",
zap.String("appID", appID),
- zap.String("allocationId", alloc.GetAllocationID()),
+ zap.String("allocationID", alloc.GetAllocationID()),
zap.String("nodeID", alloc.GetNodeID()))
continue
}
@@ -1346,14 +1346,14 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
}
log.Log(log.SchedPartition).Info("replacing placeholder allocation on node",
zap.String("nodeID", alloc.GetNodeID()),
- zap.String("allocationId", alloc.GetAllocationID()),
+ zap.String("allocationID", alloc.GetAllocationID()),
zap.String("allocation nodeID", confirmed.GetNodeID()))
} else if node.RemoveAllocation(alloc.GetAllocationID()) != nil {
// all non replacement are real removes: must update the queue usage
total.AddTo(alloc.GetAllocatedResource())
log.Log(log.SchedPartition).Info("removing allocation from node",
zap.String("nodeID", alloc.GetNodeID()),
- zap.String("allocationId", alloc.GetAllocationID()))
+ zap.String("allocationID", alloc.GetAllocationID()))
}
if alloc.IsPreempted() {
totalPreempting.AddTo(alloc.GetAllocatedResource())
@@ -1363,7 +1363,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
if err := queue.DecAllocatedResource(total); err != nil {
log.Log(log.SchedPartition).Warn("failed to release resources from queue",
zap.String("appID", appID),
- zap.String("allocationId", allocationID),
+ zap.String("allocationID", allocationID),
zap.Error(err))
}
}
diff --git a/pkg/scheduler/partition_test.go b/pkg/scheduler/partition_test.go
index ccfce2e5..14bef767 100644
--- a/pkg/scheduler/partition_test.go
+++ b/pkg/scheduler/partition_test.go
@@ -262,15 +262,15 @@ func TestAddNodeWithAllocations(t *testing.T) {
ask = newAllocationAsk("alloc-1-allocationid", appID1, appRes)
alloc = objects.NewAllocation(nodeID1, ask)
assert.Equal(t, alloc.GetAllocationID(), "alloc-1-allocationid-0")
- // reset allocationId to empty
+ // reset allocationID to empty
alloc.SetAllocationID("")
assert.Equal(t, alloc.GetAllocationID(), "")
allocs = []*objects.Allocation{alloc}
err = partition.AddNode(node, allocs)
if err == nil {
- t.Errorf("add node to partition should have failed (allocationId missing)")
+ t.Errorf("add node to partition should have failed (allocationID missing)")
}
- assert.Equal(t, partition.nodes.GetNodeCount(), 0, "error returned but node still added to the partition (allocationId)")
+ assert.Equal(t, partition.nodes.GetNodeCount(), 0, "error returned but node still added to the partition (allocationID)")
assertLimits(t, getTestUserGroup(), nil)
// fix the alloc add the node will work now
@@ -346,7 +346,7 @@ func TestRemoveNodeWithAllocations(t *testing.T) {
assert.Equal(t, 0, partition.GetTotalNodeCount(), "node list was not updated, node was not removed")
assert.Equal(t, 1, len(released), "node did not release correct allocation")
assert.Equal(t, 0, len(confirmed), "node did not confirm correct allocation")
- assert.Equal(t, released[0].GetAllocationID(), allocAllocationID, "allocationId returned by release not the same as on allocation")
+ assert.Equal(t, released[0].GetAllocationID(), allocAllocationID, "allocationID returned by release not the same as on allocation")
assertLimits(t, getTestUserGroup(), resources.Zero)
assert.NilError(t, err, "the event should have been processed")
@@ -404,7 +404,7 @@ func TestRemoveNodeWithPlaceholders(t *testing.T) {
assert.Equal(t, 0, partition.GetTotalNodeCount(), "node list was not updated, node was not removed")
assert.Equal(t, 1, len(released), "node removal did not release correct allocation")
assert.Equal(t, 0, len(confirmed), "node removal should not have confirmed allocation")
- assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationId returned by release not the same as the placeholder")
+ assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationID returned by release not the same as the placeholder")
assert.Equal(t, 0, partition.getPhAllocationCount(), "number of active placeholders")
allocs = app.GetAllAllocations()
assert.Equal(t, 0, len(allocs), "expected no allocations for the app")
@@ -813,13 +813,13 @@ func TestRemoveNodeWithReplacement(t *testing.T) {
assert.Equal(t, 1, len(node2.GetAllAllocations()), "remaining node should have allocation")
assert.Equal(t, 1, len(released), "node removal did not release correct allocation")
assert.Equal(t, 1, len(confirmed), "node removal did not confirm correct allocation")
- assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationId returned by release not the same as the placeholder")
- assert.Equal(t, alloc.GetAllocationID(), confirmed[0].GetAllocationID(), "allocationId returned by confirmed not the same as the real allocation")
+ assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationID returned by release not the same as the placeholder")
+ assert.Equal(t, alloc.GetAllocationID(), confirmed[0].GetAllocationID(), "allocationID returned by confirmed not the same as the real allocation")
assert.Assert(t, resources.IsZero(app.GetPendingResource()), "app should not have pending resources")
assert.Assert(t, !app.IsCompleting(), "app should not be COMPLETING after confirming allocation")
allocs = app.GetAllAllocations()
assert.Equal(t, 1, len(allocs), "expected one allocation for the app (real)")
- assert.Equal(t, alloc.GetAllocationID(), allocs[0].GetAllocationID(), "allocationId for the app is not the same as the real allocation")
+ assert.Equal(t, alloc.GetAllocationID(), allocs[0].GetAllocationID(), "allocationID for the app is not the same as the real allocation")
assert.Equal(t, objects.Allocated, allocs[0].GetResult(), "allocation state should be allocated")
assert.Equal(t, 0, allocs[0].GetReleaseCount(), "real allocation should have no releases linked anymore")
assertLimits(t, getTestUserGroup(), appRes)
@@ -886,7 +886,7 @@ func TestRemoveNodeWithReal(t *testing.T) {
assert.Assert(t, resources.Equals(app.GetPendingResource(), appRes), "app should have updated pending resources")
allocs = app.GetAllAllocations()
assert.Equal(t, 1, len(allocs), "expected one allocation for the app (placeholder")
- assert.Equal(t, ph.GetAllocationID(), allocs[0].GetAllocationID(), "allocationId for the app is not the same as the real allocation")
+ assert.Equal(t, ph.GetAllocationID(), allocs[0].GetAllocationID(), "allocationID for the app is not the same as the real allocation")
assert.Equal(t, 0, ph.GetReleaseCount(), "no inflight replacements linked")
assertLimits(t, getTestUserGroup(), appRes)
}
@@ -1174,7 +1174,7 @@ func TestRemoveAppAllocs(t *testing.T) {
assertLimits(t, getTestUserGroup(), appRes)
ask = newAllocationAsk("alloc-1", appNotRemoved, appRes)
- allocationId := "alloc-1-0"
+ allocationID := "alloc-1-0"
alloc = objects.NewAllocation(nodeID1, ask)
err = partition.addAllocation(alloc)
assert.NilError(t, err, "add allocation to partition should not have failed")
@@ -1202,12 +1202,12 @@ func TestRemoveAppAllocs(t *testing.T) {
assertLimits(t, getTestUserGroup(), resources.Multiply(appRes, 2))
// create a new release with app, existing allocation: should return 1 alloc
assert.Equal(t, 2, partition.GetTotalAllocationCount(), "pre-remove allocation list incorrect: %v", partition.allocations)
- release.AllocationID = allocationId
+ release.AllocationID = allocationID
allocs, _ = partition.removeAllocation(release)
assert.Equal(t, 1, len(allocs), "removal request for existing allocation returned wrong allocations: %v", allocs)
assert.Equal(t, 1, partition.GetTotalAllocationCount(), "allocation removal requests removed more than expected: %v", partition.allocations)
assertLimits(t, getTestUserGroup(), resources.Multiply(appRes, 1))
- // create a new release with app, no allocationId: should return last left alloc
+ // create a new release with app, no allocationID: should return last left alloc
release.AllocationID = ""
allocs, _ = partition.removeAllocation(release)
assert.Equal(t, 1, len(allocs), "removal request for existing allocation returned wrong allocations: %v", allocs)
@@ -2173,7 +2173,7 @@ func setupPreemptionForRequiredNode(t *testing.T) (*PartitionContext, *objects.A
assert.Equal(t, alloc.GetApplicationID(), appID1, "expected application app-1 to be allocated")
assert.Equal(t, alloc.GetAllocationKey(), allocID, "expected ask alloc-1 to be allocated")
assertUserGroupResourceMaxLimits(t, getTestUserGroup(), resources.NewResourceFromMap(map[string]resources.Quantity{"vcore": 8000}), getExpectedQueuesLimitsForPreemptionWithRequiredNode())
- allocationId := alloc.GetAllocationID()
+ allocationID := alloc.GetAllocationID()
// required node set on ask
ask2 := newAllocationAsk(allocID2, appID1, res)
@@ -2210,7 +2210,7 @@ func setupPreemptionForRequiredNode(t *testing.T) (*PartitionContext, *objects.A
release := &si.AllocationRelease{
PartitionName: partition.Name,
ApplicationID: appID1,
- AllocationID: allocationId,
+ AllocationID: allocationID,
TerminationType: si.TerminationType_PREEMPTED_BY_SCHEDULER,
}
releases, _ := partition.removeAllocation(release)
diff --git a/pkg/webservice/dao/node_util.go b/pkg/webservice/dao/node_util.go
index 46ad7cd5..678f8dfa 100644
--- a/pkg/webservice/dao/node_util.go
+++ b/pkg/webservice/dao/node_util.go
@@ -19,7 +19,7 @@
package dao
type PartitionNodesUtilDAOInfo struct {
- ClusterId string `json:"clusterId"` // no omitempty, cluster id should not be empty
+ ClusterID string `json:"clusterId"` // no omitempty, cluster id should not be empty
Partition string `json:"partition"` // no omitempty, partition should not be empty
NodesUtilList []*NodesUtilDAOInfo `json:"utilizations,omitempty"`
}
diff --git a/pkg/webservice/handlers.go b/pkg/webservice/handlers.go
index 80feb68f..2d633460 100644
--- a/pkg/webservice/handlers.go
+++ b/pkg/webservice/handlers.go
@@ -514,7 +514,7 @@ func getPartitionNodesUtilJSON(partition *scheduler.PartitionContext) *dao.Parti
}
return &dao.PartitionNodesUtilDAOInfo{
- ClusterId: partition.RmID,
+ ClusterID: partition.RmID,
Partition: common.GetPartitionNameWithoutClusterID(partition.Name),
NodesUtilList: nodesUtilList,
}
diff --git a/pkg/webservice/handlers_test.go b/pkg/webservice/handlers_test.go
index 462cab80..9cf7633c 100644
--- a/pkg/webservice/handlers_test.go
+++ b/pkg/webservice/handlers_test.go
@@ -822,7 +822,7 @@ func TestGetPartitionNodesUtilJSON(t *testing.T) {
// assert partition nodes utilization
result := getPartitionNodesUtilJSON(partition)
- assert.Equal(t, result.ClusterId, rmID)
+ assert.Equal(t, result.ClusterID, rmID)
assert.Equal(t, result.Partition, "default")
assert.Equal(t, len(result.NodesUtilList), 3, "Should have 3 resource types(CPU/Memory/GPU) in the list.")
@@ -884,8 +884,8 @@ func TestGetNodeUtilisations(t *testing.T) {
err = json.Unmarshal(resp.outputBytes, &partitionNodesUtilDAOInfo)
assert.NilError(t, err, "should decode a list of *dao.PartitionNodesUtilDAOInfo")
assert.Equal(t, len(partitionNodesUtilDAOInfo), 2)
- assert.Equal(t, partitionNodesUtilDAOInfo[0].ClusterId, rmID)
- assert.Equal(t, partitionNodesUtilDAOInfo[1].ClusterId, rmID)
+ assert.Equal(t, partitionNodesUtilDAOInfo[0].ClusterID, rmID)
+ assert.Equal(t, partitionNodesUtilDAOInfo[1].ClusterID, rmID)
defaultPartitionNodesUtilDAOInfo := partitionNodesUtilDAOInfo[0]
gpuPartitionNodesUtilDAOInfo := partitionNodesUtilDAOInfo[1]
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: issues-help@yunikorn.apache.org