You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@yunikorn.apache.org by wi...@apache.org on 2024/02/23 02:48:56 UTC

(yunikorn-core) branch master updated: [YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)

This is an automated email from the ASF dual-hosted git repository.

wilfreds pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-core.git


The following commit(s) were added to refs/heads/master by this push:
     new 3148d2b8 [YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)
3148d2b8 is described below

commit 3148d2b8300c5d0b333a4d8f5d22bc05d4e33f4d
Author: Priyansh Choudhary <im...@gmail.com>
AuthorDate: Fri Feb 23 13:47:05 2024 +1100

    [YUNIKORN-2413] Initialism fixes for allocationId, clusterId (#806)
    
    unifying allocationId to allocationID.
    updating clusterId in PartitionNodesUtilDAOInfo
    
    Closes: #806
    
    Signed-off-by: Wilfred Spiegelenburg <wi...@apache.org>
---
 pkg/scheduler/objects/node.go   |  4 ++--
 pkg/scheduler/partition.go      | 42 ++++++++++++++++++++---------------------
 pkg/scheduler/partition_test.go | 28 +++++++++++++--------------
 pkg/webservice/dao/node_util.go |  2 +-
 pkg/webservice/handlers.go      |  2 +-
 pkg/webservice/handlers_test.go |  6 +++---
 6 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/pkg/scheduler/objects/node.go b/pkg/scheduler/objects/node.go
index 7c5b8b92..edce1281 100644
--- a/pkg/scheduler/objects/node.go
+++ b/pkg/scheduler/objects/node.go
@@ -360,8 +360,8 @@ func (sn *Node) ReplaceAllocation(allocationID string, replace *Allocation, delt
 	sn.availableResource.SubFrom(delta)
 	if !before.FitIn(sn.allocatedResource) {
 		log.Log(log.SchedNode).Warn("unexpected increase in node usage after placeholder replacement",
-			zap.String("placeholder allocationId", allocationID),
-			zap.String("allocation allocationId", replace.GetAllocationID()),
+			zap.String("placeholder allocationID", allocationID),
+			zap.String("allocation allocationID", replace.GetAllocationID()),
 			zap.Stringer("delta", delta))
 	}
 }
diff --git a/pkg/scheduler/partition.go b/pkg/scheduler/partition.go
index ab05972b..ec84c9f0 100644
--- a/pkg/scheduler/partition.go
+++ b/pkg/scheduler/partition.go
@@ -402,7 +402,7 @@ func (pc *PartitionContext) removeApplication(appID string) []*objects.Allocatio
 			if nodeAlloc := node.RemoveAllocation(currentAllocationID); nodeAlloc == nil {
 				log.Log(log.SchedPartition).Warn("unknown allocation: not found on the node",
 					zap.String("appID", appID),
-					zap.String("allocationId", currentAllocationID),
+					zap.String("allocationID", currentAllocationID),
 					zap.String("nodeID", alloc.GetNodeID()))
 			}
 		}
@@ -748,9 +748,9 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
 					released = append(released, alloc)
 					log.Log(log.SchedPartition).Info("allocation removed from node and replacement confirmed",
 						zap.String("nodeID", node.NodeID),
-						zap.String("allocationId", allocID),
+						zap.String("allocationID", allocID),
 						zap.String("replacement nodeID", release.GetNodeID()),
-						zap.String("replacement allocationId", release.GetAllocationID()))
+						zap.String("replacement allocationID", release.GetAllocationID()))
 					continue
 				}
 				askAlloc = release
@@ -765,7 +765,7 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
 					zap.String("appID", askAlloc.GetApplicationID()),
 					zap.String("allocationKey", askAlloc.GetAsk().GetAllocationKey()),
 					zap.String("nodeID", node.NodeID),
-					zap.String("replacement allocationId", askAlloc.GetAllocationID()))
+					zap.String("replacement allocationID", askAlloc.GetAllocationID()))
 			} else {
 				log.Log(log.SchedPartition).Error("node removal: repeat update failure for inflight replacement",
 					zap.String("appID", askAlloc.GetApplicationID()),
@@ -777,7 +777,7 @@ func (pc *PartitionContext) removeNodeAllocations(node *objects.Node) ([]*object
 		// check allocations on the app
 		if app.RemoveAllocation(allocID, si.TerminationType_UNKNOWN_TERMINATION_TYPE) == nil {
 			log.Log(log.SchedPartition).Info("allocation is not found, skipping while removing the node",
-				zap.String("allocationId", allocID),
+				zap.String("allocationID", allocID),
 				zap.String("appID", app.ApplicationID),
 				zap.String("nodeID", node.NodeID))
 			continue
@@ -867,8 +867,8 @@ func (pc *PartitionContext) tryPlaceholderAllocate() *objects.Allocation {
 		log.Log(log.SchedPartition).Info("scheduler replace placeholder processed",
 			zap.String("appID", alloc.GetApplicationID()),
 			zap.String("allocationKey", alloc.GetAllocationKey()),
-			zap.String("allocationId", alloc.GetAllocationID()),
-			zap.String("placeholder released allocationId", alloc.GetFirstRelease().GetAllocationID()))
+			zap.String("allocationID", alloc.GetAllocationID()),
+			zap.String("placeholder released allocationID", alloc.GetFirstRelease().GetAllocationID()))
 		// pass the release back to the RM via the cluster context
 		return alloc
 	}
@@ -931,7 +931,7 @@ func (pc *PartitionContext) allocate(alloc *objects.Allocation) *objects.Allocat
 	log.Log(log.SchedPartition).Info("scheduler allocation processed",
 		zap.String("appID", alloc.GetApplicationID()),
 		zap.String("allocationKey", alloc.GetAllocationKey()),
-		zap.String("allocationId", alloc.GetAllocationID()),
+		zap.String("allocationID", alloc.GetAllocationID()),
 		zap.Stringer("allocatedResource", alloc.GetAllocatedResource()),
 		zap.Bool("placeholder", alloc.IsPlaceholder()),
 		zap.String("targetNode", alloc.GetNodeID()))
@@ -1137,11 +1137,11 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
 		return fmt.Errorf("partition %s is stopped cannot add new allocation %s", pc.Name, alloc.GetAllocationKey())
 	}
 
-	// We must not generate a new allocationId for it, we directly use the allocationId reported by shim
-	// to track this allocation, a missing allocationId is a broken allocation
+	// We must not generate a new allocationID for it, we directly use the allocationID reported by shim
+	// to track this allocation, a missing allocationID is a broken allocation
 	if alloc.GetAllocationID() == "" {
 		metrics.GetSchedulerMetrics().IncSchedulingError()
-		return fmt.Errorf("failing to restore allocation %s for application %s: missing allocationId",
+		return fmt.Errorf("failing to restore allocation %s for application %s: missing allocationID",
 			alloc.GetAllocationKey(), alloc.GetApplicationID())
 	}
 
@@ -1149,7 +1149,7 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
 		zap.String("partitionName", pc.Name),
 		zap.String("appID", alloc.GetApplicationID()),
 		zap.String("allocKey", alloc.GetAllocationKey()),
-		zap.String("allocationId", alloc.GetAllocationID()))
+		zap.String("allocationID", alloc.GetAllocationID()))
 
 	// Check if allocation violates any resource restriction, or allocate on a
 	// non-existent application or nodes.
@@ -1190,7 +1190,7 @@ func (pc *PartitionContext) addAllocation(alloc *objects.Allocation) error {
 		zap.String("partitionName", pc.Name),
 		zap.String("appID", alloc.GetApplicationID()),
 		zap.String("allocKey", alloc.GetAllocationKey()),
-		zap.String("allocationId", alloc.GetAllocationID()),
+		zap.String("allocationID", alloc.GetAllocationID()),
 		zap.Bool("placeholder", alloc.IsPlaceholder()))
 	return nil
 }
@@ -1257,7 +1257,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 	if app == nil {
 		log.Log(log.SchedPartition).Info("Application not found while releasing allocation",
 			zap.String("appID", appID),
-			zap.String("allocationId", allocationID),
+			zap.String("allocationID", allocationID),
 			zap.Stringer("terminationType", release.TerminationType))
 		return nil, nil
 	}
@@ -1270,7 +1270,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 	// temp store for allocations manipulated
 	released := make([]*objects.Allocation, 0)
 	var confirmed *objects.Allocation
-	// when allocationId is not specified, remove all allocations from the app
+	// when allocationID is not specified, remove all allocations from the app
 	if allocationID == "" {
 		log.Log(log.SchedPartition).Info("remove all allocations",
 			zap.String("appID", appID))
@@ -1280,14 +1280,14 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 		if release.TerminationType == si.TerminationType_PLACEHOLDER_REPLACED {
 			log.Log(log.SchedPartition).Info("replacing placeholder allocation",
 				zap.String("appID", appID),
-				zap.String("allocationId", allocationID))
+				zap.String("allocationID", allocationID))
 			if alloc := app.ReplaceAllocation(allocationID); alloc != nil {
 				released = append(released, alloc)
 			}
 		} else {
 			log.Log(log.SchedPartition).Info("removing allocation from application",
 				zap.String("appID", appID),
-				zap.String("allocationId", allocationID),
+				zap.String("allocationID", allocationID),
 				zap.Stringer("terminationType", release.TerminationType))
 			if alloc := app.RemoveAllocation(allocationID, release.TerminationType); alloc != nil {
 				released = append(released, alloc)
@@ -1315,7 +1315,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 		if node == nil {
 			log.Log(log.SchedPartition).Warn("node not found while releasing allocation",
 				zap.String("appID", appID),
-				zap.String("allocationId", alloc.GetAllocationID()),
+				zap.String("allocationID", alloc.GetAllocationID()),
 				zap.String("nodeID", alloc.GetNodeID()))
 			continue
 		}
@@ -1346,14 +1346,14 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 			}
 			log.Log(log.SchedPartition).Info("replacing placeholder allocation on node",
 				zap.String("nodeID", alloc.GetNodeID()),
-				zap.String("allocationId", alloc.GetAllocationID()),
+				zap.String("allocationID", alloc.GetAllocationID()),
 				zap.String("allocation nodeID", confirmed.GetNodeID()))
 		} else if node.RemoveAllocation(alloc.GetAllocationID()) != nil {
 			// all non replacement are real removes: must update the queue usage
 			total.AddTo(alloc.GetAllocatedResource())
 			log.Log(log.SchedPartition).Info("removing allocation from node",
 				zap.String("nodeID", alloc.GetNodeID()),
-				zap.String("allocationId", alloc.GetAllocationID()))
+				zap.String("allocationID", alloc.GetAllocationID()))
 		}
 		if alloc.IsPreempted() {
 			totalPreempting.AddTo(alloc.GetAllocatedResource())
@@ -1363,7 +1363,7 @@ func (pc *PartitionContext) removeAllocation(release *si.AllocationRelease) ([]*
 		if err := queue.DecAllocatedResource(total); err != nil {
 			log.Log(log.SchedPartition).Warn("failed to release resources from queue",
 				zap.String("appID", appID),
-				zap.String("allocationId", allocationID),
+				zap.String("allocationID", allocationID),
 				zap.Error(err))
 		}
 	}
diff --git a/pkg/scheduler/partition_test.go b/pkg/scheduler/partition_test.go
index ccfce2e5..14bef767 100644
--- a/pkg/scheduler/partition_test.go
+++ b/pkg/scheduler/partition_test.go
@@ -262,15 +262,15 @@ func TestAddNodeWithAllocations(t *testing.T) {
 	ask = newAllocationAsk("alloc-1-allocationid", appID1, appRes)
 	alloc = objects.NewAllocation(nodeID1, ask)
 	assert.Equal(t, alloc.GetAllocationID(), "alloc-1-allocationid-0")
-	// reset allocationId to empty
+	// reset allocationID to empty
 	alloc.SetAllocationID("")
 	assert.Equal(t, alloc.GetAllocationID(), "")
 	allocs = []*objects.Allocation{alloc}
 	err = partition.AddNode(node, allocs)
 	if err == nil {
-		t.Errorf("add node to partition should have failed (allocationId missing)")
+		t.Errorf("add node to partition should have failed (allocationID missing)")
 	}
-	assert.Equal(t, partition.nodes.GetNodeCount(), 0, "error returned but node still added to the partition (allocationId)")
+	assert.Equal(t, partition.nodes.GetNodeCount(), 0, "error returned but node still added to the partition (allocationID)")
 	assertLimits(t, getTestUserGroup(), nil)
 
 	// fix the alloc add the node will work now
@@ -346,7 +346,7 @@ func TestRemoveNodeWithAllocations(t *testing.T) {
 	assert.Equal(t, 0, partition.GetTotalNodeCount(), "node list was not updated, node was not removed")
 	assert.Equal(t, 1, len(released), "node did not release correct allocation")
 	assert.Equal(t, 0, len(confirmed), "node did not confirm correct allocation")
-	assert.Equal(t, released[0].GetAllocationID(), allocAllocationID, "allocationId returned by release not the same as on allocation")
+	assert.Equal(t, released[0].GetAllocationID(), allocAllocationID, "allocationID returned by release not the same as on allocation")
 	assertLimits(t, getTestUserGroup(), resources.Zero)
 
 	assert.NilError(t, err, "the event should have been processed")
@@ -404,7 +404,7 @@ func TestRemoveNodeWithPlaceholders(t *testing.T) {
 	assert.Equal(t, 0, partition.GetTotalNodeCount(), "node list was not updated, node was not removed")
 	assert.Equal(t, 1, len(released), "node removal did not release correct allocation")
 	assert.Equal(t, 0, len(confirmed), "node removal should not have confirmed allocation")
-	assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationId returned by release not the same as the placeholder")
+	assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationID returned by release not the same as the placeholder")
 	assert.Equal(t, 0, partition.getPhAllocationCount(), "number of active placeholders")
 	allocs = app.GetAllAllocations()
 	assert.Equal(t, 0, len(allocs), "expected no allocations for the app")
@@ -813,13 +813,13 @@ func TestRemoveNodeWithReplacement(t *testing.T) {
 	assert.Equal(t, 1, len(node2.GetAllAllocations()), "remaining node should have allocation")
 	assert.Equal(t, 1, len(released), "node removal did not release correct allocation")
 	assert.Equal(t, 1, len(confirmed), "node removal did not confirm correct allocation")
-	assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationId returned by release not the same as the placeholder")
-	assert.Equal(t, alloc.GetAllocationID(), confirmed[0].GetAllocationID(), "allocationId returned by confirmed not the same as the real allocation")
+	assert.Equal(t, ph.GetAllocationID(), released[0].GetAllocationID(), "allocationID returned by release not the same as the placeholder")
+	assert.Equal(t, alloc.GetAllocationID(), confirmed[0].GetAllocationID(), "allocationID returned by confirmed not the same as the real allocation")
 	assert.Assert(t, resources.IsZero(app.GetPendingResource()), "app should not have pending resources")
 	assert.Assert(t, !app.IsCompleting(), "app should not be COMPLETING after confirming allocation")
 	allocs = app.GetAllAllocations()
 	assert.Equal(t, 1, len(allocs), "expected one allocation for the app (real)")
-	assert.Equal(t, alloc.GetAllocationID(), allocs[0].GetAllocationID(), "allocationId for the app is not the same as the real allocation")
+	assert.Equal(t, alloc.GetAllocationID(), allocs[0].GetAllocationID(), "allocationID for the app is not the same as the real allocation")
 	assert.Equal(t, objects.Allocated, allocs[0].GetResult(), "allocation state should be allocated")
 	assert.Equal(t, 0, allocs[0].GetReleaseCount(), "real allocation should have no releases linked anymore")
 	assertLimits(t, getTestUserGroup(), appRes)
@@ -886,7 +886,7 @@ func TestRemoveNodeWithReal(t *testing.T) {
 	assert.Assert(t, resources.Equals(app.GetPendingResource(), appRes), "app should have updated pending resources")
 	allocs = app.GetAllAllocations()
 	assert.Equal(t, 1, len(allocs), "expected one allocation for the app (placeholder")
-	assert.Equal(t, ph.GetAllocationID(), allocs[0].GetAllocationID(), "allocationId for the app is not the same as the real allocation")
+	assert.Equal(t, ph.GetAllocationID(), allocs[0].GetAllocationID(), "allocationID for the app is not the same as the real allocation")
 	assert.Equal(t, 0, ph.GetReleaseCount(), "no inflight replacements linked")
 	assertLimits(t, getTestUserGroup(), appRes)
 }
@@ -1174,7 +1174,7 @@ func TestRemoveAppAllocs(t *testing.T) {
 	assertLimits(t, getTestUserGroup(), appRes)
 
 	ask = newAllocationAsk("alloc-1", appNotRemoved, appRes)
-	allocationId := "alloc-1-0"
+	allocationID := "alloc-1-0"
 	alloc = objects.NewAllocation(nodeID1, ask)
 	err = partition.addAllocation(alloc)
 	assert.NilError(t, err, "add allocation to partition should not have failed")
@@ -1202,12 +1202,12 @@ func TestRemoveAppAllocs(t *testing.T) {
 	assertLimits(t, getTestUserGroup(), resources.Multiply(appRes, 2))
 	// create a new release with app, existing allocation: should return 1 alloc
 	assert.Equal(t, 2, partition.GetTotalAllocationCount(), "pre-remove allocation list incorrect: %v", partition.allocations)
-	release.AllocationID = allocationId
+	release.AllocationID = allocationID
 	allocs, _ = partition.removeAllocation(release)
 	assert.Equal(t, 1, len(allocs), "removal request for existing allocation returned wrong allocations: %v", allocs)
 	assert.Equal(t, 1, partition.GetTotalAllocationCount(), "allocation removal requests removed more than expected: %v", partition.allocations)
 	assertLimits(t, getTestUserGroup(), resources.Multiply(appRes, 1))
-	// create a new release with app, no allocationId: should return last left alloc
+	// create a new release with app, no allocationID: should return last left alloc
 	release.AllocationID = ""
 	allocs, _ = partition.removeAllocation(release)
 	assert.Equal(t, 1, len(allocs), "removal request for existing allocation returned wrong allocations: %v", allocs)
@@ -2173,7 +2173,7 @@ func setupPreemptionForRequiredNode(t *testing.T) (*PartitionContext, *objects.A
 	assert.Equal(t, alloc.GetApplicationID(), appID1, "expected application app-1 to be allocated")
 	assert.Equal(t, alloc.GetAllocationKey(), allocID, "expected ask alloc-1 to be allocated")
 	assertUserGroupResourceMaxLimits(t, getTestUserGroup(), resources.NewResourceFromMap(map[string]resources.Quantity{"vcore": 8000}), getExpectedQueuesLimitsForPreemptionWithRequiredNode())
-	allocationId := alloc.GetAllocationID()
+	allocationID := alloc.GetAllocationID()
 
 	// required node set on ask
 	ask2 := newAllocationAsk(allocID2, appID1, res)
@@ -2210,7 +2210,7 @@ func setupPreemptionForRequiredNode(t *testing.T) (*PartitionContext, *objects.A
 	release := &si.AllocationRelease{
 		PartitionName:   partition.Name,
 		ApplicationID:   appID1,
-		AllocationID:    allocationId,
+		AllocationID:    allocationID,
 		TerminationType: si.TerminationType_PREEMPTED_BY_SCHEDULER,
 	}
 	releases, _ := partition.removeAllocation(release)
diff --git a/pkg/webservice/dao/node_util.go b/pkg/webservice/dao/node_util.go
index 46ad7cd5..678f8dfa 100644
--- a/pkg/webservice/dao/node_util.go
+++ b/pkg/webservice/dao/node_util.go
@@ -19,7 +19,7 @@
 package dao
 
 type PartitionNodesUtilDAOInfo struct {
-	ClusterId     string              `json:"clusterId"` // no omitempty, cluster id should not be empty
+	ClusterID     string              `json:"clusterId"` // no omitempty, cluster id should not be empty
 	Partition     string              `json:"partition"` // no omitempty, partition should not be empty
 	NodesUtilList []*NodesUtilDAOInfo `json:"utilizations,omitempty"`
 }
diff --git a/pkg/webservice/handlers.go b/pkg/webservice/handlers.go
index 80feb68f..2d633460 100644
--- a/pkg/webservice/handlers.go
+++ b/pkg/webservice/handlers.go
@@ -514,7 +514,7 @@ func getPartitionNodesUtilJSON(partition *scheduler.PartitionContext) *dao.Parti
 	}
 
 	return &dao.PartitionNodesUtilDAOInfo{
-		ClusterId:     partition.RmID,
+		ClusterID:     partition.RmID,
 		Partition:     common.GetPartitionNameWithoutClusterID(partition.Name),
 		NodesUtilList: nodesUtilList,
 	}
diff --git a/pkg/webservice/handlers_test.go b/pkg/webservice/handlers_test.go
index 462cab80..9cf7633c 100644
--- a/pkg/webservice/handlers_test.go
+++ b/pkg/webservice/handlers_test.go
@@ -822,7 +822,7 @@ func TestGetPartitionNodesUtilJSON(t *testing.T) {
 
 	// assert partition nodes utilization
 	result := getPartitionNodesUtilJSON(partition)
-	assert.Equal(t, result.ClusterId, rmID)
+	assert.Equal(t, result.ClusterID, rmID)
 	assert.Equal(t, result.Partition, "default")
 	assert.Equal(t, len(result.NodesUtilList), 3, "Should have 3 resource types(CPU/Memory/GPU) in the list.")
 
@@ -884,8 +884,8 @@ func TestGetNodeUtilisations(t *testing.T) {
 	err = json.Unmarshal(resp.outputBytes, &partitionNodesUtilDAOInfo)
 	assert.NilError(t, err, "should decode a list of *dao.PartitionNodesUtilDAOInfo")
 	assert.Equal(t, len(partitionNodesUtilDAOInfo), 2)
-	assert.Equal(t, partitionNodesUtilDAOInfo[0].ClusterId, rmID)
-	assert.Equal(t, partitionNodesUtilDAOInfo[1].ClusterId, rmID)
+	assert.Equal(t, partitionNodesUtilDAOInfo[0].ClusterID, rmID)
+	assert.Equal(t, partitionNodesUtilDAOInfo[1].ClusterID, rmID)
 
 	defaultPartitionNodesUtilDAOInfo := partitionNodesUtilDAOInfo[0]
 	gpuPartitionNodesUtilDAOInfo := partitionNodesUtilDAOInfo[1]


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@yunikorn.apache.org
For additional commands, e-mail: issues-help@yunikorn.apache.org