You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2026/09/01 00:23:52 UTC

(airavata-custos) 02/02: Charge SLURM usage from the billing TRES

This is an automated email from the ASF dual-hosted git repository.

lahirujayathilake pushed a commit to branch billing-tres
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git

commit b73e04ebb5aa9f612ed41577af2e92baf526c640
Author: lahiruj <la...@gmail.com>
AuthorDate: Thu Aug 13 01:05:19 2026 -0400

    Charge SLURM usage from the billing TRES
---
 .../Usage-Monitor/internal/smonitor/smonitor.go    | 25 +++++-----
 .../internal/smonitor/smonitor_test.go             | 56 +++++++++++++++-------
 2 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor.go b/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor.go
index 5505a8762..f857db293 100644
--- a/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor.go
+++ b/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor.go
@@ -167,24 +167,21 @@ func (m *SlurmMonitor) recordJob(ctx context.Context, job client.JobInfo, cluste
 			return
 		}
 
-		// A TRES is keyed by (type, name). Plain resources have an empty name and
-		// key on type alone (cpu); a GPU is {type:gres, name:gpu}, stored as the
-		// joined "gres/gpu". Matching on type alone never finds a GPU.
-		resourceAmount := int64(0)
+		// Without TRESBillingWeights and PriorityFlags=MAX_TRES on the
+		// partition, billing is just the cpu count and this charge is wrong.
+		billingAmount := float64(0)
 		for _, tres := range job.Tres.Allocated {
-			// Example tres entry Allocated:[{Type:cpu Name: Count:1} {Type:mem Name: Count:8000} {Type:energy Name: Count:-2} {Type:node Name: Count:1} {Type:gres Name:gpu Count:1} {Type:billing Name: Count:1}]
-			key := tres.Type
-			if tres.Name != "" {
-				key = tres.Type + "/" + tres.Name
-			}
-			if key == resource.ResourceType {
-				resourceAmount = tres.Count
+			if tres.Type == "billing" {
+				billingAmount = tres.Count
 			}
 		}
+		if billingAmount <= 0 {
+			slog.Warn("SLURM job has no billing charge, skipping usage recording", "job_id", job.JobID, "billing", billingAmount)
+			return
+		}
 
-		// tres.Count already is per-node amount x node count (the whole-job total),
-		// so raw = tres.Count x hours.
-		calculatedRawAmount := float64(resourceAmount) * float64(jobDurationSec) / 3600
+		// billing is already the whole-job total, so no node multiplier.
+		calculatedRawAmount := billingAmount * float64(jobDurationSec) / 3600
 
 		rate, err := m.coreService.GetEffectiveRateForResource(ctx, resource.ID, time.Unix(job.Time.End, 0))
 		if err != nil {
diff --git a/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor_test.go b/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor_test.go
index 72ec3fc3f..75fd320a7 100644
--- a/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor_test.go
+++ b/connectors/SLURM/Usage-Monitor/internal/smonitor/smonitor_test.go
@@ -44,8 +44,11 @@ func fixtureJob(jobID int64, user, partition string) client.JobInfo {
 		Time:      client.JobTime{Start: 1000, End: 4600}, // exactly 3600 seconds
 		Tres: client.JobTresInfo{
 			Allocated: []client.TRES{
+				// billing differs from every other count so a wrong read
+				// cannot pass by coincidence.
 				{Type: "cpu", Count: 2},
 				{Type: "node", Count: 1},
+				{Type: "billing", Count: 5},
 			},
 		},
 	}
@@ -105,11 +108,11 @@ func TestPollRecordsRawAndSUExactly(t *testing.T) {
 		t.Fatalf("expected 1 usage row, got %d", len(calls))
 	}
 	u := calls[0].U
-	if u.UsedRawAmount != 2.0 {
-		t.Errorf("expected used_raw == 2.0 (2 cpu x 3600s / 3600), got %v", u.UsedRawAmount)
+	if u.UsedRawAmount != 5.0 {
+		t.Errorf("expected used_raw == 5.0 (5 billing x 3600s / 3600), got %v", u.UsedRawAmount)
 	}
-	if u.UsedSUAmount != 16.0 {
-		t.Errorf("expected used_su == 16.0 (2.0 raw x 8.0 rate), got %v", u.UsedSUAmount)
+	if u.UsedSUAmount != 40.0 {
+		t.Errorf("expected used_su == 40.0 (5.0 raw x 8.0 rate), got %v", u.UsedSUAmount)
 	}
 	if u.JobID != "42" || u.ComputeAllocationID != "alloc-1" || u.ComputeAllocationResourceID != "res-debug" {
 		t.Errorf("unexpected usage row identity: %+v", u)
@@ -120,8 +123,6 @@ func TestPollRecordsRawAndSUExactly(t *testing.T) {
 	}
 }
 
-// A multi-node job must not multiply by the node count: allocated TRES is
-// already the whole-job total.
 func TestPollMultiNodeDoesNotOvercount(t *testing.T) {
 	core := newMockCore(map[string]float64{"res-debug": 1.0}, map[string]bool{"alice": true})
 	job := client.JobInfo{
@@ -132,8 +133,9 @@ func TestPollMultiNodeDoesNotOvercount(t *testing.T) {
 		Time:      client.JobTime{Start: 1000, End: 4600}, // 3600 seconds
 		Tres: client.JobTresInfo{
 			Allocated: []client.TRES{
-				{Type: "cpu", Count: 8}, // 8 cpus total across 2 nodes
+				{Type: "cpu", Count: 8},
 				{Type: "node", Count: 2},
+				{Type: "billing", Count: 12}, // total across both nodes
 			},
 		},
 	}
@@ -143,14 +145,14 @@ func TestPollMultiNodeDoesNotOvercount(t *testing.T) {
 	if len(calls) != 1 {
 		t.Fatalf("expected 1 usage row, got %d", len(calls))
 	}
-	if got := calls[0].U.UsedRawAmount; got != 8.0 {
-		t.Errorf("expected used_raw == 8.0 (8 cpu x 3600s / 3600, NOT x 2 nodes), got %v", got)
+	if got := calls[0].U.UsedRawAmount; got != 12.0 {
+		t.Errorf("expected used_raw == 12.0 (12 billing x 3600s / 3600, NOT x 2 nodes), got %v", got)
 	}
 }
 
-// A GPU resource is stored as "gres/gpu" but its TRES is {type:gres, name:gpu}.
-// Matching must join type and name, or GPU jobs record zero usage.
-func TestPollMatchesGpuTres(t *testing.T) {
+// Charging the resource the partition is named for would bill 1 here, but the
+// job holds a whole node worth 4.
+func TestPollUsesBillingTresNotThePartitionResourceType(t *testing.T) {
 	core := newMockCore(map[string]float64{"res-gpu": 2.0}, map[string]bool{"alice": true})
 	core.GetComputeAllocationResourceByNameAndClusterFunc = func(ctx context.Context, name, clusterID string) (*models.ComputeAllocationResource, error) {
 		return &models.ComputeAllocationResource{ID: "res-gpu", Name: name, ResourceType: "gres/gpu"}, nil
@@ -163,9 +165,10 @@ func TestPollMatchesGpuTres(t *testing.T) {
 		Time:      client.JobTime{Start: 1000, End: 4600}, // 3600 seconds
 		Tres: client.JobTresInfo{
 			Allocated: []client.TRES{
-				{Type: "cpu", Count: 4},
-				{Type: "gres", Name: "gpu", Count: 2},
+				{Type: "cpu", Count: 128}, // every core on the node
+				{Type: "gres", Name: "gpu", Count: 1},
 				{Type: "node", Count: 1},
+				{Type: "billing", Count: 4},
 			},
 		},
 	}
@@ -175,8 +178,29 @@ func TestPollMatchesGpuTres(t *testing.T) {
 	if len(calls) != 1 {
 		t.Fatalf("expected 1 usage row, got %d", len(calls))
 	}
-	if got := calls[0].U.UsedRawAmount; got != 2.0 {
-		t.Errorf("expected used_raw == 2.0 (2 gpu x 3600s / 3600), got %v", got)
+	if got := calls[0].U.UsedRawAmount; got != 4.0 {
+		t.Errorf("expected used_raw == 4.0 (4 billing x 3600s / 3600), got %v", got)
+	}
+	if got := calls[0].U.UsedSUAmount; got != 8.0 {
+		t.Errorf("expected used_su == 8.0 (4.0 raw x 2.0 rate), got %v", got)
+	}
+}
+
+func TestPollSkipsJobWithoutBillingTres(t *testing.T) {
+	core := newMockCore(map[string]float64{"res-debug": 8.0}, map[string]bool{"alice": true})
+	job := fixtureJob(45, "alice", "debug")
+	job.Tres.Allocated = []client.TRES{
+		{Type: "cpu", Count: 2},
+		{Type: "node", Count: 1},
+	}
+	newTestMonitor(core, job, fixtureJob(46, "alice", "debug")).poll()
+
+	calls := core.CreateComputeAllocationUsageCalls()
+	if len(calls) != 1 {
+		t.Fatalf("expected exactly 1 usage row (job without billing skipped), got %d", len(calls))
+	}
+	if calls[0].U.JobID != "46" {
+		t.Errorf("expected job 46 to be recorded after skipping job 45, got job %s", calls[0].U.JobID)
 	}
 }