You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2026/10/13 21:55:15 UTC
(airavata-custos) branch amie-alloc-resource created (now 90fa3f16d)
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a change to branch amie-alloc-resource
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
at 90fa3f16d Support one AMIE resource alias map to several catalog resources
This branch includes the following new commits:
new 90fa3f16d Support one AMIE resource alias map to several catalog resources
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airavata-custos) 01/01: Support one AMIE resource alias map to several catalog resources
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch amie-alloc-resource
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit 90fa3f16d1af35d7c2a5d510937103a0fce271ff
Author: lahiruj <la...@gmail.com>
AuthorDate: Tue Oct 13 14:12:38 2026 -0400
Support one AMIE resource alias map to several catalog resources
---
config/custos.yaml | 5 +-
connectors/ACCESS/AMIE-Processor/README.md | 3 +-
.../handler/request_project_create.go | 57 +++++++++++++---------
.../request_project_create_integration_test.go | 40 +++++++--------
.../ACCESS/AMIE-Processor/pkg/amie/loader.go | 16 ++++--
5 files changed, 67 insertions(+), 54 deletions(-)
diff --git a/config/custos.yaml b/config/custos.yaml
index ab9cf58ed..ef1d68516 100644
--- a/config/custos.yaml
+++ b/config/custos.yaml
@@ -124,10 +124,9 @@ connectors:
api_key: "${AMIE_API_KEY}"
cluster:
id: "${AMIE_CLUSTER_ID}"
- # Packet resource names that are not catalog names. Each entry maps the
- # name in a packet's ResourceList to the catalog resource it stands for.
+ # Maps a packet resource name to one or more catalog resources. The allocation is attached to each of them.
# resource_aliases:
- # my-gpu.site.access-ci.org: gpu-a100
+ # my-gpu.site.access-ci.org: [gpu-a100, gpu-h100]
# my-cpu.site.access-ci.org: cpu-general
resource_aliases: {}
polling:
diff --git a/connectors/ACCESS/AMIE-Processor/README.md b/connectors/ACCESS/AMIE-Processor/README.md
index 908216687..5336ee6a7 100644
--- a/connectors/ACCESS/AMIE-Processor/README.md
+++ b/connectors/ACCESS/AMIE-Processor/README.md
@@ -252,5 +252,6 @@ Sensitive values come from env vars sourced by the server on startup:
Poll intervals, worker interval, and HTTP timeouts are set in the YAML block. Log level and format are inherited from
the core server. A packet's `ResourceList` names are matched against the cluster's resource catalog by name; when
ACCESS uses a different name for a resource, map it under `resource_aliases` in the same block, for example
-`my-gpu.site.access-ci.org: gpu-a100`. The database DSN is shared with core via `DATABASE_DSN`; this connector does not maintain its own
+`my-gpu.site.access-ci.org: gpu-a100`. An alias can also list several catalog resources, such as
+`my-gpu.site.access-ci.org: [gpu-a100, gpu-h100]`, and the allocation is attached to each. The database DSN is shared with core via `DATABASE_DSN`; this connector does not maintain its own
connection.
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
index c8f7f321a..6187cc7f4 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
@@ -40,10 +40,10 @@ type RequestProjectCreateHandler struct {
auditSvc AuditService
// Packet resource names that are not catalog names, mapped to the catalog
// resource they stand for in the custos.yaml
- resourceAliases map[string]string
+ resourceAliases map[string][]string
}
-func NewRequestProjectCreateHandler(svc *service.Service, clusterID string, amieClient AmieClient, auditSvc AuditService, resourceAliases map[string]string) *RequestProjectCreateHandler {
+func NewRequestProjectCreateHandler(svc *service.Service, clusterID string, amieClient AmieClient, auditSvc AuditService, resourceAliases map[string][]string) *RequestProjectCreateHandler {
return &RequestProjectCreateHandler{svc: svc, clusterID: clusterID, amieClient: amieClient, auditSvc: auditSvc, resourceAliases: resourceAliases}
}
@@ -268,33 +268,44 @@ func (h *RequestProjectCreateHandler) ensureAllocation(ctx context.Context, body
// already stored on the allocation, so mappings carry no native caps;
// unregistered names are skipped rather than failing the packet.
func (h *RequestProjectCreateHandler) ensureResourceMappings(ctx context.Context, tx *sql.Tx, body map[string]any, allocation *models.ComputeAllocation, packet *model.Packet, eventID string) error {
- for _, name := range getResourceList(body) {
- if local, ok := h.resourceAliases[name]; ok {
- name = local
+ for _, packetName := range getResourceList(body) {
+ // One packet resource can stand for several catalog resources, such as two GPU partitions with their own rates.
+ names, ok := h.resourceAliases[packetName]
+ if !ok {
+ names = []string{packetName}
}
- resource, err := h.svc.GetComputeAllocationResourceByNameAndCluster(ctx, name, h.clusterID)
- if errors.Is(err, service.ErrNotFound) {
- slog.WarnContext(ctx, "amie: packet resource not in the cluster catalog, mapping skipped",
- "resource", name, "allocation_id", allocation.ID)
- continue
- }
- if err != nil {
- return fmt.Errorf("lookup resource %q: %w", name, err)
- }
- mapping, err := h.svc.AttachResourceToAllocation(ctx, allocation.ID, resource.ID, 0, 0)
- if errors.Is(err, service.ErrAlreadyExists) {
- continue
- }
- if err != nil {
- return fmt.Errorf("attach resource %q: %w", name, err)
- }
- if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditAttachResource, "compute_allocation_resource_mapping", mapping.ID, name); err != nil {
- return fmt.Errorf("audit ATTACH_RESOURCE: %w", err)
+ for _, name := range names {
+ if err := h.attachResource(ctx, tx, allocation, packet, eventID, name); err != nil {
+ return err
+ }
}
}
return nil
}
+func (h *RequestProjectCreateHandler) attachResource(ctx context.Context, tx *sql.Tx, allocation *models.ComputeAllocation, packet *model.Packet, eventID, name string) error {
+ resource, err := h.svc.GetComputeAllocationResourceByNameAndCluster(ctx, name, h.clusterID)
+ if errors.Is(err, service.ErrNotFound) {
+ slog.WarnContext(ctx, "amie: packet resource not in the cluster catalog, mapping skipped",
+ "resource", name, "allocation_id", allocation.ID)
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("lookup resource %q: %w", name, err)
+ }
+ mapping, err := h.svc.AttachResourceToAllocation(ctx, allocation.ID, resource.ID, 0, 0)
+ if errors.Is(err, service.ErrAlreadyExists) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("attach resource %q: %w", name, err)
+ }
+ if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditAttachResource, "compute_allocation_resource_mapping", mapping.ID, name); err != nil {
+ return fmt.Errorf("audit ATTACH_RESOURCE: %w", err)
+ }
+ return nil
+}
+
// ensurePIMembership asserts the PI as a role=PI membership on the allocation.
// Returns (membership, created=true) when it inserted a new row, or
// (existing, created=false) on redelivery. The service rejects a second
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_create_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_create_integration_test.go
index 344811716..a35739134 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_create_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_create_integration_test.go
@@ -593,25 +593,19 @@ func TestRequestProjectCreate_AttachesRegisteredResources(t *testing.T) {
}
}
-// Make sure a packet resource with an alias in the custos.yaml config attaches the catalog resource the alias points to.
-func TestRequestProjectCreate_AttachesAliasedResource(t *testing.T) {
+// Make sure an alias with two catalog resources attaches the allocation to both.
+func TestRequestProjectCreate_AttachesEveryAliasedResource(t *testing.T) {
database := setupTestDB(t)
-
svc := newTestCoreService(database)
- audit := newTestAuditService(database)
- amie := &fakeAmieClient{}
- h := NewRequestProjectCreateHandler(svc, testClusterID, amie, audit, map[string]string{
- "compute.access-ci.org": "gpu-b200",
- })
-
- resource, err := svc.CreateComputeAllocationResource(context.Background(), &models.ComputeAllocationResource{
- Name: "gpu-b200",
- ResourceType: "gres/gpu:b200",
- ResourceAmount: 16,
- ComputeClusterID: testClusterID,
+ h := NewRequestProjectCreateHandler(svc, testClusterID, &fakeAmieClient{}, newTestAuditService(database), map[string][]string{
+ "compute.access-ci.org": {"gpu-b200", "gpu-rtxpro"},
})
- if err != nil {
- t.Fatalf("seed catalog resource: %v", err)
+ for _, name := range []string{"gpu-b200", "gpu-rtxpro"} {
+ if _, err := svc.CreateComputeAllocationResource(context.Background(), &models.ComputeAllocationResource{
+ Name: name, ResourceType: "gres/gpu", ResourceAmount: 1, ComputeClusterID: testClusterID,
+ }); err != nil {
+ t.Fatalf("seed catalog resource %s: %v", name, err)
+ }
}
body := baseRPCBody()
@@ -622,14 +616,14 @@ func TestRequestProjectCreate_AttachesAliasedResource(t *testing.T) {
t.Fatalf("Handle returned error: %v", err)
}
- var resourceID string
- if err := database.Get(&resourceID,
- "SELECT compute_allocation_resource_id FROM compute_allocation_resource_mappings LIMIT 1",
- ); err != nil {
- t.Fatalf("read mapping: %v", err)
+ var attached []string
+ if err := database.Select(&attached,
+ `SELECT r.name FROM compute_allocation_resource_mappings m
+ JOIN compute_allocation_resources r ON r.id = m.compute_allocation_resource_id ORDER BY r.name`); err != nil {
+ t.Fatalf("read mappings: %v", err)
}
- if resourceID != resource.ID {
- t.Errorf("mapping.resource_id: got %q, want %q", resourceID, resource.ID)
+ if strings.Join(attached, ",") != "gpu-b200,gpu-rtxpro" {
+ t.Errorf("attached resources: got %v, want gpu-b200 and gpu-rtxpro", attached)
}
}
diff --git a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
index 4c07bb5ef..716f9e23f 100644
--- a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
+++ b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
@@ -221,8 +221,9 @@ func loadClusterID(connectorConfig *custosconfig.ConnectorConfig) string {
return os.Getenv("AMIE_CLUSTER_ID")
}
-func loadResourceAliases(connectorConfig *custosconfig.ConnectorConfig) map[string]string {
- aliases := map[string]string{}
+// loadResourceAliases reads each alias as one catalog name or a list of them.
+func loadResourceAliases(connectorConfig *custosconfig.ConnectorConfig) map[string][]string {
+ aliases := map[string][]string{}
if connectorConfig == nil {
return aliases
}
@@ -231,8 +232,15 @@ func loadResourceAliases(connectorConfig *custosconfig.ConnectorConfig) map[stri
return aliases
}
for name, v := range raw {
- if local, ok := v.(string); ok && local != "" {
- aliases[name] = local
+ switch local := v.(type) {
+ case string:
+ aliases[name] = []string{local}
+ case []any:
+ for _, item := range local {
+ if s, ok := item.(string); ok {
+ aliases[name] = append(aliases[name], s)
+ }
+ }
}
}
return aliases