You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2026/10/17 22:06:25 UTC
(airavata-custos) branch master updated: ensure transaction scope for db writes (#627)
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
The following commit(s) were added to refs/heads/master by this push:
new dc1ac2b87 ensure transaction scope for db writes (#627)
dc1ac2b87 is described below
commit dc1ac2b87d949f434618f1c27b139beeef97cb89
Author: Yasith Jayawardana <ya...@icloud.com>
AuthorDate: Sat Oct 17 18:06:17 2026 -0400
ensure transaction scope for db writes (#627)
---
.../ACCESS/AMIE-Processor/handler/handler.go | 8 +--
.../handler/request_account_create.go | 6 +-
.../handler/request_account_inactivate.go | 2 +-
.../request_account_inactivate_integration_test.go | 6 +-
.../handler/request_account_reactivate.go | 2 +-
.../AMIE-Processor/handler/request_person_merge.go | 2 +-
.../request_person_merge_integration_test.go | 10 ++--
.../handler/request_project_create.go | 24 ++++----
.../handler/request_project_inactivate.go | 8 +--
.../request_project_inactivate_integration_test.go | 6 +-
.../handler/request_project_reactivate.go | 8 +--
.../request_project_reactivate_integration_test.go | 6 +-
.../compute_allocation_change_request_store.go | 13 +++++
.../store/compute_allocation_membership_store.go | 13 +++++
internal/store/compute_allocation_store.go | 13 +++++
internal/store/compute_cluster_user_store.go | 20 +++++++
internal/store/project_store.go | 13 +++++
internal/store/store.go | 14 +++++
internal/store/user_store.go | 13 +++++
pkg/service/compute_allocation.go | 10 +++-
pkg/service/compute_allocation_change_request.go | 34 ++++++++---
...e_allocation_change_request_integration_test.go | 32 +++++++++++
pkg/service/compute_allocation_diff.go | 16 ++++--
pkg/service/compute_allocation_membership.go | 16 ++++--
pkg/service/compute_cluster_user.go | 65 +++++++++++-----------
.../compute_cluster_user_integration_test.go | 35 ++++++++++++
pkg/service/project.go | 16 ++++--
pkg/service/user_merge.go | 22 +++++---
28 files changed, 327 insertions(+), 106 deletions(-)
diff --git a/connectors/ACCESS/AMIE-Processor/handler/handler.go b/connectors/ACCESS/AMIE-Processor/handler/handler.go
index f3cb91bea..88db545ea 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/handler.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/handler.go
@@ -137,11 +137,11 @@ func ensureUserIdentity(ctx context.Context, svc *service.Service, userID, globa
// reactivate sets an existing membership back to active. The spec says a
// create packet for a known account also reactivates it.
-func reactivate(ctx context.Context, svc *service.Service, m models.ComputeAllocationMembership) (*models.ComputeAllocationMembership, error) {
+func reactivate(ctx context.Context, tx *sql.Tx, svc *service.Service, m models.ComputeAllocationMembership) (*models.ComputeAllocationMembership, error) {
if m.MembershipStatus == models.ACTIVE {
return &m, nil
}
- return svc.UpdateMembershipStatus(ctx, m.ID, models.ACTIVE)
+ return svc.UpdateMembershipStatusTx(ctx, tx, m.ID, models.ACTIVE)
}
// flipUserMemberships flips every ComputeAllocationMembership the user holds
@@ -149,7 +149,7 @@ func reactivate(ctx context.Context, svc *service.Service, m models.ComputeAlloc
// the given status. Returns the rows that were updated. Silently returns an
// empty slice when the project or user is unknown, the reply still goes back
// to AMIE but no state changes.
-func flipUserMemberships(ctx context.Context, svc *service.Service, projectID, userID string, status models.AllocationStatus) ([]models.ComputeAllocationMembership, error) {
+func flipUserMemberships(ctx context.Context, tx *sql.Tx, svc *service.Service, projectID, userID string, status models.AllocationStatus) ([]models.ComputeAllocationMembership, error) {
project, err := svc.GetProject(ctx, projectID)
if err != nil {
if errors.Is(err, service.ErrNotFound) {
@@ -171,7 +171,7 @@ func flipUserMemberships(ctx context.Context, svc *service.Service, projectID, u
if m.UserID != userID {
continue
}
- flipped, err := svc.UpdateMembershipStatus(ctx, m.ID, status)
+ flipped, err := svc.UpdateMembershipStatusTx(ctx, tx, m.ID, status)
if err != nil {
return nil, fmt.Errorf("update membership %s: %w", m.ID, err)
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
index ce5be406f..4f433e6dc 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
@@ -114,7 +114,7 @@ func (h *RequestAccountCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
}
role := normalizeAMIERole(normalizeRole(getString(body, "UserRole")))
- membership, err := h.ensureMembership(ctx, &allocation, user.ID)
+ membership, err := h.ensureMembership(ctx, tx, &allocation, user.ID)
if err != nil {
return fmt.Errorf("request_account_create: ensure membership: %w", err)
}
@@ -193,7 +193,7 @@ func (h *RequestAccountCreateHandler) ensureComputeClusterUser(ctx context.Conte
// ensureMembership reactivates a known member only while the allocation is active.
// On an inactive project the member stays inactive, as the spec says.
-func (h *RequestAccountCreateHandler) ensureMembership(ctx context.Context, allocation *models.ComputeAllocation, userID string) (*models.ComputeAllocationMembership, error) {
+func (h *RequestAccountCreateHandler) ensureMembership(ctx context.Context, tx *sql.Tx, allocation *models.ComputeAllocation, userID string) (*models.ComputeAllocationMembership, error) {
existing, err := h.svc.ListMembersForAllocation(ctx, allocation.ID)
if err != nil {
return nil, fmt.Errorf("list memberships: %w", err)
@@ -203,7 +203,7 @@ func (h *RequestAccountCreateHandler) ensureMembership(ctx context.Context, allo
if allocation.Status != models.ACTIVE {
return &m.ComputeAllocationMembership, nil
}
- return reactivate(ctx, h.svc, m.ComputeAllocationMembership)
+ return reactivate(ctx, tx, h.svc, m.ComputeAllocationMembership)
}
}
return h.svc.CreateComputeAllocationMembership(ctx, &models.ComputeAllocationMembership{
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate.go
index 4a0e58b4c..fc25d5540 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate.go
@@ -65,7 +65,7 @@ func (h *RequestAccountInactivateHandler) Handle(ctx context.Context, tx *sql.Tx
return err
}
- flipped, err := flipUserMemberships(ctx, h.svc, projectID, personID, models.INACTIVE)
+ flipped, err := flipUserMemberships(ctx, tx, h.svc, projectID, personID, models.INACTIVE)
if err != nil {
return fmt.Errorf("request_account_inactivate: flip memberships: %w", err)
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate_integration_test.go
index 85709bf9a..fd2e021bd 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_inactivate_integration_test.go
@@ -305,9 +305,6 @@ func TestRequestAccountInactivate_MissingRequiredField(t *testing.T) {
// TestRequestAccountInactivate_ReplyFailurePropagates asserts the handler
// surfaces a ReplyToPacket failure to the caller so the processor can decide
// to retry. The REPLY_SENT audit row must be absent when the reply failed.
-// (Note the broader atomicity gap: the membership flip is already committed
-// because UpdateMembershipStatus opens its own tx; this test only asserts
-// the error propagation + audit shape.)
func TestRequestAccountInactivate_ReplyFailurePropagates(t *testing.T) {
database := setupTestDB(t)
seed := seedMembership(t, database)
@@ -332,6 +329,9 @@ func TestRequestAccountInactivate_ReplyFailurePropagates(t *testing.T) {
if got := countAuditActions(t, database, pkt.ID, model.AuditReplySent); got != 0 {
t.Errorf("audit REPLY_SENT on failed reply: got %d, want 0", got)
}
+ if got := getMembershipStatus(t, database, seed.Membership.ID); got != string(models.ACTIVE) {
+ t.Errorf("membership after failed reply: got %q, want ACTIVE", got)
+ }
}
// silence unused-import false positives when only some tests reference these.
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_reactivate.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_reactivate.go
index fc9e56e0a..e1353d4b6 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_reactivate.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_reactivate.go
@@ -65,7 +65,7 @@ func (h *RequestAccountReactivateHandler) Handle(ctx context.Context, tx *sql.Tx
return err
}
- flipped, err := flipUserMemberships(ctx, h.svc, projectID, personID, models.ACTIVE)
+ flipped, err := flipUserMemberships(ctx, tx, h.svc, projectID, personID, models.ACTIVE)
if err != nil {
return fmt.Errorf("request_account_reactivate: flip memberships: %w", err)
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
index e50053e88..8bbc67332 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
@@ -143,7 +143,7 @@ func (h *RequestPersonMergeHandler) Handle(ctx context.Context, tx *sql.Tx, pack
return fmt.Errorf("request_person_merge: reprocess packets held for %s: %w", account.ID, err)
}
}
- if _, err := h.svc.MergeUsers(ctx, survivor.ID, retiring.ID); err != nil {
+ if _, err := h.svc.MergeUsersTx(ctx, tx, survivor.ID, retiring.ID); err != nil {
return fmt.Errorf("request_person_merge: merge: %w", err)
}
if err := h.userDNStore.ReassignUser(ctx, tx, retiring.ID, survivor.ID); err != nil {
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
index cf7e0faba..d8559dc19 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
@@ -512,12 +512,6 @@ func TestRequestPersonMerge_UnknownRetiringOrSurviving(t *testing.T) {
// a ReplyToPacket failure to the caller so the processor can decide to retry.
// The REPLY_SENT audit row must NOT be written when the reply did not actually
// go out.
-//
-// As with the analogous test in request_project_create_integration_test.go:
-// the merge itself goes through service.MergeUsers, which opens its own
-// transaction. So the surviving/retiring state changes commit independently
-// of the handler tx; we do not assert rollback shape here, only the (1)
-// error propagation and (2) audit invariant.
func TestRequestPersonMerge_ReplyFailurePropagates(t *testing.T) {
database := setupTestDB(t)
seed := seedMergeUsers(t, database)
@@ -547,6 +541,10 @@ func TestRequestPersonMerge_ReplyFailurePropagates(t *testing.T) {
if len(amie.Replies) != 0 {
t.Errorf("amie reply count on failed reply: got %d, want 0", len(amie.Replies))
}
+ // A merge left committed would make the retry skip the DN move and the held-packet reprocessing.
+ if got := getUserStatus(t, database, seed.retiringID); got == string(models.UserMerged) {
+ t.Errorf("retiring user after failed reply: got %q, want it unmerged", got)
+ }
}
// Make sure a packet held on the retiring person's cluster account is marked
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
index 6187cc7f4..8a6ba846e 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
@@ -109,7 +109,7 @@ func (h *RequestProjectCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
if title == "" {
title = grantNumber
}
- project, err := h.ensureProject(ctx, projectOriginatedID, title, pi.ID)
+ project, err := h.ensureProject(ctx, tx, projectOriginatedID, title, pi.ID)
if err != nil {
return fmt.Errorf("request_project_create: ensure project: %w", err)
}
@@ -117,7 +117,7 @@ func (h *RequestProjectCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
return fmt.Errorf("request_project_create: audit CREATE_PROJECT: %w", err)
}
- allocation, err := h.ensureAllocation(ctx, body, project.ID, grantNumber)
+ allocation, err := h.ensureAllocation(ctx, tx, body, project.ID, grantNumber)
if err != nil {
return fmt.Errorf("request_project_create: ensure allocation: %w", err)
}
@@ -129,7 +129,7 @@ func (h *RequestProjectCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
return fmt.Errorf("request_project_create: ensure resource mappings: %w", err)
}
- piMembership, created, err := h.ensurePIMembership(ctx, allocation, pi.ID)
+ piMembership, created, err := h.ensurePIMembership(ctx, tx, allocation, pi.ID)
if err != nil {
return fmt.Errorf("request_project_create: ensure PI membership: %w", err)
}
@@ -208,11 +208,11 @@ func (h *RequestProjectCreateHandler) ensurePIClusterUser(ctx context.Context, u
return allocateAndCreateClusterUser(ctx, h.svc, h.clusterID, userID)
}
-func (h *RequestProjectCreateHandler) ensureProject(ctx context.Context, originatedID, title, piID string) (*models.Project, error) {
+func (h *RequestProjectCreateHandler) ensureProject(ctx context.Context, tx *sql.Tx, originatedID, title, piID string) (*models.Project, error) {
if p, err := h.svc.GetProjectByOriginatedID(ctx, originatedID); err == nil {
// The spec says a project packet for an inactive project reactivates it.
if p.Status != models.ProjectActive {
- return h.svc.UpdateProjectStatus(ctx, p.ID, models.ProjectActive)
+ return h.svc.UpdateProjectStatusTx(ctx, tx, p.ID, models.ProjectActive)
}
return p, nil
} else if !errors.Is(err, service.ErrNotFound) {
@@ -227,7 +227,7 @@ func (h *RequestProjectCreateHandler) ensureProject(ctx context.Context, origina
}
// ensureAllocation also changes the allocation when the project already has one.
-func (h *RequestProjectCreateHandler) ensureAllocation(ctx context.Context, body map[string]any, projectID, grantNumber string) (*models.ComputeAllocation, error) {
+func (h *RequestProjectCreateHandler) ensureAllocation(ctx context.Context, tx *sql.Tx, body map[string]any, projectID, grantNumber string) (*models.ComputeAllocation, error) {
su, err := getNumber(body, "ServiceUnitsAllocated")
if err != nil {
return nil, err
@@ -238,7 +238,7 @@ func (h *RequestProjectCreateHandler) ensureAllocation(ctx context.Context, body
}
if len(existing) > 0 {
// A grant has one allocation, active or not. This handler never makes a second.
- return h.changeAllocation(ctx, body, &existing[0], su)
+ return h.changeAllocation(ctx, tx, body, &existing[0], su)
}
if su < 0 {
return nil, fmt.Errorf("%w: negative service units for a project with no allocation", model.ErrInvalidPacket)
@@ -310,14 +310,14 @@ func (h *RequestProjectCreateHandler) attachResource(ctx context.Context, tx *sq
// Returns (membership, created=true) when it inserted a new row, or
// (existing, created=false) on redelivery. The service rejects a second
// distinct PI on the same allocation.
-func (h *RequestProjectCreateHandler) ensurePIMembership(ctx context.Context, allocation *models.ComputeAllocation, piID string) (*models.ComputeAllocationMembership, bool, error) {
+func (h *RequestProjectCreateHandler) ensurePIMembership(ctx context.Context, tx *sql.Tx, allocation *models.ComputeAllocation, piID string) (*models.ComputeAllocationMembership, bool, error) {
existing, err := h.svc.ListMembersForAllocation(ctx, allocation.ID)
if err != nil {
return nil, false, fmt.Errorf("list memberships: %w", err)
}
for _, m := range existing {
if m.UserID == piID {
- active, err := reactivate(ctx, h.svc, m.ComputeAllocationMembership)
+ active, err := reactivate(ctx, tx, h.svc, m.ComputeAllocationMembership)
return active, false, err
}
}
@@ -336,7 +336,7 @@ func (h *RequestProjectCreateHandler) ensurePIMembership(ctx context.Context, al
// changeAllocation applies a repeat packet by its AllocationType and keeps a
// diff row as history. See "Allocation types" in ../README.md.
-func (h *RequestProjectCreateHandler) changeAllocation(ctx context.Context, body map[string]any, alloc *models.ComputeAllocation, su float64) (*models.ComputeAllocation, error) {
+func (h *RequestProjectCreateHandler) changeAllocation(ctx context.Context, tx *sql.Tx, body map[string]any, alloc *models.ComputeAllocation, su float64) (*models.ComputeAllocation, error) {
diffType := strings.ToUpper(strings.TrimSpace(getString(body, "AllocationType")))
if diffType == "" {
diffType = "GRANT"
@@ -363,10 +363,10 @@ func (h *RequestProjectCreateHandler) changeAllocation(ctx context.Context, body
alloc.InitialSUAmount += su
}
alloc.Status = models.ACTIVE
- if err := h.svc.UpdateComputeAllocation(ctx, alloc); err != nil {
+ if err := h.svc.UpdateComputeAllocationTx(ctx, tx, alloc); err != nil {
return nil, fmt.Errorf("change allocation: %w", err)
}
- if _, err := h.svc.CreateComputeAllocationDiff(ctx, &models.ComputeAllocationDiff{
+ if _, err := h.svc.CreateComputeAllocationDiffTx(ctx, tx, &models.ComputeAllocationDiff{
ComputeAllocationID: alloc.ID,
DiffType: diffType,
NewSUAmount: su,
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate.go
index b14c1434d..85d7828de 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate.go
@@ -74,7 +74,7 @@ func (h *RequestProjectInactivateHandler) Handle(ctx context.Context, tx *sql.Tx
return fmt.Errorf("request_project_inactivate: lookup project: %w", err)
}
- if _, err := h.svc.UpdateProjectStatus(ctx, project.ID, models.ProjectInactive); err != nil {
+ if _, err := h.svc.UpdateProjectStatusTx(ctx, tx, project.ID, models.ProjectInactive); err != nil {
return fmt.Errorf("request_project_inactivate: update project status: %w", err)
}
if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditInactivateProject, "project", project.ID, ""); err != nil {
@@ -98,10 +98,10 @@ func (h *RequestProjectInactivateHandler) deactivateAllocations(ctx context.Cont
}
for _, a := range allocations {
a.Status = models.INACTIVE
- if err := h.svc.UpdateComputeAllocation(ctx, &a); err != nil {
+ if err := h.svc.UpdateComputeAllocationTx(ctx, tx, &a); err != nil {
return fmt.Errorf("inactivate allocation %s: %w", a.ID, err)
}
- if _, err := h.svc.CreateComputeAllocationDiff(ctx, &models.ComputeAllocationDiff{
+ if _, err := h.svc.CreateComputeAllocationDiffTx(ctx, tx, &models.ComputeAllocationDiff{
ComputeAllocationID: a.ID,
DiffType: "ALLOCATION_STATUS_CHANGE",
Status: models.INACTIVE,
@@ -114,7 +114,7 @@ func (h *RequestProjectInactivateHandler) deactivateAllocations(ctx context.Cont
return fmt.Errorf("list memberships for allocation %s: %w", a.ID, err)
}
for _, m := range members {
- if _, err := h.svc.UpdateMembershipStatus(ctx, m.ID, models.INACTIVE); err != nil {
+ if _, err := h.svc.UpdateMembershipStatusTx(ctx, tx, m.ID, models.INACTIVE); err != nil {
return fmt.Errorf("inactivate membership %s: %w", m.ID, err)
}
if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditInactivateMembership, "compute_allocation_membership", m.ID,
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate_integration_test.go
index 9b45118ff..b8a1beb33 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_inactivate_integration_test.go
@@ -378,8 +378,7 @@ func TestRequestProjectInactivate_MissingProjectID(t *testing.T) {
// TestRequestProjectInactivate_ReplyFailurePropagates asserts the handler
// surfaces a ReplyToPacket failure to the caller. REPLY_SENT must NOT exist
-// when the reply failed. The protocol does not mandate atomicity on reply
-// failure, so this test does not check rollback shape.
+// when the reply failed.
func TestRequestProjectInactivate_ReplyFailurePropagates(t *testing.T) {
database := setupTestDB(t)
svc := newTestCoreService(database)
@@ -406,4 +405,7 @@ func TestRequestProjectInactivate_ReplyFailurePropagates(t *testing.T) {
if got := countAuditActions(t, database, pkt.ID, model.AuditReplySent); got != 0 {
t.Errorf("audit REPLY_SENT on failed reply: got %d, want 0", got)
}
+ if got := membershipStatus(t, database, seed.PIMembership); got != models.ACTIVE {
+ t.Errorf("PI membership after failed reply: got %q, want ACTIVE", got)
+ }
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate.go
index 2ea169d8c..45255b7f2 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate.go
@@ -77,7 +77,7 @@ func (h *RequestProjectReactivateHandler) Handle(ctx context.Context, tx *sql.Tx
return fmt.Errorf("request_project_reactivate: lookup project: %w", err)
}
- if _, err := h.svc.UpdateProjectStatus(ctx, project.ID, models.ProjectActive); err != nil {
+ if _, err := h.svc.UpdateProjectStatusTx(ctx, tx, project.ID, models.ProjectActive); err != nil {
return fmt.Errorf("request_project_reactivate: update project status: %w", err)
}
if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditReactivateProject, "project", project.ID, ""); err != nil {
@@ -102,10 +102,10 @@ func (h *RequestProjectReactivateHandler) reactivateAllocationsAndPI(ctx context
}
for _, a := range allocations {
a.Status = models.ACTIVE
- if err := h.svc.UpdateComputeAllocation(ctx, &a); err != nil {
+ if err := h.svc.UpdateComputeAllocationTx(ctx, tx, &a); err != nil {
return fmt.Errorf("reactivate allocation %s: %w", a.ID, err)
}
- if _, err := h.svc.CreateComputeAllocationDiff(ctx, &models.ComputeAllocationDiff{
+ if _, err := h.svc.CreateComputeAllocationDiffTx(ctx, tx, &models.ComputeAllocationDiff{
ComputeAllocationID: a.ID,
DiffType: "ALLOCATION_STATUS_CHANGE",
Status: models.ACTIVE,
@@ -124,7 +124,7 @@ func (h *RequestProjectReactivateHandler) reactivateAllocationsAndPI(ctx context
if m.UserID != project.ProjectPIID {
continue
}
- if _, err := h.svc.UpdateMembershipStatus(ctx, m.ID, models.ACTIVE); err != nil {
+ if _, err := h.svc.UpdateMembershipStatusTx(ctx, tx, m.ID, models.ACTIVE); err != nil {
return fmt.Errorf("reactivate PI membership %s: %w", m.ID, err)
}
if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditReactivateMembership, "compute_allocation_membership", m.ID,
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate_integration_test.go
index f3dd32ad9..37ec80ba2 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_reactivate_integration_test.go
@@ -269,8 +269,7 @@ func TestRequestProjectReactivate_MissingProjectID(t *testing.T) {
// TestRequestProjectReactivate_ReplyFailurePropagates asserts the handler
// surfaces a ReplyToPacket failure to the caller. REPLY_SENT must NOT exist
-// when the reply failed. (Core writes commit in their own txns, so this
-// test does not check rollback shape.)
+// when the reply failed.
func TestRequestProjectReactivate_ReplyFailurePropagates(t *testing.T) {
database := setupTestDB(t)
svc := newTestCoreService(database)
@@ -298,4 +297,7 @@ func TestRequestProjectReactivate_ReplyFailurePropagates(t *testing.T) {
if got := countAuditActions(t, database, pkt.ID, model.AuditReplySent); got != 0 {
t.Errorf("audit REPLY_SENT on failed reply: got %d, want 0", got)
}
+ if got := membershipStatus(t, database, seed.PIMembership); got != models.INACTIVE {
+ t.Errorf("PI membership after failed reply: got %q, want INACTIVE", got)
+ }
}
diff --git a/internal/store/compute_allocation_change_request_store.go b/internal/store/compute_allocation_change_request_store.go
index da574dd50..aad599ea7 100644
--- a/internal/store/compute_allocation_change_request_store.go
+++ b/internal/store/compute_allocation_change_request_store.go
@@ -52,6 +52,19 @@ func (s *pgComputeAllocationChangeRequestStore) FindByID(ctx context.Context, id
return &c, nil
}
+func (s *pgComputeAllocationChangeRequestStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocationChangeRequest, error) {
+ var c models.ComputeAllocationChangeRequest
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &c,
+ `SELECT `+computeAllocationChangeRequestColumns+` FROM compute_allocation_change_requests WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &c, nil
+}
+
func (s *pgComputeAllocationChangeRequestStore) FindByAllocation(ctx context.Context, allocationID string) ([]models.ComputeAllocationChangeRequest, error) {
var rows []models.ComputeAllocationChangeRequest
err := s.db.SelectContext(ctx, &rows,
diff --git a/internal/store/compute_allocation_membership_store.go b/internal/store/compute_allocation_membership_store.go
index 3ef1e286e..e72cd929b 100644
--- a/internal/store/compute_allocation_membership_store.go
+++ b/internal/store/compute_allocation_membership_store.go
@@ -70,6 +70,19 @@ func (s *pgComputeAllocationMembershipStore) FindByID(ctx context.Context, id st
return &m, nil
}
+func (s *pgComputeAllocationMembershipStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocationMembership, error) {
+ var r models.ComputeAllocationMembership
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &r,
+ `SELECT `+computeAllocationMembershipColumns+` FROM compute_allocation_memberships WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &r, nil
+}
+
func (s *pgComputeAllocationMembershipStore) FindByPair(ctx context.Context, allocationID, userID string) (*models.ComputeAllocationMembership, error) {
var m models.ComputeAllocationMembership
err := s.db.GetContext(ctx, &m,
diff --git a/internal/store/compute_allocation_store.go b/internal/store/compute_allocation_store.go
index b32a4c95a..1a96c0fe7 100644
--- a/internal/store/compute_allocation_store.go
+++ b/internal/store/compute_allocation_store.go
@@ -53,6 +53,19 @@ func (s *pgComputeAllocationStore) FindByID(ctx context.Context, id string) (*mo
return &a, nil
}
+func (s *pgComputeAllocationStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocation, error) {
+ var r models.ComputeAllocation
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &r,
+ `SELECT `+computeAllocationColumns+` FROM compute_allocations WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &r, nil
+}
+
func (s *pgComputeAllocationStore) FindByProject(ctx context.Context, projectID string) ([]models.ComputeAllocation, error) {
var allocs []models.ComputeAllocation
err := s.db.SelectContext(ctx, &allocs,
diff --git a/internal/store/compute_cluster_user_store.go b/internal/store/compute_cluster_user_store.go
index 2425304d3..ea47e533b 100644
--- a/internal/store/compute_cluster_user_store.go
+++ b/internal/store/compute_cluster_user_store.go
@@ -63,6 +63,19 @@ func (s *pgComputeClusterUserStore) FindByID(ctx context.Context, id string) (*m
return &c, nil
}
+func (s *pgComputeClusterUserStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeClusterUser, error) {
+ var c models.ComputeClusterUser
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &c,
+ `SELECT `+computeClusterUserColumns+` FROM compute_cluster_users WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &c, nil
+}
+
func (s *pgComputeClusterUserStore) FindByPair(ctx context.Context, clusterID, userID string) (*models.ComputeClusterUser, error) {
var c models.ComputeClusterUser
err := s.db.GetContext(ctx, &c,
@@ -119,6 +132,13 @@ func (s *pgComputeClusterUserStore) FindByUser(ctx context.Context, userID strin
return users, nil
}
+func (s *pgComputeClusterUserStore) FindByUserTx(ctx context.Context, tx *sql.Tx, userID string) ([]models.ComputeClusterUser, error) {
+ var users []models.ComputeClusterUser
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).SelectContext(ctx, &users,
+ `SELECT `+computeClusterUserColumns+` FROM compute_cluster_users WHERE user_id = $1 ORDER BY compute_cluster_id FOR UPDATE`, userID)
+ return users, err
+}
+
func (s *pgComputeClusterUserStore) Create(ctx context.Context, tx *sql.Tx, c *models.ComputeClusterUser) error {
_, err := tx.ExecContext(ctx,
`INSERT INTO compute_cluster_users (id, compute_cluster_id, user_id, local_username, access_level, approval_status, reviewed_at, reviewed_by)
diff --git a/internal/store/project_store.go b/internal/store/project_store.go
index 56a1bd47e..efc6f455c 100644
--- a/internal/store/project_store.go
+++ b/internal/store/project_store.go
@@ -86,6 +86,19 @@ func (s *pgProjectStore) FindByID(ctx context.Context, id string) (*models.Proje
return &p, nil
}
+func (s *pgProjectStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.Project, error) {
+ var r models.Project
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &r,
+ `SELECT `+projectColumns+` FROM projects WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &r, nil
+}
+
func (s *pgProjectStore) FindByOriginatedID(ctx context.Context, originatedID string) (*models.Project, error) {
var p models.Project
err := s.db.GetContext(ctx, &p,
diff --git a/internal/store/store.go b/internal/store/store.go
index 56898f55e..d478f6b6f 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -29,6 +29,8 @@ import (
type UserStore interface {
// FindByID returns the user with the given ID, or nil if not found.
FindByID(ctx context.Context, id string) (*models.User, error)
+ // FindByIDTx returns the user locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.User, error)
// FindByEmail returns the user with the given email, or nil if not found.
FindByEmail(ctx context.Context, email string) (*models.User, error)
// List returns a page of users ordered by email, plus the total count.
@@ -85,6 +87,8 @@ type ComputeClusterStore interface {
type ComputeClusterUserStore interface {
// FindByID returns the mapping with the given ID, or nil if not found.
FindByID(ctx context.Context, id string) (*models.ComputeClusterUser, error)
+ // FindByIDTx returns the mapping locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeClusterUser, error)
// FindByPair returns the mapping for a (compute_cluster_id, user_id) pair, or nil if absent.
FindByPair(ctx context.Context, clusterID, userID string) (*models.ComputeClusterUser, error)
// FindByClusterAndLocalUsername returns the mapping for a (local_username, compute_cluster_id) pair, or nil if absent.
@@ -93,6 +97,8 @@ type ComputeClusterUserStore interface {
FindByCluster(ctx context.Context, clusterID string) ([]models.ComputeClusterUser, error)
// FindByUser returns every cluster mapping held by the given Custos user.
FindByUser(ctx context.Context, userID string) ([]models.ComputeClusterUser, error)
+ // FindByUserTx returns the user's mappings locked FOR UPDATE in tx, ordered by compute_cluster_id.
+ FindByUserTx(ctx context.Context, tx *sql.Tx, userID string) ([]models.ComputeClusterUser, error)
// ListByApprovalStatus returns mappings joined with the user and cluster, newest first, plus the total. An empty status means any status.
ListByApprovalStatus(ctx context.Context, status models.ClusterAccountApproval, limit, offset int) ([]ComputeClusterUserWithUser, int, error)
// Create inserts a new mapping within the provided transaction.
@@ -138,6 +144,8 @@ type UserIdentityStore interface {
type ProjectStore interface {
// FindByID returns the project with the given ID, or nil if not found.
FindByID(ctx context.Context, id string) (*models.Project, error)
+ // FindByIDTx returns the project locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.Project, error)
// FindByOriginatedID returns the project matching the external originated ID, or nil if not found.
FindByOriginatedID(ctx context.Context, originatedID string) (*models.Project, error)
// Create inserts a new project within the provided transaction.
@@ -174,6 +182,8 @@ type ProjectListFilter struct {
type ComputeAllocationStore interface {
// FindByID returns the allocation with the given ID, or nil if not found.
FindByID(ctx context.Context, id string) (*models.ComputeAllocation, error)
+ // FindByIDTx returns the allocation locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocation, error)
// FindByProject returns all allocations attached to the given project.
FindByProject(ctx context.Context, projectID string) ([]models.ComputeAllocation, error)
// FindByCluster returns all allocations attached to the given compute cluster.
@@ -301,6 +311,8 @@ type ComputeAllocationDiffStore interface {
type ComputeAllocationChangeRequestStore interface {
// FindByID returns the change request with the given ID, or nil if it does not exist.
FindByID(ctx context.Context, id string) (*models.ComputeAllocationChangeRequest, error)
+ // FindByIDTx returns the change request locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocationChangeRequest, error)
// FindByAllocation returns every change request ever recorded against the
// given allocation, ordered by timestamp ascending.
FindByAllocation(ctx context.Context, allocationID string) ([]models.ComputeAllocationChangeRequest, error)
@@ -366,6 +378,8 @@ type ProjectMembershipStore interface {
type ComputeAllocationMembershipStore interface {
// FindByID returns the membership with the given ID, or nil if it does not exist.
FindByID(ctx context.Context, id string) (*models.ComputeAllocationMembership, error)
+ // FindByIDTx returns the membership locked FOR UPDATE in tx, or nil if not found.
+ FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.ComputeAllocationMembership, error)
// FindByPair returns the membership for a (allocation, user) pair, or nil if absent.
FindByPair(ctx context.Context, allocationID, userID string) (*models.ComputeAllocationMembership, error)
// FindByUser returns every membership held by the given user, ordered by
diff --git a/internal/store/user_store.go b/internal/store/user_store.go
index 566adf4a8..7191536e0 100644
--- a/internal/store/user_store.go
+++ b/internal/store/user_store.go
@@ -52,6 +52,19 @@ func (s *pgUserStore) FindByID(ctx context.Context, id string) (*models.User, er
return &u, nil
}
+func (s *pgUserStore) FindByIDTx(ctx context.Context, tx *sql.Tx, id string) (*models.User, error) {
+ var r models.User
+ err := (&sqlx.Tx{Tx: tx, Mapper: s.db.Mapper}).GetContext(ctx, &r,
+ `SELECT `+userColumns+` FROM users WHERE id = $1 FOR UPDATE`, id)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &r, nil
+}
+
func (s *pgUserStore) List(ctx context.Context, limit, offset int) ([]models.User, int, error) {
var total int
if err := s.db.GetContext(ctx, &total, `SELECT COUNT(*) FROM users`); err != nil {
diff --git a/pkg/service/compute_allocation.go b/pkg/service/compute_allocation.go
index 50d6bf442..e30a266b8 100644
--- a/pkg/service/compute_allocation.go
+++ b/pkg/service/compute_allocation.go
@@ -161,15 +161,21 @@ func (s *Service) ListComputeAllocationsByCluster(ctx context.Context, clusterID
// UpdateComputeAllocation persists changes to an existing compute allocation.
func (s *Service) UpdateComputeAllocation(ctx context.Context, alloc *models.ComputeAllocation) error {
+ return s.inTx(ctx, func(tx *sql.Tx) error {
+ return s.UpdateComputeAllocationTx(ctx, tx, alloc)
+ })
+}
+
+func (s *Service) UpdateComputeAllocationTx(ctx context.Context, tx *sql.Tx, alloc *models.ComputeAllocation) error {
if alloc == nil || alloc.ID == "" {
return fmt.Errorf("%w: compute allocation id is required", ErrInvalidInput)
}
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := func() error {
if err := s.allocs.Update(ctx, tx, alloc); err != nil {
return err
}
return s.eventBus.Publish(ctx, tx, events.ComputeAllocationUpdateEvent, alloc)
- }); err != nil {
+ }(); err != nil {
return fmt.Errorf("update compute allocation: %w", err)
}
return nil
diff --git a/pkg/service/compute_allocation_change_request.go b/pkg/service/compute_allocation_change_request.go
index 6c2e4be30..d44fc5e1d 100644
--- a/pkg/service/compute_allocation_change_request.go
+++ b/pkg/service/compute_allocation_change_request.go
@@ -140,6 +140,9 @@ func (s *Service) UpdateComputeAllocationChangeRequest(ctx context.Context, req
if existing == nil {
return nil, ErrNotFound
}
+ if req.ChangeStatus != "" && existing.ChangeStatus != "PENDING" {
+ return nil, fmt.Errorf("%w: change request is already %s", ErrInvalidInput, existing.ChangeStatus)
+ }
if req.ComputeAllocationID == "" {
req.ComputeAllocationID = existing.ComputeAllocationID
}
@@ -175,34 +178,47 @@ func (s *Service) UpdateComputeAllocationChangeRequest(ctx context.Context, req
approved := req.ChangeStatus == "APPROVED" && existing.ChangeStatus != "APPROVED"
if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ locked, err := s.changeRequests.FindByIDTx(ctx, tx, req.ID)
+ if err != nil {
+ return err
+ }
+ if locked == nil {
+ return ErrNotFound
+ }
+ if locked.ChangeStatus != existing.ChangeStatus {
+ return fmt.Errorf("%w: change request changed concurrently", ErrInvalidInput)
+ }
if err := s.changeRequests.Update(ctx, tx, req); err != nil {
return err
}
- return s.changeEvents.Create(ctx, tx, &models.ComputeAllocationChangeRequestEvent{
+ if err := s.changeEvents.Create(ctx, tx, &models.ComputeAllocationChangeRequestEvent{
ID: newID(),
ComputeAllocationChangeRequestID: req.ID,
EventType: eventType,
Description: eventDescription,
Timestamp: nowUTC(),
- })
- }); err != nil {
- return nil, fmt.Errorf("update compute allocation change request: %w", err)
- }
-
- if approved {
+ }); err != nil {
+ return err
+ }
+ if !approved {
+ return nil
+ }
diffStatus := req.RequestedStatus
if diffStatus == "" {
diffStatus = models.ACTIVE
}
- if _, err := s.CreateComputeAllocationDiff(ctx, &models.ComputeAllocationDiff{
+ if _, err := s.CreateComputeAllocationDiffTx(ctx, tx, &models.ComputeAllocationDiff{
ComputeAllocationID: req.ComputeAllocationID,
DiffType: "CHANGE_REQUEST_APPROVED",
NewSUAmount: req.RequestedSUAmount,
Status: diffStatus,
Description: fmt.Sprintf("Applied approved change request %s", req.ID),
}); err != nil {
- return nil, fmt.Errorf("record approved change request diff: %w", err)
+ return fmt.Errorf("record approved change request diff: %w", err)
}
+ return nil
+ }); err != nil {
+ return nil, fmt.Errorf("update compute allocation change request: %w", err)
}
return req, nil
}
diff --git a/pkg/service/compute_allocation_change_request_integration_test.go b/pkg/service/compute_allocation_change_request_integration_test.go
index 835c49fdd..dc6dff3cf 100644
--- a/pkg/service/compute_allocation_change_request_integration_test.go
+++ b/pkg/service/compute_allocation_change_request_integration_test.go
@@ -21,6 +21,8 @@ package service
import (
"fmt"
+ "sync"
+ "sync/atomic"
"testing"
"github.com/google/uuid"
@@ -50,3 +52,33 @@ func TestApproveChangeRequestKeepsRequestedFields(t *testing.T) {
t.Fatalf("approved request lost its fields: %+v", got)
}
}
+
+func TestChangeRequestIsDecidedOnce(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestService(database)
+ pi := seedUser(t, database, fmt.Sprintf("cr-%s@example.edu", uuid.NewString()))
+ _, allocID := newClusterAllocation(t, svc, pi)
+ cr, err := svc.CreateComputeAllocationChangeRequest(ctx(), &models.ComputeAllocationChangeRequest{
+ ComputeAllocationID: allocID, RequesterID: pi, RequestedSUAmount: 1500, RequestedStatus: models.ACTIVE, Reason: "more compute",
+ })
+ if err != nil {
+ t.Fatalf("create: %v", err)
+ }
+ var applied atomic.Int32
+ var wg sync.WaitGroup
+ for range 5 {
+ wg.Go(func() {
+ if _, err := svc.UpdateComputeAllocationChangeRequest(ctx(), &models.ComputeAllocationChangeRequest{ID: cr.ID, ChangeStatus: "APPROVED"}); err == nil {
+ applied.Add(1)
+ }
+ })
+ }
+ wg.Wait()
+ if n := applied.Load(); n != 1 {
+ t.Fatalf("approvals applied: got %d, want 1", n)
+ }
+ var diffs int
+ if err := database.Get(&diffs, "SELECT COUNT(*) FROM compute_allocation_diffs WHERE compute_allocation_id = $1 AND diff_type = 'CHANGE_REQUEST_APPROVED'", allocID); err != nil || diffs != 1 {
+ t.Fatalf("approval diffs: got %d (%v), want 1", diffs, err)
+ }
+}
diff --git a/pkg/service/compute_allocation_diff.go b/pkg/service/compute_allocation_diff.go
index 881d0cef5..a235389da 100644
--- a/pkg/service/compute_allocation_diff.go
+++ b/pkg/service/compute_allocation_diff.go
@@ -30,7 +30,15 @@ import (
// The referenced allocation must exist. If Timestamp is zero the server's
// current UTC time is used. Diff records are intended to be append-only and
// have no update operation.
-func (s *Service) CreateComputeAllocationDiff(ctx context.Context, diff *models.ComputeAllocationDiff) (*models.ComputeAllocationDiff, error) {
+func (s *Service) CreateComputeAllocationDiff(ctx context.Context, diff *models.ComputeAllocationDiff) (d *models.ComputeAllocationDiff, err error) {
+ err = s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ d, err = s.CreateComputeAllocationDiffTx(ctx, tx, diff)
+ return
+ })
+ return
+}
+
+func (s *Service) CreateComputeAllocationDiffTx(ctx context.Context, tx *sql.Tx, diff *models.ComputeAllocationDiff) (*models.ComputeAllocationDiff, error) {
if diff == nil {
return nil, fmt.Errorf("%w: compute allocation diff is nil", ErrInvalidInput)
}
@@ -44,7 +52,7 @@ func (s *Service) CreateComputeAllocationDiff(ctx context.Context, diff *models.
return nil, fmt.Errorf("%w: status is required", ErrInvalidInput)
}
- if alloc, err := s.allocs.FindByID(ctx, diff.ComputeAllocationID); err != nil {
+ if alloc, err := s.allocs.FindByIDTx(ctx, tx, diff.ComputeAllocationID); err != nil {
return nil, fmt.Errorf("lookup compute allocation: %w", err)
} else if alloc == nil {
return nil, fmt.Errorf("%w: compute allocation %q not found", ErrInvalidInput, diff.ComputeAllocationID)
@@ -57,12 +65,12 @@ func (s *Service) CreateComputeAllocationDiff(ctx context.Context, diff *models.
diff.Timestamp = nowUTC()
}
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := func() error {
if err := s.allocDiffs.Create(ctx, tx, diff); err != nil {
return err
}
return s.eventBus.Publish(ctx, tx, events.ComputeAllocationDiffCreateEvent, diff)
- }); err != nil {
+ }(); err != nil {
return nil, fmt.Errorf("create compute allocation diff: %w", err)
}
return diff, nil
diff --git a/pkg/service/compute_allocation_membership.go b/pkg/service/compute_allocation_membership.go
index 517679187..f4ade864f 100644
--- a/pkg/service/compute_allocation_membership.go
+++ b/pkg/service/compute_allocation_membership.go
@@ -172,14 +172,22 @@ func (s *Service) UpdateComputeAllocationMembership(ctx context.Context, m *mode
// UpdateMembershipStatus sets the lifecycle status (ACTIVE, INACTIVE, etc.)
// of the membership identified by the given ID. Other fields are preserved.
-func (s *Service) UpdateMembershipStatus(ctx context.Context, id string, status models.AllocationStatus) (*models.ComputeAllocationMembership, error) {
+func (s *Service) UpdateMembershipStatus(ctx context.Context, id string, status models.AllocationStatus) (m *models.ComputeAllocationMembership, err error) {
+ err = s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ m, err = s.UpdateMembershipStatusTx(ctx, tx, id, status)
+ return
+ })
+ return
+}
+
+func (s *Service) UpdateMembershipStatusTx(ctx context.Context, tx *sql.Tx, id string, status models.AllocationStatus) (*models.ComputeAllocationMembership, error) {
if id == "" {
return nil, fmt.Errorf("%w: compute allocation membership id is required", ErrInvalidInput)
}
if status == "" {
return nil, fmt.Errorf("%w: membership_status is required", ErrInvalidInput)
}
- existing, err := s.memberships.FindByID(ctx, id)
+ existing, err := s.memberships.FindByIDTx(ctx, tx, id)
if err != nil {
return nil, fmt.Errorf("lookup compute allocation membership: %w", err)
}
@@ -187,12 +195,12 @@ func (s *Service) UpdateMembershipStatus(ctx context.Context, id string, status
return nil, ErrNotFound
}
existing.MembershipStatus = status
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := func() error {
if err := s.memberships.Update(ctx, tx, existing); err != nil {
return err
}
return s.eventBus.Publish(ctx, tx, events.ComputeAllocationMembershipUpdateEvent, existing)
- }); err != nil {
+ }(); err != nil {
return nil, fmt.Errorf("update compute allocation membership status: %w", err)
}
return existing, nil
diff --git a/pkg/service/compute_cluster_user.go b/pkg/service/compute_cluster_user.go
index c3efc2194..62ae9d2bc 100644
--- a/pkg/service/compute_cluster_user.go
+++ b/pkg/service/compute_cluster_user.go
@@ -157,21 +157,20 @@ func (s *Service) ListComputeClusterUsersByApproval(ctx context.Context, status
// ApproveComputeClusterUser records the admin's approval and publishes the
// `approve event`, which is what creates the account on the cluster. A denied account can be approved later.
func (s *Service) ApproveComputeClusterUser(ctx context.Context, id, reviewerID string) (*models.ComputeClusterUser, error) {
- cu, err := s.clusterUserForReview(ctx, id, reviewerID)
- if err != nil {
- return nil, err
- }
-
- // Only a denied or pending cluster account can be approved
- if cu.ApprovalStatus == models.ClusterAccountApproved {
- return nil, fmt.Errorf("%w: cluster user %q is already approved", ErrAlreadyExists, id)
- }
- now := nowUTC()
- cu.ApprovalStatus = models.ClusterAccountApproved
- cu.ReviewedAt = &now
- cu.ReviewedBy = &reviewerID
- cu.ReviewNote = nil
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ var cu *models.ComputeClusterUser
+ if err := s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ if cu, err = s.clusterUserForReview(ctx, tx, id, reviewerID); err != nil {
+ return err
+ }
+ // Only a denied or pending cluster account can be approved
+ if cu.ApprovalStatus == models.ClusterAccountApproved {
+ return fmt.Errorf("%w: cluster user %q is already approved", ErrAlreadyExists, id)
+ }
+ now := nowUTC()
+ cu.ApprovalStatus = models.ClusterAccountApproved
+ cu.ReviewedAt = &now
+ cu.ReviewedBy = &reviewerID
+ cu.ReviewNote = nil
if err := s.clusterUsers.Review(ctx, tx, id, models.ClusterAccountApproved, reviewerID, "", now); err != nil {
return err
}
@@ -198,22 +197,22 @@ func (s *Service) DenyComputeClusterUser(ctx context.Context, id, reviewerID, no
if note == "" {
return nil, fmt.Errorf("%w: a reason is required to deny", ErrInvalidInput)
}
- cu, err := s.clusterUserForReview(ctx, id, reviewerID)
- if err != nil {
- return nil, err
- }
- switch cu.ApprovalStatus {
- case models.ClusterAccountApproved:
- return nil, fmt.Errorf("%w: cluster user %q is already approved, remove it instead", ErrAlreadyExists, id)
- case models.ClusterAccountDenied:
- return nil, fmt.Errorf("%w: cluster user %q is already denied", ErrAlreadyExists, id)
- }
- now := nowUTC()
- cu.ApprovalStatus = models.ClusterAccountDenied
- cu.ReviewedAt = &now
- cu.ReviewedBy = &reviewerID
- cu.ReviewNote = ¬e
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ var cu *models.ComputeClusterUser
+ if err := s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ if cu, err = s.clusterUserForReview(ctx, tx, id, reviewerID); err != nil {
+ return err
+ }
+ switch cu.ApprovalStatus {
+ case models.ClusterAccountApproved:
+ return fmt.Errorf("%w: cluster user %q is already approved, remove it instead", ErrAlreadyExists, id)
+ case models.ClusterAccountDenied:
+ return fmt.Errorf("%w: cluster user %q is already denied", ErrAlreadyExists, id)
+ }
+ now := nowUTC()
+ cu.ApprovalStatus = models.ClusterAccountDenied
+ cu.ReviewedAt = &now
+ cu.ReviewedBy = &reviewerID
+ cu.ReviewNote = ¬e
if err := s.clusterUsers.Review(ctx, tx, id, models.ClusterAccountDenied, reviewerID, note, now); err != nil {
return err
}
@@ -233,14 +232,14 @@ func (s *Service) DenyComputeClusterUser(ctx context.Context, id, reviewerID, no
return cu, nil
}
-func (s *Service) clusterUserForReview(ctx context.Context, id, reviewerID string) (*models.ComputeClusterUser, error) {
+func (s *Service) clusterUserForReview(ctx context.Context, tx *sql.Tx, id, reviewerID string) (*models.ComputeClusterUser, error) {
if id == "" {
return nil, fmt.Errorf("%w: compute cluster user id is required", ErrInvalidInput)
}
if reviewerID == "" {
return nil, fmt.Errorf("%w: reviewer is required", ErrInvalidInput)
}
- cu, err := s.clusterUsers.FindByID(ctx, id)
+ cu, err := s.clusterUsers.FindByIDTx(ctx, tx, id)
if err != nil {
return nil, fmt.Errorf("lookup compute cluster user: %w", err)
}
diff --git a/pkg/service/compute_cluster_user_integration_test.go b/pkg/service/compute_cluster_user_integration_test.go
index cf4e77456..1016634de 100644
--- a/pkg/service/compute_cluster_user_integration_test.go
+++ b/pkg/service/compute_cluster_user_integration_test.go
@@ -24,6 +24,7 @@ import (
"errors"
"fmt"
"slices"
+ "sync"
"testing"
"time"
@@ -445,3 +446,37 @@ func TestCreateComputeAllocation_SuffixesACollidingPosixGroup(t *testing.T) {
t.Fatalf("groups %v, want %s, -2, -3", groups, want)
}
}
+
+func TestDenyComputeClusterUser_ConcurrentDenialsRecordOnce(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestService(database)
+ cu := newClusterUser(t, svc, database)
+ admin := seedUser(t, database, fmt.Sprintf("admin-%s@example.edu", uuid.NewString()))
+
+ hold, err := database.Begin()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hold.Exec("SELECT id FROM compute_cluster_users WHERE id = $1 FOR UPDATE", cu.ID); err != nil {
+ t.Fatal(err)
+ }
+ var wg sync.WaitGroup
+ for range 2 {
+ wg.Go(func() { svc.DenyComputeClusterUser(ctx(), cu.ID, admin, "no") })
+ }
+ for n, deadline := 0, time.Now().Add(5*time.Second); n < 2; time.Sleep(10 * time.Millisecond) {
+ if time.Now().After(deadline) {
+ t.Fatal("reviewers never blocked on the row lock")
+ }
+ if err := database.Get(&n, "SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'Lock'"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := hold.Commit(); err != nil {
+ t.Fatal(err)
+ }
+ wg.Wait()
+ if n := countAuditEventsOfType(t, database, clusterUserAuditDenied, cu.ID); n != 1 {
+ t.Fatalf("denial audits: got %d, want 1", n)
+ }
+}
diff --git a/pkg/service/project.go b/pkg/service/project.go
index ffc3629bc..a01aa19e3 100644
--- a/pkg/service/project.go
+++ b/pkg/service/project.go
@@ -214,14 +214,22 @@ func (s *Service) UpdateProject(ctx context.Context, project *models.Project, pr
// UpdateProjectStatus sets the lifecycle status of the project identified by
// id. Other fields are preserved.
-func (s *Service) UpdateProjectStatus(ctx context.Context, id string, status models.ProjectStatus) (*models.Project, error) {
+func (s *Service) UpdateProjectStatus(ctx context.Context, id string, status models.ProjectStatus) (p *models.Project, err error) {
+ err = s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ p, err = s.UpdateProjectStatusTx(ctx, tx, id, status)
+ return
+ })
+ return
+}
+
+func (s *Service) UpdateProjectStatusTx(ctx context.Context, tx *sql.Tx, id string, status models.ProjectStatus) (*models.Project, error) {
if id == "" {
return nil, fmt.Errorf("%w: project id is required", ErrInvalidInput)
}
if status == "" {
return nil, fmt.Errorf("%w: status is required", ErrInvalidInput)
}
- existing, err := s.projs.FindByID(ctx, id)
+ existing, err := s.projs.FindByIDTx(ctx, tx, id)
if err != nil {
return nil, fmt.Errorf("lookup project: %w", err)
}
@@ -229,12 +237,12 @@ func (s *Service) UpdateProjectStatus(ctx context.Context, id string, status mod
return nil, ErrNotFound
}
existing.Status = status
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := func() error {
if err := s.projs.UpdateStatus(ctx, tx, id, status); err != nil {
return err
}
return s.eventBus.Publish(ctx, tx, events.ProjectUpdateEvent, existing)
- }); err != nil {
+ }(); err != nil {
return nil, fmt.Errorf("update project status: %w", err)
}
return existing, nil
diff --git a/pkg/service/user_merge.go b/pkg/service/user_merge.go
index faea905c7..3919affb2 100644
--- a/pkg/service/user_merge.go
+++ b/pkg/service/user_merge.go
@@ -47,7 +47,15 @@ import (
// Not idempotent — a retiring user already in status=MERGED is rejected with
// ErrAlreadyExists. Callers that need replay safety should fetch the retiring
// user by ID and skip the call when its status is already MERGED.
-func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string) (*models.User, error) {
+func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string) (survivor *models.User, err error) {
+ err = s.inTx(ctx, func(tx *sql.Tx) (err error) {
+ survivor, err = s.MergeUsersTx(ctx, tx, survivingID, retiringID)
+ return
+ })
+ return
+}
+
+func (s *Service) MergeUsersTx(ctx context.Context, tx *sql.Tx, survivingID, retiringID string) (*models.User, error) {
if survivingID == "" || retiringID == "" {
return nil, fmt.Errorf("%w: surviving and retiring user IDs are required", ErrInvalidInput)
}
@@ -55,7 +63,7 @@ func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string
return nil, fmt.Errorf("%w: cannot merge a user with itself", ErrInvalidInput)
}
- survivor, err := s.users.FindByID(ctx, survivingID)
+ survivor, err := s.users.FindByIDTx(ctx, tx, survivingID)
if err != nil {
return nil, fmt.Errorf("lookup surviving user: %w", err)
}
@@ -66,7 +74,7 @@ func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string
return nil, fmt.Errorf("%w: surviving user %q must be ACTIVE or PENDING (got %s)",
ErrInvalidInput, survivingID, survivor.Status)
}
- retiring, err := s.users.FindByID(ctx, retiringID)
+ retiring, err := s.users.FindByIDTx(ctx, tx, retiringID)
if err != nil {
return nil, fmt.Errorf("lookup retiring user: %w", err)
}
@@ -83,11 +91,11 @@ func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string
// The reassigning below deletes the retiring user's cluster account on every
// cluster where the survivor already has one. Those need a delete event.
- retiringAccounts, err := s.clusterUsers.FindByUser(ctx, retiringID)
+ retiringAccounts, err := s.clusterUsers.FindByUserTx(ctx, tx, retiringID)
if err != nil {
return nil, fmt.Errorf("list retiring cluster users: %w", err)
}
- survivorAccounts, err := s.clusterUsers.FindByUser(ctx, survivingID)
+ survivorAccounts, err := s.clusterUsers.FindByUserTx(ctx, tx, survivingID)
if err != nil {
return nil, fmt.Errorf("list surviving cluster users: %w", err)
}
@@ -96,7 +104,7 @@ func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string
survivorClusters[cu.ComputeClusterID] = true
}
- if err := s.inTx(ctx, func(tx *sql.Tx) error {
+ if err := func() error {
if err := s.userIdentities.ReassignUser(ctx, tx, retiringID, survivingID); err != nil {
return fmt.Errorf("reassign user identities: %w", err)
}
@@ -128,7 +136,7 @@ func (s *Service) MergeUsers(ctx context.Context, survivingID, retiringID string
return err
}
return s.eventBus.Publish(ctx, tx, events.UserUpdateEvent, survivor)
- }); err != nil {
+ }(); err != nil {
return nil, fmt.Errorf("merge users: %w", err)
}
s.InvalidateAllIdentities()