You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2026/10/01 17:04:30 UTC
(airavata-custos) 02/02: Hold AMIE account replies until the cluster account is approved
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch amie-account-creation
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit c787910902ceeebc324b300876efe9163e67565f
Author: lahiruj <la...@gmail.com>
AuthorDate: Thu Oct 1 12:41:14 2026 -0400
Hold AMIE account replies until the cluster account is approved
---
.../db/migrations/000001_initial_schema.up.sql | 55 ++++---
.../ACCESS/AMIE-Processor/handler/held_reply.go | 123 ++++++++++++++
.../handler/held_reply_integration_test.go | 183 +++++++++++++++++++++
.../AMIE-Processor/handler/integration_common.go | 33 ++++
.../handler/request_account_create.go | 7 +-
.../request_account_create_integration_test.go | 69 ++++++--
.../AMIE-Processor/handler/request_person_merge.go | 24 ++-
.../request_person_merge_integration_test.go | 50 +++++-
.../handler/request_project_create.go | 4 +
connectors/ACCESS/AMIE-Processor/model/audit.go | 1 +
connectors/ACCESS/AMIE-Processor/model/packet.go | 9 +
.../pipeline/baseline_integration_test.go | 8 +-
.../AMIE-Processor/pipeline/integration_common.go | 43 ++++-
.../ACCESS/AMIE-Processor/pkg/amie/loader.go | 4 +-
.../ACCESS/AMIE-Processor/store/event_store.go | 12 ++
.../ACCESS/AMIE-Processor/store/packet_store.go | 18 +-
.../testdata/scenarios/baseline.yaml | 5 +-
.../ACCESS/AMIE-Processor/worker/processor.go | 11 +-
18 files changed, 591 insertions(+), 68 deletions(-)
diff --git a/connectors/ACCESS/AMIE-Processor/db/migrations/000001_initial_schema.up.sql b/connectors/ACCESS/AMIE-Processor/db/migrations/000001_initial_schema.up.sql
index 9346b4f78..a469f9e91 100644
--- a/connectors/ACCESS/AMIE-Processor/db/migrations/000001_initial_schema.up.sql
+++ b/connectors/ACCESS/AMIE-Processor/db/migrations/000001_initial_schema.up.sql
@@ -20,34 +20,39 @@
CREATE TABLE IF NOT EXISTS amie_packets
(
- id VARCHAR(255) NOT NULL,
- amie_id BIGINT NOT NULL,
- type VARCHAR(64) NOT NULL,
- status VARCHAR(32) NOT NULL,
- raw_json TEXT NOT NULL,
+ id VARCHAR(255) NOT NULL,
+ amie_id BIGINT NOT NULL,
+ type VARCHAR(64) NOT NULL,
+ status VARCHAR(32) NOT NULL,
+ raw_json TEXT NOT NULL,
received_at TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
decoded_at TIMESTAMPTZ(6) NULL,
processed_at TIMESTAMPTZ(6) NULL,
- retries INT NOT NULL DEFAULT 0,
- last_error TEXT NULL,
+ retries INT NOT NULL DEFAULT 0,
+ last_error TEXT NULL,
+ -- A reply that has to wait for something on Custos side is kept here, with
+ -- held_for naming what it waits on. Used for cluster account approval.
+ held_reply TEXT NULL,
+ held_for VARCHAR(255) NULL,
PRIMARY KEY (id),
CONSTRAINT uq_amie_packets_amie_id UNIQUE (amie_id)
);
CREATE INDEX IF NOT EXISTS idx_amie_packets_status ON amie_packets (status);
+CREATE INDEX IF NOT EXISTS idx_amie_packets_held_for ON amie_packets (held_for);
CREATE INDEX IF NOT EXISTS idx_amie_packets_received_at ON amie_packets (received_at);
CREATE TABLE IF NOT EXISTS amie_processing_events
(
- id VARCHAR(255) NOT NULL,
- packet_id VARCHAR(255) NOT NULL,
- type VARCHAR(64) NOT NULL,
- status VARCHAR(32) NOT NULL,
- attempts INT NOT NULL DEFAULT 0,
+ id VARCHAR(255) NOT NULL,
+ packet_id VARCHAR(255) NOT NULL,
+ type VARCHAR(64) NOT NULL,
+ status VARCHAR(32) NOT NULL,
+ attempts INT NOT NULL DEFAULT 0,
created_at TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
started_at TIMESTAMPTZ(6) NULL,
finished_at TIMESTAMPTZ(6) NULL,
- last_error TEXT NULL,
- next_retry_at TIMESTAMPTZ(6) NULL DEFAULT NULL,
+ last_error TEXT NULL,
+ next_retry_at TIMESTAMPTZ(6) NULL DEFAULT NULL,
PRIMARY KEY (id),
CONSTRAINT fk_amie_events_packet FOREIGN KEY (packet_id) REFERENCES amie_packets (id) ON DELETE CASCADE,
CONSTRAINT uq_amie_events_packet_type UNIQUE (packet_id, type)
@@ -59,12 +64,12 @@ CREATE INDEX IF NOT EXISTS idx_amie_events_next_retry_at ON amie_processing_even
CREATE TABLE IF NOT EXISTS amie_processing_errors
(
- id BIGINT GENERATED BY DEFAULT AS IDENTITY,
- packet_id VARCHAR(255) NULL,
- event_id VARCHAR(255) NULL,
+ id BIGINT GENERATED BY DEFAULT AS IDENTITY,
+ packet_id VARCHAR(255) NULL,
+ event_id VARCHAR(255) NULL,
occurred_at TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
- summary TEXT NOT NULL,
- detail TEXT NULL,
+ summary TEXT NOT NULL,
+ detail TEXT NULL,
PRIMARY KEY (id),
CONSTRAINT fk_amie_errors_packet FOREIGN KEY (packet_id) REFERENCES amie_packets (id) ON DELETE SET NULL,
CONSTRAINT fk_amie_errors_event FOREIGN KEY (event_id) REFERENCES amie_processing_events (id) ON DELETE SET NULL
@@ -82,9 +87,9 @@ CREATE TABLE IF NOT EXISTS amie_audit_extras
packet_id VARCHAR(255) NOT NULL,
event_id VARCHAR(255) NULL,
PRIMARY KEY (audit_event_id),
- CONSTRAINT fk_amie_audit_extras_event FOREIGN KEY (audit_event_id) REFERENCES audit_events(id) ON DELETE CASCADE,
- CONSTRAINT fk_amie_audit_extras_packet FOREIGN KEY (packet_id) REFERENCES amie_packets(id) ON DELETE CASCADE,
- CONSTRAINT fk_amie_audit_extras_procev FOREIGN KEY (event_id) REFERENCES amie_processing_events(id) ON DELETE SET NULL
+ CONSTRAINT fk_amie_audit_extras_event FOREIGN KEY (audit_event_id) REFERENCES audit_events (id) ON DELETE CASCADE,
+ CONSTRAINT fk_amie_audit_extras_packet FOREIGN KEY (packet_id) REFERENCES amie_packets (id) ON DELETE CASCADE,
+ CONSTRAINT fk_amie_audit_extras_procev FOREIGN KEY (event_id) REFERENCES amie_processing_events (id) ON DELETE SET NULL
);
CREATE INDEX IF NOT EXISTS idx_amie_audit_extras_event_id ON amie_audit_extras (event_id);
CREATE INDEX IF NOT EXISTS idx_amie_audit_extras_packet_id ON amie_audit_extras (packet_id);
@@ -96,9 +101,9 @@ CREATE INDEX IF NOT EXISTS idx_amie_audit_extras_packet_id ON amie_audit_extras
-- stays independent of core.
CREATE TABLE IF NOT EXISTS amie_user_dns
(
- id VARCHAR(255) NOT NULL,
- user_id VARCHAR(255) NOT NULL,
- dn VARCHAR(512) NOT NULL,
+ id VARCHAR(255) NOT NULL,
+ user_id VARCHAR(255) NOT NULL,
+ dn VARCHAR(512) NOT NULL,
created_at TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
PRIMARY KEY (id),
CONSTRAINT uq_amie_user_dns_dn UNIQUE (dn)
diff --git a/connectors/ACCESS/AMIE-Processor/handler/held_reply.go b/connectors/ACCESS/AMIE-Processor/handler/held_reply.go
new file mode 100644
index 000000000..9df1efb6f
--- /dev/null
+++ b/connectors/ACCESS/AMIE-Processor/handler/held_reply.go
@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package handler
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "fmt"
+
+ "github.com/jmoiron/sqlx"
+
+ custosdb "github.com/apache/airavata-custos/connectors/ACCESS/AMIE-Processor/db"
+ "github.com/apache/airavata-custos/connectors/ACCESS/AMIE-Processor/model"
+ "github.com/apache/airavata-custos/connectors/ACCESS/AMIE-Processor/store"
+ "github.com/apache/airavata-custos/pkg/events"
+ "github.com/apache/airavata-custos/pkg/models"
+)
+
+// replyWhenAccountApproved replies to ACCESS only if the cluster account is
+// approved. Otherwise, the reply is held and sent once an admin approves it.
+func replyWhenAccountApproved(ctx context.Context, tx *sql.Tx, client AmieClient, auditSvc AuditService, packet *model.Packet, eventID string, account *models.ComputeClusterUser, reply map[string]any) error {
+ if account.ApprovalStatus == models.ClusterAccountApproved {
+ if err := client.ReplyToPacket(ctx, packet.AmieID, reply); err != nil {
+ return fmt.Errorf("sending reply: %w", err)
+ }
+ // A packet processed again can still carry an earlier held reply.
+ packet.HeldReply, packet.HeldFor = nil, nil
+ return auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditReplySent, "reply", "", "")
+ }
+ raw, err := json.Marshal(reply)
+ if err != nil {
+ return fmt.Errorf("encode held reply: %w", err)
+ }
+ held := string(raw)
+ packet.Status = model.PacketStatusWaitingApproval
+ packet.HeldReply = &held
+ packet.HeldFor = &account.ID
+ return auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditReplyHeld, "compute_cluster_user", account.ID, account.LocalUsername)
+}
+
+// HeldReplies answers packets that are held for a reason, for example a
+// cluster account approval.
+type HeldReplies struct {
+ db *sqlx.DB
+ packets store.PacketStore
+ client AmieClient
+ auditSvc AuditService
+}
+
+func NewHeldReplies(db *sqlx.DB, packets store.PacketStore, client AmieClient, auditSvc AuditService) *HeldReplies {
+ return &HeldReplies{db: db, packets: packets, client: client, auditSvc: auditSvc}
+}
+
+func (h *HeldReplies) Subscribe(bus *events.Bus, subscriber string) {
+ bus.SubscribeComputeClusterUserApproved(subscriber, h.approved)
+ bus.SubscribeComputeClusterUserDenied(subscriber, h.denied)
+}
+
+func (h *HeldReplies) approved(ctx context.Context, cu models.ComputeClusterUser) error {
+ return h.answer(ctx, cu.ID, model.PacketStatusDecoded, nil)
+}
+
+// denied ends the transaction on the ACCESS side with a failure that carries
+// the admin's reason, which the protocol allows in reply to any packet.
+func (h *HeldReplies) denied(ctx context.Context, cu models.ComputeClusterUser) error {
+ reason := ""
+ if cu.ReviewNote != nil {
+ reason = *cu.ReviewNote
+ }
+ return h.answer(ctx, cu.ID, model.PacketStatusRefused, map[string]any{
+ "type": "inform_transaction_complete",
+ "body": map[string]any{"StatusCode": "Failure", "DetailCode": 2, "Message": reason},
+ })
+}
+
+// answer sends each packet its held reply, or the failure when one is given.
+// Each packet gets its own transaction, so a failed send leaves only that
+// packet waiting for the bus to retry.
+func (h *HeldReplies) answer(ctx context.Context, clusterUserID string, status model.PacketStatus, failure map[string]any) error {
+ packets, err := h.packets.ListWaitingFor(ctx, clusterUserID)
+ if err != nil {
+ return fmt.Errorf("list held packets: %w", err)
+ }
+ for _, p := range packets {
+ packetReply := failure
+ // No failure means approval, so send the reply held on this packet.
+ if packetReply == nil {
+ if err := json.Unmarshal([]byte(*p.HeldReply), &packetReply); err != nil {
+ return fmt.Errorf("decode held reply of packet %s: %w", p.ID, err)
+ }
+ }
+ if err := custosdb.TxFn(ctx, h.db, func(tx *sql.Tx) error {
+ if err := h.client.ReplyToPacket(ctx, p.AmieID, packetReply); err != nil {
+ return fmt.Errorf("reply to packet %s: %w", p.ID, err)
+ }
+ p.Status = status
+ p.HeldReply, p.HeldFor = nil, nil
+ if err := h.packets.Update(ctx, tx, &p); err != nil {
+ return err
+ }
+ return h.auditSvc.Log(ctx, tx, p.ID, "", model.AuditReplySent, "reply", "", string(status))
+ }); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/held_reply_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/held_reply_integration_test.go
new file mode 100644
index 000000000..4cada00b1
--- /dev/null
+++ b/connectors/ACCESS/AMIE-Processor/handler/held_reply_integration_test.go
@@ -0,0 +1,183 @@
+//go:build integration
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package handler
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "testing"
+
+ "github.com/jmoiron/sqlx"
+
+ "github.com/apache/airavata-custos/connectors/ACCESS/AMIE-Processor/model"
+ "github.com/apache/airavata-custos/connectors/ACCESS/AMIE-Processor/store"
+ "github.com/apache/airavata-custos/pkg/models"
+ coreservice "github.com/apache/airavata-custos/pkg/service"
+)
+
+// holdAccountReply runs request_account_create for a new person and saves the held packet, the way the processor would.
+func holdAccountReply(t *testing.T, database *sqlx.DB, svc *coreservice.Service) *model.Packet {
+ t.Helper()
+ body := baseRACBody()
+ body["ProjectID"] = seedProjectForRAC(t, database)
+ pkt := insertPacket(t, database, "request_account_create", body)
+ h := NewRequestAccountCreateHandler(svc, testClusterID, &fakeAmieClient{}, newTestAuditService(database))
+ if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
+ if err := h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, ""); err != nil {
+ return err
+ }
+ return store.NewPacketStore(database).Update(ctx, tx, pkt)
+ }); err != nil {
+ t.Fatalf("hold reply: %v", err)
+ }
+ return pkt
+}
+
+func reviewedAccount(t *testing.T, svc *coreservice.Service, id string) models.ComputeClusterUser {
+ t.Helper()
+ cu, err := svc.GetComputeClusterUser(context.Background(), id)
+ if err != nil {
+ t.Fatalf("get cluster account: %v", err)
+ }
+ return *cu
+}
+
+func packetStatus(t *testing.T, database *sqlx.DB, id string) model.PacketStatus {
+ t.Helper()
+ var status model.PacketStatus
+ if err := database.Get(&status, "SELECT status FROM amie_packets WHERE id = $1", id); err != nil {
+ t.Fatalf("read packet status: %v", err)
+ }
+ return status
+}
+
+// Make sure approving the cluster account sends the held reply to ACCESS once, even
+// when the bus delivers the approve event a second time.
+func TestHeldReplies_ApprovalSendsHeldReplyOnce(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestCoreService(database)
+ pkt := holdAccountReply(t, database, svc)
+ approvePendingAccounts(t, database, svc)
+
+ amie := &fakeAmieClient{}
+ held := NewHeldReplies(database, store.NewPacketStore(database), amie, newTestAuditService(database))
+ account := reviewedAccount(t, svc, *pkt.HeldFor)
+ for range 2 {
+ if err := held.approved(context.Background(), account); err != nil {
+ t.Fatalf("approved: %v", err)
+ }
+ }
+
+ if len(amie.Replies) != 1 || amie.lastReplyType() != "notify_account_create" {
+ t.Fatalf("expected one notify_account_create, got %+v", amie.Replies)
+ }
+ if got := packetStatus(t, database, pkt.ID); got != model.PacketStatusDecoded {
+ t.Errorf("packet status: got %s, want %s", got, model.PacketStatusDecoded)
+ }
+ var stillHeld int
+ if err := database.Get(&stillHeld, "SELECT COUNT(*) FROM amie_packets WHERE id = $1 AND (held_reply IS NOT NULL OR held_for IS NOT NULL)", pkt.ID); err != nil {
+ t.Fatalf("read held fields: %v", err)
+ }
+ if stillHeld != 0 {
+ t.Errorf("expected the held reply to be cleared once it is sent")
+ }
+}
+
+// Make sure a held packet that is processed again after the account is
+// approved gets its reply and no longer carries the old held reply. This is
+// what happens to a held packet after a person merge.
+func TestHeldReplies_ProcessedAgainAfterApprovalClearsHeldReply(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestCoreService(database)
+ pkt := holdAccountReply(t, database, svc)
+ approvePendingAccounts(t, database, svc)
+
+ var packetJSON map[string]any
+ if err := json.Unmarshal([]byte(pkt.RawJSON), &packetJSON); err != nil {
+ t.Fatalf("decode packet: %v", err)
+ }
+ amie := &fakeAmieClient{}
+ h := NewRequestAccountCreateHandler(svc, testClusterID, amie, newTestAuditService(database))
+ pkt.Status = model.PacketStatusDecoded // the processor resets it before the handler runs
+ if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
+ return h.Handle(ctx, tx, packetJSON, pkt, "")
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+
+ if amie.lastReplyType() != "notify_account_create" {
+ t.Fatalf("reply type: got %q, want notify_account_create", amie.lastReplyType())
+ }
+ if pkt.HeldReply != nil || pkt.HeldFor != nil {
+ t.Errorf("expected the old held reply to be cleared, got held_for %v", pkt.HeldFor)
+ }
+}
+
+// Make sure denying the cluster account ends the transaction on the ACCESS side with a
+// failure that carries the admin's reason.
+func TestHeldReplies_DenialSendsFailureWithReason(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestCoreService(database)
+ pkt := holdAccountReply(t, database, svc)
+ var reviewer string
+ if err := database.Get(&reviewer, "SELECT id FROM users LIMIT 1"); err != nil {
+ t.Fatalf("pick a reviewer: %v", err)
+ }
+ if _, err := svc.DenyComputeClusterUser(context.Background(), *pkt.HeldFor, reviewer, "not on the collaborator list"); err != nil {
+ t.Fatalf("deny: %v", err)
+ }
+
+ amie := &fakeAmieClient{}
+ held := NewHeldReplies(database, store.NewPacketStore(database), amie, newTestAuditService(database))
+ if err := held.denied(context.Background(), reviewedAccount(t, svc, *pkt.HeldFor)); err != nil {
+ t.Fatalf("denied: %v", err)
+ }
+
+ if amie.lastReplyType() != "inform_transaction_complete" {
+ t.Fatalf("reply type: got %q, want inform_transaction_complete", amie.lastReplyType())
+ }
+ body := amie.lastReplyBody()
+ if body["StatusCode"] != "Failure" || body["Message"] != "not on the collaborator list" {
+ t.Errorf("expected a failure with the reason, got %v", body)
+ }
+ if got := packetStatus(t, database, pkt.ID); got != model.PacketStatusRefused {
+ t.Errorf("packet status: got %s, want %s", got, model.PacketStatusRefused)
+ }
+}
+
+// Make sure a failed send leaves the packet waiting and returns the error, so
+// the bus retries the delivery.
+func TestHeldReplies_FailedSendKeepsPacketWaiting(t *testing.T) {
+ database := setupTestDB(t)
+ svc := newTestCoreService(database)
+ pkt := holdAccountReply(t, database, svc)
+ approvePendingAccounts(t, database, svc)
+
+ amie := &fakeAmieClient{FailWith: errors.New("simulated AMIE outage")}
+ held := NewHeldReplies(database, store.NewPacketStore(database), amie, newTestAuditService(database))
+ if err := held.approved(context.Background(), reviewedAccount(t, svc, *pkt.HeldFor)); err == nil {
+ t.Fatal("expected an error when the reply fails")
+ }
+ if got := packetStatus(t, database, pkt.ID); got != model.PacketStatusWaitingApproval {
+ t.Errorf("packet status: got %s, want %s", got, model.PacketStatusWaitingApproval)
+ }
+}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/integration_common.go b/connectors/ACCESS/AMIE-Processor/handler/integration_common.go
index 0ef1ab0f4..a6e9028cb 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/integration_common.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/integration_common.go
@@ -206,6 +206,39 @@ func (f *fakeAmieClient) lastReplyType() string {
return t
}
+// heldReply returns the type and body of the reply a handler kept on the packet.
+func heldReply(t *testing.T, pkt *model.Packet) (string, map[string]any) {
+ t.Helper()
+ if pkt.Status != model.PacketStatusWaitingApproval || pkt.HeldReply == nil {
+ t.Fatalf("expected the packet to hold its reply, got status %s", pkt.Status)
+ }
+ var reply map[string]any
+ if err := json.Unmarshal([]byte(*pkt.HeldReply), &reply); err != nil {
+ t.Fatalf("decode held reply: %v", err)
+ }
+ typ, _ := reply["type"].(string)
+ body, _ := reply["body"].(map[string]any)
+ return typ, body
+}
+
+// approvePendingAccounts approves every pending cluster account, as an admin would.
+func approvePendingAccounts(t *testing.T, database *sqlx.DB, svc *coreservice.Service) {
+ t.Helper()
+ var ids []string
+ if err := database.Select(&ids, "SELECT id FROM compute_cluster_users WHERE approval_status = 'PENDING'"); err != nil {
+ t.Fatalf("list pending cluster accounts: %v", err)
+ }
+ var reviewer string
+ if err := database.Get(&reviewer, "SELECT id FROM users LIMIT 1"); err != nil {
+ t.Fatalf("pick a reviewer: %v", err)
+ }
+ for _, id := range ids {
+ if _, err := svc.ApproveComputeClusterUser(context.Background(), id, reviewer); err != nil {
+ t.Fatalf("approve %s: %v", id, err)
+ }
+ }
+}
+
func insertPacket(t *testing.T, database *sqlx.DB, packetType string, body map[string]any) *model.Packet {
t.Helper()
raw, err := json.Marshal(map[string]any{"type": packetType, "body": body})
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
index 340350809..2f479883c 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_create.go
@@ -134,11 +134,8 @@ func (h *RequestAccountCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
replyBody["UserOrgCode"] = v
}
reply := map[string]any{"type": "notify_account_create", "body": replyBody}
- if err := h.amieClient.ReplyToPacket(ctx, packet.AmieID, reply); err != nil {
- return fmt.Errorf("request_account_create: sending reply: %w", err)
- }
- if err := h.auditSvc.Log(ctx, tx, packet.ID, eventID, model.AuditReplySent, "reply", "", ""); err != nil {
- return fmt.Errorf("request_account_create: audit REPLY_SENT: %w", err)
+ if err := replyWhenAccountApproved(ctx, tx, h.amieClient, h.auditSvc, packet, eventID, account, reply); err != nil {
+ return fmt.Errorf("request_account_create: %w", err)
}
return nil
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_account_create_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_account_create_integration_test.go
index 7b332294c..ac17df627 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_account_create_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_account_create_integration_test.go
@@ -217,21 +217,22 @@ func TestRequestAccountCreate_HappyPath(t *testing.T) {
model.AuditCreatePerson,
model.AuditCreateAccount,
model.AuditCreateMembership,
- model.AuditReplySent,
+ model.AuditReplyHeld,
} {
if got := countAuditActions(t, database, pkt.ID, action); got != 1 {
t.Errorf("audit %s: got %d, want 1", action, got)
}
}
- // Reply: notify_account_create with the required fields:
- // AccountActivityTime, ProjectID, ResourceList, UserRemoteSiteLogin.
- if got, want := amie.lastReplyType(), "notify_account_create"; got != want {
- t.Fatalf("reply type: got %q, want %q", got, want)
+ // The cluster account is new and not approved yet, so nothing goes to ACCESS. The
+ // reply is kept on the packet: notify_account_create with the required
+ // fields AccountActivityTime, ProjectID, ResourceList, UserRemoteSiteLogin.
+ if len(amie.Replies) != 0 {
+ t.Fatalf("expected no reply before approval, got %d", len(amie.Replies))
}
- reply := amie.lastReplyBody()
- if reply == nil {
- t.Fatal("reply body missing")
+ replyType, reply := heldReply(t, pkt)
+ if replyType != "notify_account_create" {
+ t.Fatalf("reply type: got %q, want %q", replyType, "notify_account_create")
}
if got, _ := reply["ProjectID"].(string); got != projectID {
t.Errorf("reply.ProjectID: got %q, want %q", got, projectID)
@@ -317,7 +318,7 @@ func TestRequestAccountCreate_IdempotentReDelivery(t *testing.T) {
model.AuditCreatePerson,
model.AuditCreateAccount,
model.AuditCreateMembership,
- model.AuditReplySent,
+ model.AuditReplyHeld,
} {
if got := countAuditActions(t, database, firstPkt.ID, action); got != 1 {
t.Errorf("first delivery audit %s: got %d, want 1", action, got)
@@ -327,8 +328,40 @@ func TestRequestAccountCreate_IdempotentReDelivery(t *testing.T) {
}
}
- if len(amie.Replies) != 2 {
- t.Errorf("amie replies after two deliveries: got %d, want 2", len(amie.Replies))
+ if len(amie.Replies) != 0 {
+ t.Errorf("amie replies before approval: got %d, want 0", len(amie.Replies))
+ }
+}
+
+// Make sure a person who already has an approved cluster account gets the
+// reply at once, since a new project for them needs no new approval.
+func TestRequestAccountCreate_ApprovedAccountRepliesAtOnce(t *testing.T) {
+ database := setupTestDB(t)
+ projectID := seedProjectForRAC(t, database)
+ body := baseRACBody()
+ body["ProjectID"] = projectID
+
+ svc := newTestCoreService(database)
+ amie := &fakeAmieClient{}
+ h := NewRequestAccountCreateHandler(svc, testClusterID, amie, newTestAuditService(database))
+ handle := func(pkt *model.Packet) {
+ if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
+ return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+ }
+ handle(insertPacket(t, database, "request_account_create", body))
+ approvePendingAccounts(t, database, svc)
+
+ pkt := insertPacket(t, database, "request_account_create", body)
+ handle(pkt)
+
+ if got := amie.lastReplyType(); got != "notify_account_create" {
+ t.Fatalf("reply type: got %q, want notify_account_create", got)
+ }
+ if pkt.Status == model.PacketStatusWaitingApproval || pkt.HeldReply != nil {
+ t.Errorf("expected the reply to be sent, not held, got status %s", pkt.Status)
}
}
@@ -458,13 +491,23 @@ func TestRequestAccountCreate_ReplyFailurePropagates(t *testing.T) {
body := baseRACBody()
body["ProjectID"] = projectID
- pkt := insertPacket(t, database, "request_account_create", body)
svc := newTestCoreService(database)
audit := newTestAuditService(database)
- amie := &fakeAmieClient{FailWith: errors.New("simulated AMIE outage")}
+ amie := &fakeAmieClient{}
h := NewRequestAccountCreateHandler(svc, testClusterID, amie, audit)
+ // Only an approved cluster account is replied to at once, so create and approve it first.
+ first := insertPacket(t, database, "request_account_create", body)
+ if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
+ return h.Handle(ctx, tx, map[string]any{"type": first.Type, "body": body}, first, "")
+ }); err != nil {
+ t.Fatalf("first Handle: %v", err)
+ }
+ approvePendingAccounts(t, database, svc)
+
+ amie.FailWith = errors.New("simulated AMIE outage")
+ pkt := insertPacket(t, database, "request_account_create", body)
err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
})
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
index 06a9e9352..e50053e88 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge.go
@@ -35,12 +35,13 @@ import (
type RequestPersonMergeHandler struct {
svc *service.Service
userDNStore store.UserDNStore
+ eventStore store.EventStore
amieClient AmieClient
auditSvc AuditService
}
-func NewRequestPersonMergeHandler(svc *service.Service, userDNStore store.UserDNStore, amieClient AmieClient, auditSvc AuditService) *RequestPersonMergeHandler {
- return &RequestPersonMergeHandler{svc: svc, userDNStore: userDNStore, amieClient: amieClient, auditSvc: auditSvc}
+func NewRequestPersonMergeHandler(svc *service.Service, userDNStore store.UserDNStore, eventStore store.EventStore, amieClient AmieClient, auditSvc AuditService) *RequestPersonMergeHandler {
+ return &RequestPersonMergeHandler{svc: svc, userDNStore: userDNStore, eventStore: eventStore, amieClient: amieClient, auditSvc: auditSvc}
}
func (h *RequestPersonMergeHandler) SupportsType() string { return "request_person_merge" }
@@ -123,6 +124,25 @@ func (h *RequestPersonMergeHandler) Handle(ctx context.Context, tx *sql.Tx, pack
"retiring_id", retiring.ID,
)
+ retiringAccounts, err := h.svc.ListComputeClusterUsersByUser(ctx, retiring.ID)
+ if err != nil {
+ return fmt.Errorf("request_person_merge: list retiring cluster accounts: %w", err)
+ }
+ // The retiring person can have a request_account_create packet whose reply
+ // is held until their cluster account is approved. The merge moves that
+ // account to the survivor, or deletes it when the survivor already has one
+ // on the cluster. If it is deleted, the held reply waits on an account that
+ // no longer exists, so it would never be sent.
+ //
+ // So those packets are processed again after the merge. The merge moves the
+ // retiring person's ACCESS identity to the survivor, so the handler finds the
+ // survivor's account. It replies at once if that account is approved, or
+ // holds the reply on it until an admin reviews it.
+ for _, account := range retiringAccounts {
+ if err := h.eventStore.ReprocessPacketsHeldFor(ctx, tx, account.ID); err != nil {
+ return fmt.Errorf("request_person_merge: reprocess packets held for %s: %w", account.ID, err)
+ }
+ }
if _, err := h.svc.MergeUsers(ctx, survivor.ID, retiring.ID); err != nil {
return fmt.Errorf("request_person_merge: merge: %w", err)
}
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
index c3b933c1c..f991c99b7 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_person_merge_integration_test.go
@@ -188,7 +188,7 @@ func TestRequestPersonMerge_HappyPath(t *testing.T) {
audit := newTestAuditService(database)
amie := &fakeAmieClient{}
userDNStore := store.NewUserDNStore(database)
- h := NewRequestPersonMergeHandler(svc, userDNStore, amie, audit)
+ h := NewRequestPersonMergeHandler(svc, userDNStore, store.NewEventStore(database), amie, audit)
if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
@@ -331,7 +331,7 @@ func TestRequestPersonMerge_ReplaySafeAfterMerge(t *testing.T) {
audit := newTestAuditService(database)
amie := &fakeAmieClient{}
userDNStore := store.NewUserDNStore(database)
- h := NewRequestPersonMergeHandler(svc, userDNStore, amie, audit)
+ h := NewRequestPersonMergeHandler(svc, userDNStore, store.NewEventStore(database), amie, audit)
if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": firstPkt.Type, "body": body}, firstPkt, "")
@@ -420,7 +420,7 @@ func TestRequestPersonMerge_SameKeepAndDelete(t *testing.T) {
audit := newTestAuditService(database)
amie := &fakeAmieClient{}
userDNStore := store.NewUserDNStore(database)
- h := NewRequestPersonMergeHandler(svc, userDNStore, amie, audit)
+ h := NewRequestPersonMergeHandler(svc, userDNStore, store.NewEventStore(database), amie, audit)
err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
@@ -476,7 +476,7 @@ func TestRequestPersonMerge_UnknownRetiringOrSurviving(t *testing.T) {
audit := newTestAuditService(database)
amie := &fakeAmieClient{}
userDNStore := store.NewUserDNStore(database)
- h := NewRequestPersonMergeHandler(svc, userDNStore, amie, audit)
+ h := NewRequestPersonMergeHandler(svc, userDNStore, store.NewEventStore(database), amie, audit)
err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
@@ -529,7 +529,7 @@ func TestRequestPersonMerge_ReplyFailurePropagates(t *testing.T) {
audit := newTestAuditService(database)
amie := &fakeAmieClient{FailWith: errors.New("simulated AMIE outage")}
userDNStore := store.NewUserDNStore(database)
- h := NewRequestPersonMergeHandler(svc, userDNStore, amie, audit)
+ h := NewRequestPersonMergeHandler(svc, userDNStore, store.NewEventStore(database), amie, audit)
err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
@@ -549,5 +549,45 @@ func TestRequestPersonMerge_ReplyFailurePropagates(t *testing.T) {
}
}
+// Make sure a packet held on the retiring person's cluster account is marked
+// to be processed again, since the merge deletes that account when the
+// survivor already has one on the same cluster.
+func TestRequestPersonMerge_ReprocessesPacketsHeldOnRetiringAccount(t *testing.T) {
+ database := setupTestDB(t)
+ seed := seedMergeUsers(t, database)
+
+ var retiringAccount string
+ if err := database.Get(&retiringAccount, "SELECT id FROM compute_cluster_users WHERE user_id = $1", seed.retiringID); err != nil {
+ t.Fatalf("read retiring cluster account: %v", err)
+ }
+ held := insertPacket(t, database, "request_account_create", baseRACBody())
+ if _, err := database.Exec("UPDATE amie_packets SET status = $1, held_reply = '{}', held_for = $2 WHERE id = $3",
+ model.PacketStatusWaitingApproval, retiringAccount, held.ID); err != nil {
+ t.Fatalf("hold packet: %v", err)
+ }
+ eventID := uuid.NewString()
+ if _, err := database.Exec("INSERT INTO amie_processing_events (id, packet_id, type, status) VALUES ($1, $2, $3, $4)",
+ eventID, held.ID, held.Type, model.ProcessingStatusSucceeded); err != nil {
+ t.Fatalf("insert processing event: %v", err)
+ }
+
+ body := baseRPMBody(seed)
+ pkt := insertPacket(t, database, "request_person_merge", body)
+ h := NewRequestPersonMergeHandler(newTestCoreService(database), store.NewUserDNStore(database), store.NewEventStore(database), &fakeAmieClient{}, newTestAuditService(database))
+ if err := runHandlerInTx(t, database, func(ctx context.Context, tx *sql.Tx) error {
+ return h.Handle(ctx, tx, map[string]any{"type": pkt.Type, "body": body}, pkt, "")
+ }); err != nil {
+ t.Fatalf("Handle: %v", err)
+ }
+
+ var status model.ProcessingStatus
+ if err := database.Get(&status, "SELECT status FROM amie_processing_events WHERE id = $1", eventID); err != nil {
+ t.Fatalf("read processing event: %v", err)
+ }
+ if status != model.ProcessingStatusNew {
+ t.Errorf("held packet's processing event: got %s, want %s", status, model.ProcessingStatusNew)
+ }
+}
+
// silence unused-import false positives when only some tests reference these.
var _ = model.AuditMergePersons
diff --git a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
index 37edce93c..a74375c59 100644
--- a/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
+++ b/connectors/ACCESS/AMIE-Processor/handler/request_project_create.go
@@ -139,6 +139,10 @@ func (h *RequestProjectCreateHandler) Handle(ctx context.Context, tx *sql.Tx, pa
"PiRemoteSiteLogin": piClusterUser.LocalUsername,
"ResourceList": getResourceList(body),
}
+
+ // NOTE - Not held for the PI's cluster account approval. This reply gives ACCESS the
+ // project ID, and holding it would block the whole project there. ACCESS gets
+ // the PI's login early, but the account is only created after admin approval.
reply := map[string]any{"type": "notify_project_create", "body": replyBody}
if err := h.amieClient.ReplyToPacket(ctx, packet.AmieID, reply); err != nil {
return fmt.Errorf("request_project_create: sending reply: %w", err)
diff --git a/connectors/ACCESS/AMIE-Processor/model/audit.go b/connectors/ACCESS/AMIE-Processor/model/audit.go
index 778338aa9..8b6d54182 100644
--- a/connectors/ACCESS/AMIE-Processor/model/audit.go
+++ b/connectors/ACCESS/AMIE-Processor/model/audit.go
@@ -37,6 +37,7 @@ const (
AuditReactivateMembership AuditAction = "REACTIVATE_MEMBERSHIP"
AuditPersistDNs AuditAction = "PERSIST_DNS"
AuditReplySent AuditAction = "REPLY_SENT"
+ AuditReplyHeld AuditAction = "REPLY_HELD"
AuditTransactionComplete AuditAction = "TRANSACTION_COMPLETE"
)
diff --git a/connectors/ACCESS/AMIE-Processor/model/packet.go b/connectors/ACCESS/AMIE-Processor/model/packet.go
index 3276d1994..83728e4fd 100644
--- a/connectors/ACCESS/AMIE-Processor/model/packet.go
+++ b/connectors/ACCESS/AMIE-Processor/model/packet.go
@@ -27,6 +27,12 @@ const (
PacketStatusDecoded PacketStatus = "DECODED"
PacketStatusProcessed PacketStatus = "PROCESSED"
PacketStatusFailed PacketStatus = "FAILED"
+ // PacketStatusWaitingApproval means the reply is held until an approval on
+ // Custos side, for e.g., a cluster account approval.
+ PacketStatusWaitingApproval PacketStatus = "WAITING_APPROVAL"
+ // PacketStatusRefused means Custos declined the request and ACCESS was sent a
+ // failure, for e.g., when a cluster account is denied.
+ PacketStatusRefused PacketStatus = "REFUSED"
)
// Packet stores a raw AMIE packet and its processing state.
@@ -41,4 +47,7 @@ type Packet struct {
ProcessedAt *time.Time `db:"processed_at" json:"processed_at,omitempty"`
Retries int `db:"retries" json:"retries"`
LastError *string `db:"last_error" json:"last_error,omitempty"`
+ HeldReply *string `db:"held_reply" json:"held_reply,omitempty"`
+ // HeldFor names what HeldReply waits on, for example, a compute cluster user.
+ HeldFor *string `db:"held_for" json:"held_for,omitempty"`
}
diff --git a/connectors/ACCESS/AMIE-Processor/pipeline/baseline_integration_test.go b/connectors/ACCESS/AMIE-Processor/pipeline/baseline_integration_test.go
index e78ddee6e..eeb5ebe4d 100644
--- a/connectors/ACCESS/AMIE-Processor/pipeline/baseline_integration_test.go
+++ b/connectors/ACCESS/AMIE-Processor/pipeline/baseline_integration_test.go
@@ -32,6 +32,7 @@ func TestPipeline_BaselineDeterminism(t *testing.T) {
defer pipe.stop()
pipe.fireScenario(t, "baseline")
+ pipe.startApproving()
decoded := pipe.waitForDrain(t, 12, 90*time.Second)
// Counts match baseline.yaml.
@@ -48,7 +49,7 @@ func TestPipeline_BaselineDeterminism(t *testing.T) {
{"compute_allocations", 2},
{"compute_allocation_diffs", 1},
{"amie_user_dns", 2},
- {"amie_audit_extras", 54},
+ {"amie_audit_extras", 57},
{"compute_cluster_users", 4},
{"compute_allocation_memberships", 5},
{"project_memberships", 4},
@@ -71,8 +72,8 @@ func TestPipeline_BaselineDeterminism(t *testing.T) {
); err != nil {
t.Fatalf("count amie audit_events: %v", err)
}
- if amieAuditEvents != 54 {
- t.Errorf("audit_events source='amie': got %d, want 54", amieAuditEvents)
+ if amieAuditEvents != 57 {
+ t.Errorf("audit_events source='amie': got %d, want 57", amieAuditEvents)
}
// audit_log.by_action.
@@ -85,6 +86,7 @@ func TestPipeline_BaselineDeterminism(t *testing.T) {
{"CREATE_ACCOUNT", 6},
{"CREATE_PROJECT", 3},
{"CREATE_ALLOCATION", 3},
+ {"REPLY_HELD", 3},
{"CREATE_MEMBERSHIP", 5},
{"REPLY_SENT", 11},
{"PERSIST_DNS", 3},
diff --git a/connectors/ACCESS/AMIE-Processor/pipeline/integration_common.go b/connectors/ACCESS/AMIE-Processor/pipeline/integration_common.go
index b2e9d5f54..84699d604 100644
--- a/connectors/ACCESS/AMIE-Processor/pipeline/integration_common.go
+++ b/connectors/ACCESS/AMIE-Processor/pipeline/integration_common.go
@@ -174,6 +174,8 @@ func seedCluster(t *testing.T, database *sqlx.DB) {
// server.
type testPipeline struct {
db *sqlx.DB
+ svc *coreservice.Service
+ ctx context.Context
baseURL string
siteCode string
cancel context.CancelFunc
@@ -218,7 +220,7 @@ func newTestPipeline(t *testing.T) *testPipeline {
handler.NewRequestProjectReactivateHandler(coreSvc, amieClient, auditSvc),
handler.NewRequestAccountInactivateHandler(coreSvc, amieClient, auditSvc),
handler.NewRequestAccountReactivateHandler(coreSvc, amieClient, auditSvc),
- handler.NewRequestPersonMergeHandler(coreSvc, userDNStore, amieClient, auditSvc),
+ handler.NewRequestPersonMergeHandler(coreSvc, userDNStore, eventStore, amieClient, auditSvc),
handler.NewRequestUserModifyHandler(coreSvc, userDNStore, amieClient, auditSvc),
handler.NewDataProjectCreateHandler(coreSvc, userDNStore, amieClient, auditSvc),
handler.NewDataAccountCreateHandler(coreSvc, userDNStore, amieClient, auditSvc),
@@ -226,6 +228,8 @@ func newTestPipeline(t *testing.T) *testPipeline {
handler.NewNoOpHandler(),
)
+ handler.NewHeldReplies(database, packetStore, amieClient, auditSvc).Subscribe(eventBus, "amie-processor")
+
met := metrics.NewWithRegistry(prometheus.NewRegistry())
poller := worker.NewPoller(amieClient, packetStore, eventStore, met, database, cfg)
processor := worker.NewProcessor(eventStore, packetStore, errorStore, router, met, auditSvc, database, cfg)
@@ -233,11 +237,13 @@ func newTestPipeline(t *testing.T) *testPipeline {
ctx, cancel := context.WithCancel(context.Background())
pipe := &testPipeline{
db: database,
+ svc: coreSvc,
+ ctx: ctx,
baseURL: cfg.BaseURL,
siteCode: cfg.SiteCode,
cancel: cancel,
}
- pipe.wg.Add(2)
+ pipe.wg.Add(3)
go func() {
defer pipe.wg.Done()
poller.Run(ctx)
@@ -246,9 +252,40 @@ func newTestPipeline(t *testing.T) *testPipeline {
defer pipe.wg.Done()
processor.Run(ctx)
}()
+ go func() {
+ defer pipe.wg.Done()
+ eventBus.Run(ctx)
+ }()
return pipe
}
+// startApproving stands in for an admin who approves every new cluster
+// account, so held replies are released, and the scenario can finish.
+func (p *testPipeline) startApproving() {
+ p.wg.Add(1)
+ go func() {
+ defer p.wg.Done()
+ ticker := time.NewTicker(200 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-p.ctx.Done():
+ return
+ case <-ticker.C:
+ }
+ var ids []string
+ var reviewer string
+ if p.db.SelectContext(p.ctx, &ids, "SELECT id FROM compute_cluster_users WHERE approval_status = 'PENDING'") != nil ||
+ len(ids) == 0 || p.db.GetContext(p.ctx, &reviewer, "SELECT id FROM users LIMIT 1") != nil {
+ continue
+ }
+ for _, id := range ids {
+ _, _ = p.svc.ApproveComputeClusterUser(p.ctx, id, reviewer)
+ }
+ }
+ }()
+}
+
func (p *testPipeline) stop() {
p.cancel()
p.wg.Wait()
@@ -288,7 +325,7 @@ func (p *testPipeline) waitForDrain(t *testing.T, expectPackets int, deadline ti
t.Fatalf("count packets: %v", err)
}
if err := p.db.Get(&pendingDecode,
- "SELECT COUNT(*) FROM amie_packets WHERE status = 'NEW'",
+ "SELECT COUNT(*) FROM amie_packets WHERE status IN ('NEW', 'WAITING_APPROVAL')",
); err != nil {
t.Fatalf("count pending packets: %v", err)
}
diff --git a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
index 296c93d8c..338310eeb 100644
--- a/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
+++ b/connectors/ACCESS/AMIE-Processor/pkg/amie/loader.go
@@ -109,7 +109,7 @@ func LoadConnector(ctx context.Context, database *sqlx.DB, eventBus *events.Bus,
handler.NewRequestProjectReactivateHandler(coreService, amie, auditSvc),
handler.NewRequestAccountInactivateHandler(coreService, amie, auditSvc),
handler.NewRequestAccountReactivateHandler(coreService, amie, auditSvc),
- handler.NewRequestPersonMergeHandler(coreService, userDNStore, amie, auditSvc),
+ handler.NewRequestPersonMergeHandler(coreService, userDNStore, eventStore, amie, auditSvc),
handler.NewRequestUserModifyHandler(coreService, userDNStore, amie, auditSvc),
handler.NewDataProjectCreateHandler(coreService, userDNStore, amie, auditSvc),
handler.NewDataAccountCreateHandler(coreService, userDNStore, amie, auditSvc),
@@ -117,6 +117,8 @@ func LoadConnector(ctx context.Context, database *sqlx.DB, eventBus *events.Bus,
handler.NewNoOpHandler(),
)
+ handler.NewHeldReplies(database, packetStore, amie, auditSvc).Subscribe(eventBus, Type)
+
met := metrics.New()
poller := worker.NewPoller(amie, packetStore, eventStore, met, database, cfg)
processor := worker.NewProcessor(eventStore, packetStore, errorStore, dispatcher, met, auditSvc, database, cfg)
diff --git a/connectors/ACCESS/AMIE-Processor/store/event_store.go b/connectors/ACCESS/AMIE-Processor/store/event_store.go
index df7b08420..f6eb07648 100644
--- a/connectors/ACCESS/AMIE-Processor/store/event_store.go
+++ b/connectors/ACCESS/AMIE-Processor/store/event_store.go
@@ -32,6 +32,7 @@ type EventStore interface {
FindTop50EventsToProcess(ctx context.Context, statuses []model.ProcessingStatus, now time.Time) ([]model.EventWithPacket, error)
Save(ctx context.Context, tx *sql.Tx, e *model.ProcessingEvent) error
Update(ctx context.Context, tx *sql.Tx, e *model.ProcessingEvent) error
+ ReprocessPacketsHeldFor(ctx context.Context, tx *sql.Tx, heldFor string) error
}
type pgEventStore struct {
@@ -101,3 +102,14 @@ func (s *pgEventStore) Update(ctx context.Context, tx *sql.Tx, e *model.Processi
e.LastError, e.NextRetryAt, e.ID)
return err
}
+
+// ReprocessPacketsHeldFor marks the packets whose reply waits on heldFor to be
+// processed again, so their handler runs with the current state. For example, a
+// person merge deletes the cluster account those packets were held on.
+func (s *pgEventStore) ReprocessPacketsHeldFor(ctx context.Context, tx *sql.Tx, heldFor string) error {
+ _, err := tx.ExecContext(ctx,
+ `UPDATE amie_processing_events SET status = $1, next_retry_at = NULL
+ WHERE packet_id IN (SELECT id FROM amie_packets WHERE held_for = $2 AND status = $3)`,
+ model.ProcessingStatusNew, heldFor, model.PacketStatusWaitingApproval)
+ return err
+}
diff --git a/connectors/ACCESS/AMIE-Processor/store/packet_store.go b/connectors/ACCESS/AMIE-Processor/store/packet_store.go
index 06df401af..de579b64b 100644
--- a/connectors/ACCESS/AMIE-Processor/store/packet_store.go
+++ b/connectors/ACCESS/AMIE-Processor/store/packet_store.go
@@ -56,6 +56,7 @@ type PacketStore interface {
FindByID(ctx context.Context, id string) (*model.Packet, error)
Save(ctx context.Context, tx *sql.Tx, p *model.Packet) error
Update(ctx context.Context, tx *sql.Tx, p *model.Packet) error
+ ListWaitingFor(ctx context.Context, clusterUserID string) ([]model.Packet, error)
ListPackets(ctx context.Context, f PacketListFilter) ([]model.Packet, int, error)
ListPacketEvents(ctx context.Context, packetID string) ([]model.ProcessingEvent, error)
@@ -70,7 +71,7 @@ func NewPacketStore(db *sqlx.DB) PacketStore {
return &pgPacketStore{db: db}
}
-const packetColumns = `id, amie_id, type, status, raw_json, received_at, decoded_at, processed_at, retries, last_error`
+const packetColumns = `id, amie_id, type, status, raw_json, received_at, decoded_at, processed_at, retries, last_error, held_reply, held_for`
func (s *pgPacketStore) FindByAmieID(ctx context.Context, amieID int64) (*model.Packet, error) {
var p model.Packet
@@ -110,13 +111,22 @@ func (s *pgPacketStore) Save(ctx context.Context, tx *sql.Tx, p *model.Packet) e
func (s *pgPacketStore) Update(ctx context.Context, tx *sql.Tx, p *model.Packet) error {
_, err := tx.ExecContext(ctx,
- `UPDATE amie_packets SET type = $1, status = $2, raw_json = $3, decoded_at = $4, processed_at = $5, retries = $6, last_error = $7
- WHERE id = $8`,
+ `UPDATE amie_packets SET type = $1, status = $2, raw_json = $3, decoded_at = $4, processed_at = $5, retries = $6, last_error = $7,
+ held_reply = $8, held_for = $9
+ WHERE id = $10`,
p.Type, p.Status, p.RawJSON, p.DecodedAt, p.ProcessedAt,
- p.Retries, p.LastError, p.ID)
+ p.Retries, p.LastError, p.HeldReply, p.HeldFor, p.ID)
return err
}
+func (s *pgPacketStore) ListWaitingFor(ctx context.Context, clusterUserID string) ([]model.Packet, error) {
+ var packets []model.Packet
+ err := s.db.SelectContext(ctx, &packets,
+ `SELECT `+packetColumns+` FROM amie_packets WHERE held_for = $1 AND status = $2 ORDER BY received_at`,
+ clusterUserID, model.PacketStatusWaitingApproval)
+ return packets, err
+}
+
func (s *pgPacketStore) ListPackets(ctx context.Context, f PacketListFilter) ([]model.Packet, int, error) {
where, args := buildPacketFilter(f)
diff --git a/connectors/ACCESS/AMIE-Processor/testdata/scenarios/baseline.yaml b/connectors/ACCESS/AMIE-Processor/testdata/scenarios/baseline.yaml
index abee44cda..04ff9767c 100644
--- a/connectors/ACCESS/AMIE-Processor/testdata/scenarios/baseline.yaml
+++ b/connectors/ACCESS/AMIE-Processor/testdata/scenarios/baseline.yaml
@@ -169,7 +169,7 @@ expectations:
audit_log:
# audit_events (source='amie') and amie_audit_extras carry the same count;
# one row each per AMIE audit.
- total_count: 54
+ total_count: 57
by_action:
PACKET_RECEIVED: 12 # one per packet
CREATE_PERSON: 6 # 3 project_create + 3 account_create
@@ -178,7 +178,8 @@ expectations:
CREATE_ALLOCATION: 3 # supplement-delivery still audits CREATE_ALLOCATION
ATTACH_RESOURCE: 2 # one per allocation; supplement attach is a no-op
CREATE_MEMBERSHIP: 5 # 2 PIs + 3 account_create
- REPLY_SENT: 11 # everything except inform_transaction_complete
+ REPLY_HELD: 3 # each account_create waits for its cluster account approval
+ REPLY_SENT: 11 # everything except inform_transaction_complete, each once
PERSIST_DNS: 3 # 2 from data_*, 1 delete from request_user_modify
UPDATE_PERSON: 1
MERGE_PERSONS: 1
diff --git a/connectors/ACCESS/AMIE-Processor/worker/processor.go b/connectors/ACCESS/AMIE-Processor/worker/processor.go
index f1126951d..691c3f7ea 100644
--- a/connectors/ACCESS/AMIE-Processor/worker/processor.go
+++ b/connectors/ACCESS/AMIE-Processor/worker/processor.go
@@ -208,12 +208,16 @@ func (p *Processor) executeInTransaction(ctx context.Context, ewp model.EventWit
return fmt.Errorf("unmarshal packet raw JSON: %w", err)
}
- // Load the full packet from DB to preserve all fields (e.g. retries).
+ // Load the full packet from DB to preserve all fields (e.g., retries).
packet, err := p.packetStore.FindByID(ctx, ewp.PacketID)
if err != nil {
return fmt.Errorf("load packet %s: %w", ewp.PacketID, err)
}
+ // A handler that holds its reply sets the status to waiting.
+ decodedAt := time.Now().UTC()
+ packet.Status = model.PacketStatusDecoded
+ packet.DecodedAt = &decodedAt
if err := p.router.Route(ctx, tx, packetJSON, packet, ewp.ID); err != nil {
return fmt.Errorf("route packet: %w", err)
}
@@ -226,11 +230,8 @@ func (p *Processor) executeInTransaction(ctx context.Context, ewp model.EventWit
return fmt.Errorf("update event to SUCCEEDED: %w", err)
}
- decodedAt := time.Now().UTC()
- packet.Status = model.PacketStatusDecoded
- packet.DecodedAt = &decodedAt
if err := p.packetStore.Update(ctx, tx, packet); err != nil {
- return fmt.Errorf("update packet to DECODED: %w", err)
+ return fmt.Errorf("update packet: %w", err)
}
p.metrics.RecordPacketProcessed(ewp.PacketType, "succeeded")