(airavata-custos) 19/29: Delete Slurm associations correctly and add allocation teardown

[la...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lahirujayathilake pushed a commit to branch pearc26
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git

commit d602979da75bf4a0064d8856f704a0729d978909
Author: lahiruj <la...@gmail.com>
AuthorDate: Thu Jul 16 13:26:35 2026 -0400

    Delete Slurm associations correctly and add allocation teardown
---
 dev-ops/compose/seeds/pearc26/teardown-live.sh | 85 ++++++++++++++++++++++++++
 scripts/reset-trial-test-user.sh               | 10 ++-
 2 files changed, 92 insertions(+), 3 deletions(-)

diff --git a/dev-ops/compose/seeds/pearc26/teardown-live.sh b/dev-ops/compose/seeds/pearc26/teardown-live.sh
new file mode 100755
index 000000000..97f459c2c
--- /dev/null
+++ b/dev-ops/compose/seeds/pearc26/teardown-live.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Tear down the PEARC26 tutorial allocation so setup-live.sh can recreate it
+# from scratch: removes the Slurm account (and every association under it) and
+# the custos project/allocation/usage/event rows. Leaves the org, cluster,
+# partition catalog, rate, and approver role (seeds 01-03) in place.
+#
+# Usage: dev-ops/compose/seeds/pearc26/teardown-live.sh [--yes]
+
+set -euo pipefail
+cd "$(dirname "$0")/../../../.."
+
+ACCOUNT="pearc26-tutorial"
+CONFIRM="${1:-}"
+
+set -a; source .env; set +a
+
+# DB credentials from DATABASE_DSN when present (deployment hosts differ
+# from the local compose defaults).
+DB_USER=admin; DB_PASS=admin
+if grep -q "^DATABASE_DSN=" .env; then
+  _dsn=$(grep "^DATABASE_DSN=" .env | cut -d= -f2- | tr -d '"' | tr -d "'")
+  DB_USER=$(echo "$_dsn" | cut -d: -f1)
+  DB_PASS=$(echo "$_dsn" | sed -E 's/^[^:]*:([^@]*)@.*/\1/')
+fi
+DB_EXEC=(docker exec -i custos_db mariadb -u"$DB_USER" -p"$DB_PASS" custos)
+
+echo "== Current state"
+alloc_ids=$("${DB_EXEC[@]}" -N -e "SELECT id FROM compute_allocations WHERE name='$ACCOUNT'" 2>/dev/null || true)
+echo "  custos allocations named $ACCOUNT: ${alloc_ids:-none}"
+if [ -n "${SLURM_API_URL:-}" ] && [ -n "${SLURM_TOKEN:-}" ]; then
+  ver="v0.0.${SLURM_API_VERSION:-40}"
+  n=$(curl -s -H "X-SLURM-USER-NAME: ${SLURM_API_USERNAME:-nexus-provisioner}" -H "X-SLURM-USER-TOKEN: $SLURM_TOKEN" \
+    "$SLURM_API_URL/slurmdb/$ver/associations?account=$ACCOUNT" \
+    | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('associations',[])))" 2>/dev/null || echo "?")
+  echo "  slurm associations under $ACCOUNT: $n"
+else
+  echo "  SLURM_API_URL/SLURM_TOKEN not set; Slurm teardown will be skipped"
+fi
+
+if [ -z "$alloc_ids" ] && { [ -z "${SLURM_API_URL:-}" ] || [ "${n:-0}" = "0" ]; }; then
+  echo "Nothing to tear down."; exit 0
+fi
+
+if [ "$CONFIRM" != "--yes" ]; then
+  read -r -p "Tear all of this down? [y/N] " ans
+  [ "$ans" = "y" ] || { echo "Aborted."; exit 1; }
+fi
+
+if [ -n "${SLURM_API_URL:-}" ] && [ -n "${SLURM_TOKEN:-}" ]; then
+  echo "== Removing the Slurm account (cascades its associations)"
+  removed=$(curl -s -X DELETE \
+    -H "X-SLURM-USER-NAME: ${SLURM_API_USERNAME:-nexus-provisioner}" -H "X-SLURM-USER-TOKEN: $SLURM_TOKEN" \
+    "$SLURM_API_URL/slurmdb/$ver/account/$ACCOUNT" \
+    | python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d.get('removed_associations',[])), '/', (d.get('errors') or 'ok'))" 2>/dev/null || echo "delete failed")
+  echo "  removed associations / errors: $removed"
+fi
+
+echo "== Cleaning custos allocation rows"
+for aid in $alloc_ids; do
+  "${DB_EXEC[@]}" <<SQL
+DELETE FROM compute_allocation_usages WHERE compute_allocation_id='$aid';
+DELETE FROM compute_allocation_membership_resource_overrides WHERE compute_allocation_membership_id IN
+  (SELECT id FROM compute_allocation_memberships WHERE compute_allocation_id='$aid');
+DELETE FROM compute_allocation_memberships WHERE compute_allocation_id='$aid';
+DELETE FROM compute_allocation_resource_mappings WHERE compute_allocation_id='$aid';
+DELETE FROM access_events WHERE compute_allocation_id='$aid';
+DELETE FROM compute_allocations WHERE id='$aid';
+SQL
+  echo "  allocation $aid removed"
+done
+
+# Drop the tutorial project only when nothing else hangs off it.
+"${DB_EXEC[@]}" <<SQL 2>/dev/null || true
+DELETE pm FROM project_memberships pm
+  JOIN projects p ON p.id = pm.project_id
+ WHERE p.originated_id='PEARC26'
+   AND NOT EXISTS (SELECT 1 FROM compute_allocations a WHERE a.project_id = p.id);
+DELETE p FROM projects p
+ WHERE p.originated_id='PEARC26'
+   AND NOT EXISTS (SELECT 1 FROM compute_allocations a WHERE a.project_id = p.id);
+SQL
+echo "  tutorial project removed (if it had no other allocations)"
+
+echo
+echo "Done. Re-run setup-live.sh to recreate the allocation through the API."
diff --git a/scripts/reset-trial-test-user.sh b/scripts/reset-trial-test-user.sh
index 94a7abd14..616879c97 100755
--- a/scripts/reset-trial-test-user.sh
+++ b/scripts/reset-trial-test-user.sh
@@ -135,11 +135,15 @@ echo "== Slurm accounting cleanup"
 # slurmctld binds an association to the uid at load time; a recycled
 # username returns with a new uid, so stale user associations must go.
 if [ -n "${SLURM_API_URL:-}" ] && [ -n "${SLURM_TOKEN:-}" ]; then
+  ver="v0.0.${SLURM_API_VERSION:-40}"
   for u in $(printf '%s\n' $local_usernames $registry_uids | sort -u); do
-    code=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE \
+    # Delete the association (not the user record): the /user endpoint
+    # 304s without removing the association, leaving a stale uid binding.
+    removed=$(curl -s -X DELETE \
       -H "X-SLURM-USER-NAME: ${SLURM_API_USERNAME:-nexus-provisioner}" -H "X-SLURM-USER-TOKEN: $SLURM_TOKEN" \
-      "$SLURM_API_URL/slurmdb/v0.0.${SLURM_API_VERSION:-40}/user/$u")
-    echo "  slurm user $u delete -> HTTP $code (404/does-not-exist is fine)"
+      "$SLURM_API_URL/slurmdb/$ver/associations?user=$u" \
+      | python3 -c "import json,sys; print(len(json.load(sys.stdin).get('removed_associations',[])))" 2>/dev/null || echo 0)
+    echo "  slurm associations removed for $u: $removed"
   done
 else
   echo "  SLURM_API_URL/SLURM_TOKEN not set; skipped (delete stale associations manually if the username is reused)"