(beam) 01/02: Fix clean up resource workflow; gix XVR Flink

[yh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch fixspannerwritetest
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 37f97f09205eb3991ea4351ad0d76890c4411aaf
Author: Yi Hu <ya...@google.com>
AuthorDate: Wed Nov 8 18:57:48 2023 -0500

    Fix clean up resource workflow; gix XVR Flink
    
    * Move spanner cleaner to correct workflow
    
    * Fix k8s naming missing cases
---
 .github/workflows/beam_PostCommit_XVR_Flink.yml  |  2 +-
 .test-infra/tools/build.gradle                   |  5 +++++
 .test-infra/tools/stale_dataflow_jobs_cleaner.sh |  7 -------
 .test-infra/tools/stale_k8s_workload_cleaner.sh  |  2 +-
 .test-infra/tools/stale_spanner_cleaner.sh       | 11 +++++++++++
 5 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/beam_PostCommit_XVR_Flink.yml b/.github/workflows/beam_PostCommit_XVR_Flink.yml
index 75d3353e8a1..00c756a714f 100644
--- a/.github/workflows/beam_PostCommit_XVR_Flink.yml
+++ b/.github/workflows/beam_PostCommit_XVR_Flink.yml
@@ -56,7 +56,7 @@ jobs:
       github.event_name == 'pull_request_target' ||
       (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
       github.event.comment.body == 'Run XVR_Flink PostCommit'
-    runs-on: [self-hosted, ubuntu-20.04, main]
+    runs-on: [self-hosted, ubuntu-20.04, highmem]
     timeout-minutes: 100
     name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }})
     strategy:
diff --git a/.test-infra/tools/build.gradle b/.test-infra/tools/build.gradle
index 048fc0a8d02..274cd302291 100644
--- a/.test-infra/tools/build.gradle
+++ b/.test-infra/tools/build.gradle
@@ -36,9 +36,14 @@ task removeStaleK8sWorkload(type: Exec) {
   commandLine './stale_k8s_workload_cleaner.sh'
 }
 
+task removeStaleSpannerResources(type: Exec) {
+  commandLine './stale_spanner_cleaner.sh'
+}
+
 task cleanupOtherStaleResources {
   // declared as finalizedBy dependency so that other task continue even if one dep task fails
   finalizedBy tasks.removeStaleBqDatasets
   finalizedBy tasks.removeStaleCbtInstances
   finalizedBy tasks.removeStaleK8sWorkload
+  finalizedBy tasks.removeStaleSpannerResources
 }
diff --git a/.test-infra/tools/stale_dataflow_jobs_cleaner.sh b/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
index e6df7058427..e5630480747 100755
--- a/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
+++ b/.test-infra/tools/stale_dataflow_jobs_cleaner.sh
@@ -27,10 +27,3 @@ if [[ ${STALE_JOBS} ]]; then
 else
   echo "No stale jobs found."
 fi
-
-# Delete spanner databases older than 1 day.
-gcloud spanner databases list \
---instance beam-test \
---filter="createTime < $(date --iso-8601=s -d '1 day ago')" \
---format="value(name)" | \
-xargs -I{} gcloud spanner databases delete {} --instance beam-test --quiet
diff --git a/.test-infra/tools/stale_k8s_workload_cleaner.sh b/.test-infra/tools/stale_k8s_workload_cleaner.sh
index f4d6f9b8832..9ddaf17f2ce 100755
--- a/.test-infra/tools/stale_k8s_workload_cleaner.sh
+++ b/.test-infra/tools/stale_k8s_workload_cleaner.sh
@@ -43,7 +43,7 @@ function should_teardown() {
 gcloud container clusters get-credentials io-datastores --zone us-central1-a --project apache-beam-testing
 
 while read NAME STATUS AGE; do
-  if [[ $NAME =~ ^beam-.+test ]] && should_teardown $AGE; then
+  if [[ $NAME =~ ^beam-.+(test|-it) ]] && should_teardown $AGE; then
     kubectl delete namespace $NAME
   fi
 done < <( kubectl get namespaces --context=gke_${PROJECT}_${LOCATION}_${CLUSTER} )
diff --git a/.test-infra/tools/stale_spanner_cleaner.sh b/.test-infra/tools/stale_spanner_cleaner.sh
new file mode 100755
index 00000000000..b4a66c8c67e
--- /dev/null
+++ b/.test-infra/tools/stale_spanner_cleaner.sh
@@ -0,0 +1,11 @@
+set -euo pipefail
+
+PROJECT=apache-beam-testing
+
+# Delete spanner databases older than 1 day.
+gcloud spanner databases list \
+--instance beam-test \
+--project $PROJECT \
+--filter="createTime < $(date --iso-8601=s -d '1 day ago')" \
+--format="value(name)" | \
+xargs -I{} gcloud spanner databases delete {} --instance beam-test --quiet