[geode] branch develop updated: Fixes for cleaning GCP resources from pipelines (#7759)

[rh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rhoughton pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git


The following commit(s) were added to refs/heads/develop by this push:
     new 1e2e34781a Fixes for cleaning GCP resources from pipelines (#7759)
1e2e34781a is described below

commit 1e2e34781a3abe004aa0ca5b712df857c158d77e
Author: Robert Houghton <rh...@pivotal.io>
AuthorDate: Thu Jun 16 15:49:24 2022 -0700

    Fixes for cleaning GCP resources from pipelines (#7759)
    
    * Fixes for cleaning GCP resources from pipelines
    
    * Use meta.properties to feed pipeline cleanup script
    
    * Add comments on use case for this script.
---
 ci/pipelines/clean_fork_pipeline.sh  | 110 +++++++++++++++++++++++++++++++++++
 ci/pipelines/clean_fork_pipelines.sh |  76 ------------------------
 2 files changed, 110 insertions(+), 76 deletions(-)

diff --git a/ci/pipelines/clean_fork_pipeline.sh b/ci/pipelines/clean_fork_pipeline.sh
new file mode 100755
index 0000000000..5ca11ceadb
--- /dev/null
+++ b/ci/pipelines/clean_fork_pipeline.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## This script deletes a given Concoure pipeline, and deletes the associated Google Compute Images and Google Container
+## Images. Tokens like fork-name and pipeline-name are read from ./ci/pipelines/meta/meta.properties(.local). Branch-
+## name may be passed in on the CLI, or else the current branch is assumed.
+##
+## usage: ./clean_fork_pipeline.sh [branch-name]
+##
+
+set -eu
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+META_PROPERTIES=${SCRIPTDIR}/meta/meta.properties
+LOCAL_META_PROPERTIES=${SCRIPTDIR}/meta/meta.properties.local
+
+## Load default properties
+echo "**************************************************"
+echo "Default Environment variables for this deployment:"
+cat ${META_PROPERTIES} | grep -v "^#"
+source ${META_PROPERTIES}
+
+## Load local overrides properties file
+if [[ -f ${LOCAL_META_PROPERTIES} ]]; then
+  echo "Local Environment overrides for this deployment:"
+  cat ${LOCAL_META_PROPERTIES} | grep -v "^#"
+  source ${LOCAL_META_PROPERTIES}
+  echo "**************************************************"
+fi
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+GEODE_BRANCH=${1:-${CURRENT_BRANCH}}
+
+. ${SCRIPTDIR}/shared/utilities.sh
+SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
+SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})
+
+if [[ -z "${GEODE_FORK}" ]]; then
+  echo "No fork provided!"
+  exit 1
+fi
+
+if [[ "${GEODE_FORK}" == "apache" ]]; then
+  echo "This utility is not for primary (Apache) pipelines."
+  exit 1
+fi
+
+TARGET=${CONCOURSE_HOST}-${CONCOURSE_TEAM}
+CONCOURSE_URL="https://${CONCOURSE_HOST}"
+CONCOURSE_TEAM="${CONCOURSE_TEAM:-main}"
+
+fly -t "${TARGET}" status || \
+fly -t "${TARGET}" login \
+  --team-name "${CONCOURSE_TEAM}" \
+  --concourse-url="${CONCOURSE_URL}"
+
+echo "Fork is ${GEODE_FORK}"
+echo "Branch is ${GEODE_BRANCH}"
+echo "GCP Project is ${GCP_PROJECT}"
+
+echo "Deleting meta pipeline if it exists..."
+META_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-meta"
+fly -t "${TARGET}" destroy-pipeline --non-interactive -p "${META_PIPELINE}"
+
+echo "Deleting images pipeline if it exists..."
+IMAGES_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-images"
+fly -t "${TARGET}" destroy-pipeline --non-interactive -p "${IMAGES_PIPELINE}"
+
+echo "Deleting reaper pipeline if it exists..."
+REAPER_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-reaper"
+fly -t "${TARGET}" destroy-pipeline --non-interactive -p "${REAPER_PIPELINE}"
+
+echo "Deleting build pipeline if it exists..."
+BUILD_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-main"
+fly -t "${TARGET}" destroy-pipeline --non-interactive -p "${BUILD_PIPELINE}"
+
+gcloud --project="${GCP_PROJECT}" container images list | grep "${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}" | while IFS= read -r line; do
+  echo "Deleting image: ${line}"
+  gcloud --project="${GCP_PROJECT}" container images delete ${line}:latest --quiet
+  gcloud --project="${GCP_PROJECT}" container images list-tags ${line} --filter='-tags:*'  --format='get(digest)' | while IFS= read -r line2; do
+  echo "Deleting image: ${line2}"
+  gcloud --project="${GCP_PROJECT}" container images delete ${line}@${line2} --quiet
+  done
+done
+set -x
+gcloud --project="${GCP_PROJECT}" compute images list --filter="family ~ ${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}" --format='value(name)' | while IFS= read -r line; do
+  echo "Deleting image: ${line}"
+  gcloud --project="${GCP_PROJECT}" compute images delete ${line} --quiet
+done
+
diff --git a/ci/pipelines/clean_fork_pipelines.sh b/ci/pipelines/clean_fork_pipelines.sh
deleted file mode 100755
index c02ad57dc9..0000000000
--- a/ci/pipelines/clean_fork_pipelines.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-TARGET=geode
-GEODE_FORK=${1}
-CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-GEODE_BRANCH=${2:-${CURRENT_BRANCH}}
-
-. ${SCRIPTDIR}/shared/utilities.sh
-SANITIZED_GEODE_BRANCH=$(getSanitizedBranch ${GEODE_BRANCH})
-SANITIZED_GEODE_FORK=$(getSanitizedFork ${GEODE_FORK})
-
-if [[ -z "${GEODE_FORK}" ]]; then
-  echo "No fork provided!"
-  exit 1
-fi
-
-if [[ "${GEODE_FORK}" == "apache" ]]; then
-  echo "This utility is not for primary pipelines."
-  exit 1
-fi
-
-echo "Fork is ${GEODE_FORK}"
-echo "Branch is ${GEODE_BRANCH}"
-
-echo "Deleting meta pipeline if it exists..."
-META_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-meta"
-fly -t ${TARGET} destroy-pipeline --non-interactive -p ${META_PIPELINE}
-
-echo "Deleting images pipeline if it exists..."
-IMAGES_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-images"
-fly -t ${TARGET} destroy-pipeline --non-interactive -p ${IMAGES_PIPELINE}
-
-echo "Deleting reaper pipeline if it exists..."
-REAPER_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-reaper"
-fly -t ${TARGET} destroy-pipeline --non-interactive -p ${REAPER_PIPELINE}
-
-echo "Deleting build pipeline if it exists..."
-BUILD_PIPELINE="${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-main"
-fly -t ${TARGET} destroy-pipeline --non-interactive -p ${BUILD_PIPELINE}
-
-gcloud container images list | grep "${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}" | while IFS= read -r line; do
-  echo "Deleting image: ${line}"
-  gcloud container images delete ${line}:latest --quiet
-  gcloud container images list-tags ${line} --filter='-tags:*'  --format='get(digest)' | while IFS= read -r line2; do
-    echo "Deleting image: ${line2}"
-    gcloud container images delete ${line}@${line2} --quiet
-  done
-done
-
-gcloud compute images list | awk "/^${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}/ {print \$1}" | while IFS= read -r line; do
-  echo "Deleting image: ${line}"
-  gcloud compute images delete ${line} --quiet
-done
-