[airflow-ci-infra] 01/01: Runners are more resiliant to docker login failure

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch add-parallel-to-packages
in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git

commit e5bd60b4fa0bc180107990328c7d18de45a1f6ca
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 20 14:15:15 2021 +0100

    Runners are more resiliant to docker login failure
    
    Also added `set -eux -o pipefail` to be better protected against
    any silent failures.
---
 cloud-init.yml | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/cloud-init.yml b/cloud-init.yml
index a2c1e63..7a9982a 100644
--- a/cloud-init.yml
+++ b/cloud-init.yml
@@ -30,6 +30,7 @@ runcmd:
     - -c
     # https://github.com/actions/virtual-environments/blob/525f79f479cca77aef4e0a680548b65534c64a18/images/linux/scripts/installers/docker-compose.sh
     - |
+      set -exu -o pipefail
       URL=$(curl -s https://api.github.com/repos/docker/compose/releases/latest | jq -r '.assets[].browser_download_url | select(endswith("docker-compose-Linux-x86_64"))')
       curl -L $URL -o /usr/local/bin/docker-compose
       chmod +x /usr/local/bin/docker-compose
@@ -49,6 +50,7 @@ runcmd:
     - bash
     - -c
     - |
+      set -exu -o pipefail
       python3 -mvenv /opt/runner-supervisor
       /opt/runner-supervisor/bin/pip install -U pip python-dynamodb-lock-whatnick==0.9.3 click==7.1.2 psutil 'tenacity~=6.0'
   -
@@ -74,10 +76,6 @@ runcmd:
       aws s3 cp s3://airflow-ci-assets/runner-supervisor.py /opt/runner-supervisor/bin/runner-supervisor
       chmod 755 /opt/runner-supervisor/bin/runner-supervisor
 
-      # Log in to a paid docker user to get unlimited docker pulls
-      aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
-        jq .Parameter.Value -r | \
-        sudo -u runner docker login --username airflowcirunners --password-stdin
     - 2.277.1-airflow1
   - [systemctl, enable, --now, iptables.service]
   # Restart docker after applying the user firewall -- else some rules/chains might be list!
@@ -88,8 +86,8 @@ runcmd:
     - bash
     - -c
     - |
-      echo "Pre-loading commonly used docker images from S3"
       set -eux -o pipefail
+      echo "Pre-loading commonly used docker images from S3"
       aws s3 cp s3://airflow-ci-assets/pre-baked-images.tar.gz - | docker load
 
 write_files:
@@ -121,10 +119,16 @@ write_files:
   - path: /usr/local/sbin/runner-cleanup-workdir.sh
     content: |
       #!/bin/bash
+      set -exu -o pipefail
       echo "Left-over containers:"
       docker ps -a
       docker ps -qa | xargs --verbose --no-run-if-empty docker rm -fv
 
+      echo "Log in to a paid docker user to get unlimited docker pulls"
+      aws ssm get-parameter --with-decryption --name /runners/apache/airflow/dockerPassword | \
+        jq .Parameter.Value -r | \
+        sudo -u runner docker login --username airflowcirunners --password-stdin
+
       if [[ -d ~runner/actions-runner/_work/airflow/airflow ]]; then
         cd ~runner/actions-runner/_work/airflow/airflow
 
@@ -143,7 +147,7 @@ write_files:
   - path: /usr/local/bin/stop-runner-if-no-job.sh
     content: |
       #!/bin/bash
-      set -u
+      set -exu -o pipefail
 
       MAINPID="${MAINPID:-${1:-}}"
 
@@ -193,7 +197,7 @@ write_files:
     permissions: '0775'
     content: |
       #!/bin/bash
-
+      set -exu -o pipefail
       if pgrep -c Runner.Worker >/dev/null; then
           # Only report metric when we're doing something -- no point paying to submit zeros
           aws cloudwatch put-metric-data --metric-name jobs-running --value "$(pgrep -c Runner.Worker)" --namespace github.actions
@@ -212,6 +216,7 @@ write_files:
       [Service]
       Type=notify
       ExecStart=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor
+      ExecPost=/opt/runner-supervisor/bin/python /opt/runner-supervisor/bin/runner-supervisor
       # We need to run as root to have the ability to open the netlink connector socket
       User=root
       WorkingDirectory=/home/runner/actions-runner