You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by rh...@apache.org on 2019/05/06 15:54:15 UTC
[geode] 01/02: GEODE-6739: Create Concourse heavy-lift workers in
the current zone
This is an automated email from the ASF dual-hosted git repository.
rhoughton pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 1e46eeda20b1a28ac85d28791108f5c7d80bbdf3
Author: Robert Houghton <rh...@pivotal.io>
AuthorDate: Fri May 3 13:02:14 2019 -0700
GEODE-6739: Create Concourse heavy-lift workers in the current zone
GCP costs are incurred on data egress between zones, even the same
region. Track our `create-instance` attempts, to put the heavy lifter in
the same zone on the first attempt, and roulette only on failure
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Helena Bales <hb...@pivotal.io>
---
ci/pipelines/geode-build/jinja.template.yml | 6 ++++++
ci/pipelines/pull-request/jinja.template.yml | 6 ++++++
ci/pipelines/shared/shared_jinja.yml | 10 ++++++++++
ci/scripts/create_instance.sh | 18 ++++++++++++++----
4 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 4ea7643..c5f16b5 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -18,6 +18,7 @@
{% from 'shared_jinja.yml' import alpine_tools_config with context %}
{% from 'shared_jinja.yml' import pipeline_prefix with context %}
{% from 'shared_jinja.yml' import github_access with context %}
+{% from 'shared_jinja.yml' import init_retry with context %}
---
@@ -224,6 +225,7 @@ jobs:
pre: ((semver-prerelease-token))
- do:
- put: concourse-metadata-resource
+ {{ init_retry()|indent(6) }}
- task: create_instance
{{- alpine_tools_config()|indent(8) }}
params:
@@ -234,9 +236,13 @@ jobs:
run:
path: geode-ci/ci/scripts/create_instance.sh
inputs:
+ - name: attempts-log
+ path: old
- name: concourse-metadata-resource
- name: geode-ci
outputs:
+ - name: attempts-log
+ path: new
- name: instance-data
timeout: 15m
attempts: 10
diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
index bcec2a1..1d2e50e 100644
--- a/ci/pipelines/pull-request/jinja.template.yml
+++ b/ci/pipelines/pull-request/jinja.template.yml
@@ -17,6 +17,7 @@
{% from 'shared_jinja.yml' import alpine_tools_config with context %}
{% from 'shared_jinja.yml' import pipeline_prefix with context %}
+{% from 'shared_jinja.yml' import init_retry with context %}
groups:
- name: main
@@ -87,6 +88,7 @@ jobs:
get_params: {skip_download: true}
- do:
- put: concourse-metadata-resource
+ {{ init_retry()|indent(8) }}
- task: create_instance
{{- alpine_tools_config()|indent(10) }}
params:
@@ -103,8 +105,12 @@ jobs:
- name: concourse-metadata-resource
- name: geode
- name: geode-ci
+ - name: attempts-log
+ path: old
outputs:
- name: instance-data
+ - name: attempts-log
+ path: new
timeout: 15m
attempts: 100
- task: rsync_code_up
diff --git a/ci/pipelines/shared/shared_jinja.yml b/ci/pipelines/shared/shared_jinja.yml
index 5c2d80e..080e411 100644
--- a/ci/pipelines/shared/shared_jinja.yml
+++ b/ci/pipelines/shared/shared_jinja.yml
@@ -38,3 +38,13 @@ username: ((github-username))
password: ((github-password))
{%- endif %}
{%- endmacro %}
+
+{%- macro init_retry() -%}
+- task: initial-output
+ {{- docker_config()|indent(2) }}
+ outputs:
+ - name: attempts-log
+ run:
+ path: touch
+ args: ['attempts-log/attempts']
+{%- endmacro %}
\ No newline at end of file
diff --git a/ci/scripts/create_instance.sh b/ci/scripts/create_instance.sh
index 27692ff..02cc769 100755
--- a/ci/scripts/create_instance.sh
+++ b/ci/scripts/create_instance.sh
@@ -59,10 +59,6 @@ if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then
WINDOWS_PREFIX="windows-"
fi
-PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f)
-ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]}
-echo "Deploying to zone ${ZONE}"
-
INSTANCE_NAME_STRING="${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-build${JAVA_BUILD_VERSION}-test${JAVA_TEST_VERSION}-job#${BUILD_NAME}"
INSTANCE_NAME="heavy-lifter-$(uuidgen -n @dns -s -N "${INSTANCE_NAME_STRING}")"
@@ -77,6 +73,20 @@ GCP_NETWORK=${GCP_NETWORK##*/}
GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
+# Determine and store our attempt number
+cp old/attempts new/
+echo attempt >> new/attempts
+attempts=$(cat new/attempts | wc -l)
+echo $attempts > /tmp/retry_number
+
+if [ $attempts -eq 1 ]; then
+ ZONE=${MY_ZONE}
+else
+ PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f)
+ ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]}
+fi
+echo "Deploying to zone ${ZONE}"
+
#in a retry loop we intentionally generate the same instance name, so make sure prior attempt is cleaned up
gcloud compute instances delete ${INSTANCE_NAME} --zone=${ZONE} --quiet &>/dev/null || true