You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by rh...@apache.org on 2019/05/06 15:54:14 UTC
[geode] branch develop updated (aa63083 -> 4a84bcf)
This is an automated email from the ASF dual-hosted git repository.
rhoughton pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git.
from aa63083 GEODE-6687: replace ByteArrayInputStream+DataInputStream with ByteArrayDataInput (#3547)
new 1e46eed GEODE-6739: Create Concourse heavy-lift workers in the current zone
new 4a84bcf Removing unused file per PR comments.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/pipelines/geode-build/jinja.template.yml | 6 ++++++
ci/pipelines/pull-request/jinja.template.yml | 6 ++++++
ci/pipelines/shared/shared_jinja.yml | 10 ++++++++++
ci/scripts/create_instance.sh | 17 +++++++++++++----
4 files changed, 35 insertions(+), 4 deletions(-)
[geode] 02/02: Removing unused file per PR comments.
Posted by rh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rhoughton pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 4a84bcf4d3924707dab91d72ebe917ddb87193d3
Author: Helena Bales <hb...@pivotal.io>
AuthorDate: Mon May 6 08:51:29 2019 -0700
Removing unused file per PR comments.
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Helena Bales <hb...@pivotal.io>
---
ci/scripts/create_instance.sh | 1 -
1 file changed, 1 deletion(-)
diff --git a/ci/scripts/create_instance.sh b/ci/scripts/create_instance.sh
index 02cc769..db9476e 100755
--- a/ci/scripts/create_instance.sh
+++ b/ci/scripts/create_instance.sh
@@ -77,7 +77,6 @@ GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
cp old/attempts new/
echo attempt >> new/attempts
attempts=$(cat new/attempts | wc -l)
-echo $attempts > /tmp/retry_number
if [ $attempts -eq 1 ]; then
ZONE=${MY_ZONE}
[geode] 01/02: GEODE-6739: Create Concourse heavy-lift workers in
the current zone
Posted by rh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rhoughton pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 1e46eeda20b1a28ac85d28791108f5c7d80bbdf3
Author: Robert Houghton <rh...@pivotal.io>
AuthorDate: Fri May 3 13:02:14 2019 -0700
GEODE-6739: Create Concourse heavy-lift workers in the current zone
GCP costs are incurred on data egress between zones, even the same
region. Track our `create-instance` attempts, to put the heavy lifter in
the same zone on the first attempt, and roulette only on failure
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Helena Bales <hb...@pivotal.io>
---
ci/pipelines/geode-build/jinja.template.yml | 6 ++++++
ci/pipelines/pull-request/jinja.template.yml | 6 ++++++
ci/pipelines/shared/shared_jinja.yml | 10 ++++++++++
ci/scripts/create_instance.sh | 18 ++++++++++++++----
4 files changed, 36 insertions(+), 4 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 4ea7643..c5f16b5 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -18,6 +18,7 @@
{% from 'shared_jinja.yml' import alpine_tools_config with context %}
{% from 'shared_jinja.yml' import pipeline_prefix with context %}
{% from 'shared_jinja.yml' import github_access with context %}
+{% from 'shared_jinja.yml' import init_retry with context %}
---
@@ -224,6 +225,7 @@ jobs:
pre: ((semver-prerelease-token))
- do:
- put: concourse-metadata-resource
+ {{ init_retry()|indent(6) }}
- task: create_instance
{{- alpine_tools_config()|indent(8) }}
params:
@@ -234,9 +236,13 @@ jobs:
run:
path: geode-ci/ci/scripts/create_instance.sh
inputs:
+ - name: attempts-log
+ path: old
- name: concourse-metadata-resource
- name: geode-ci
outputs:
+ - name: attempts-log
+ path: new
- name: instance-data
timeout: 15m
attempts: 10
diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
index bcec2a1..1d2e50e 100644
--- a/ci/pipelines/pull-request/jinja.template.yml
+++ b/ci/pipelines/pull-request/jinja.template.yml
@@ -17,6 +17,7 @@
{% from 'shared_jinja.yml' import alpine_tools_config with context %}
{% from 'shared_jinja.yml' import pipeline_prefix with context %}
+{% from 'shared_jinja.yml' import init_retry with context %}
groups:
- name: main
@@ -87,6 +88,7 @@ jobs:
get_params: {skip_download: true}
- do:
- put: concourse-metadata-resource
+ {{ init_retry()|indent(8) }}
- task: create_instance
{{- alpine_tools_config()|indent(10) }}
params:
@@ -103,8 +105,12 @@ jobs:
- name: concourse-metadata-resource
- name: geode
- name: geode-ci
+ - name: attempts-log
+ path: old
outputs:
- name: instance-data
+ - name: attempts-log
+ path: new
timeout: 15m
attempts: 100
- task: rsync_code_up
diff --git a/ci/pipelines/shared/shared_jinja.yml b/ci/pipelines/shared/shared_jinja.yml
index 5c2d80e..080e411 100644
--- a/ci/pipelines/shared/shared_jinja.yml
+++ b/ci/pipelines/shared/shared_jinja.yml
@@ -38,3 +38,13 @@ username: ((github-username))
password: ((github-password))
{%- endif %}
{%- endmacro %}
+
+{%- macro init_retry() -%}
+- task: initial-output
+ {{- docker_config()|indent(2) }}
+ outputs:
+ - name: attempts-log
+ run:
+ path: touch
+ args: ['attempts-log/attempts']
+{%- endmacro %}
\ No newline at end of file
diff --git a/ci/scripts/create_instance.sh b/ci/scripts/create_instance.sh
index 27692ff..02cc769 100755
--- a/ci/scripts/create_instance.sh
+++ b/ci/scripts/create_instance.sh
@@ -59,10 +59,6 @@ if [[ "${SANITIZED_BUILD_JOB_NAME}" =~ [Ww]indows ]]; then
WINDOWS_PREFIX="windows-"
fi
-PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f)
-ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]}
-echo "Deploying to zone ${ZONE}"
-
INSTANCE_NAME_STRING="${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-build${JAVA_BUILD_VERSION}-test${JAVA_TEST_VERSION}-job#${BUILD_NAME}"
INSTANCE_NAME="heavy-lifter-$(uuidgen -n @dns -s -N "${INSTANCE_NAME_STRING}")"
@@ -77,6 +73,20 @@ GCP_NETWORK=${GCP_NETWORK##*/}
GCP_SUBNETWORK=$(echo ${NETWORK_INTERFACE_INFO} | jq -r '.networkInterfaces[0].subnetwork')
GCP_SUBNETWORK=${GCP_SUBNETWORK##*/}
+# Determine and store our attempt number
+cp old/attempts new/
+echo attempt >> new/attempts
+attempts=$(cat new/attempts | wc -l)
+echo $attempts > /tmp/retry_number
+
+if [ $attempts -eq 1 ]; then
+ ZONE=${MY_ZONE}
+else
+ PERMITTED_ZONES=(us-central1-a us-central1-b us-central1-c us-central1-f)
+ ZONE=${PERMITTED_ZONES[$((${RANDOM} % 4))]}
+fi
+echo "Deploying to zone ${ZONE}"
+
#in a retry loop we intentionally generate the same instance name, so make sure prior attempt is cleaned up
gcloud compute instances delete ${INSTANCE_NAME} --zone=${ZONE} --quiet &>/dev/null || true