You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by mh...@apache.org on 2019/12/27 23:57:17 UTC
[geode] branch release/1.11.0 updated (3b95992 -> b72638e)
This is an automated email from the ASF dual-hosted git repository.
mhanson pushed a change to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git.
from 3b95992 Benchmark against previous release. (#4541)
new f5b706a GEODE-7431: limit Benchmarks to three-at-a-time to stay within quota (#4306)
new 3621f68 Separate benchmark tasks into jobs. (#4404)
new 6e7681d GEODE-7554: Add retry mechanism for failed tests (#4461)
new b72638e Fixes bash error in run_benchmarks.sh script. (#4498)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/pipelines/geode-build/jinja.template.yml | 93 ++++++++++++++++-------------
ci/pipelines/shared/jinja.variables.yml | 10 ++++
ci/scripts/run_benchmarks.sh | 46 ++++++++++++--
3 files changed, 102 insertions(+), 47 deletions(-)
[geode] 04/04: Fixes bash error in run_benchmarks.sh script. (#4498)
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhanson pushed a commit to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git
commit b72638e0ba01cc8f5843ec3c26bd6bcb862c966f
Author: Jacob Barrett <jb...@pivotal.io>
AuthorDate: Wed Dec 18 11:21:19 2019 -0800
Fixes bash error in run_benchmarks.sh script. (#4498)
(cherry picked from commit 6232299e7759d24c58150bfbbb13ce64e0d763cb)
---
ci/scripts/run_benchmarks.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ci/scripts/run_benchmarks.sh b/ci/scripts/run_benchmarks.sh
index ae18968..b17546e 100755
--- a/ci/scripts/run_benchmarks.sh
+++ b/ci/scripts/run_benchmarks.sh
@@ -41,7 +41,7 @@ pushd geode
GEODE_SHA=$(git rev-parse --verify HEAD)
popd
-input= "$(pwd)/results/failedTests"
+input="$(pwd)/results/failedTests"
pushd geode-benchmarks/infrastructure/scripts/aws/
./launch_cluster.sh -t ${CLUSTER_TAG} -c ${CLUSTER_COUNT} --ci
[geode] 01/04: GEODE-7431: limit Benchmarks to three-at-a-time to
stay within quota (#4306)
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhanson pushed a commit to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git
commit f5b706aefc1dc70a5b2fef6a7071660275083aee
Author: Owen Nichols <34...@users.noreply.github.com>
AuthorDate: Mon Nov 11 10:18:11 2019 -0800
GEODE-7431: limit Benchmarks to three-at-a-time to stay within quota (#4306)
(cherry picked from commit 9ac4454614f5e25ff691cacec2ec059a04d94437)
---
ci/pipelines/geode-build/jinja.template.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 1d54c81..a400a4a 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -407,6 +407,7 @@ jobs:
- name: Benchmark
public: true
+ max_in_flight: 3
plan:
- get: geode-ci
passed:
[geode] 02/04: Separate benchmark tasks into jobs. (#4404)
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhanson pushed a commit to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 3621f68a69ee01b79483783e4aac05daa1bfe3d8
Author: Robert Houghton <rh...@pivotal.io>
AuthorDate: Wed Dec 4 12:12:13 2019 -0800
Separate benchmark tasks into jobs. (#4404)
* Separate benchmark tasks into jobs.
To enable more granular re-run in Concourse, split the Benchmarks into
independently triggerable jobs in the main pipeline.
Extract variables from the template into the jinja.variables file.
Make better use of YML anchors.
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Helena Bales <hb...@pivotal.io>
* empty to trigger pr
(cherry picked from commit 1fcadf987706af7515a64869e088e1573695dae0)
---
ci/pipelines/geode-build/jinja.template.yml | 92 ++++++++++++++++-------------
ci/pipelines/shared/jinja.variables.yml | 10 ++++
2 files changed, 60 insertions(+), 42 deletions(-)
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index a400a4a..b33fb8e 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -100,7 +100,9 @@ groups:
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{%- endif %}
- - Benchmark
+ {%- for flavor in (benchmarks.flavors) %}
+ - Benchmark{{flavor.title}}
+ {%- endfor %}
- name: complete
jobs:
- {{ build_test.name }}
@@ -113,7 +115,9 @@ groups:
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{%- endif %}
- - Benchmark
+ {%- for flavor in (benchmarks.flavors) %}
+ - Benchmark{{flavor.title}}
+ {%- endfor %}
- name: linux
jobs:
- {{ build_test.name }}
@@ -122,7 +126,9 @@ groups:
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor %}
- - Benchmark
+ {%- for flavor in (benchmarks.flavors) %}
+ - Benchmark{{flavor.title}}
+ {%- endfor %}
- name: windows
jobs:
- {{ build_test.name }}
@@ -362,7 +368,9 @@ jobs:
{%- if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- PublishArtifacts
{% else %}
- - Benchmark
+ {% for flavor in (benchmarks.flavors) %}
+ - Benchmark{{flavor.title}}
+ {% endfor %}
{% endif %}
trigger: true
- get: geode-build-version
@@ -381,7 +389,7 @@ jobs:
path: bash
args:
- -cx
- - |-
+ - |
pushd geode
GEODE_SHA=$(git rev-parse HEAD)
popd
@@ -405,15 +413,16 @@ jobs:
params:
file: geode-passing-tokens/passing-build-tokens.json
-- name: Benchmark
+{% for run_var in (benchmarks.flavors) %}
+- name: Benchmark{{ run_var.title }}
public: true
max_in_flight: 3
plan:
- get: geode-ci
passed:
{{ all_gating_jobs() | indent(4) }}
+ - get: alpine-tools-image
- aggregate:
- - get: alpine-tools-image
- get: geode
passed:
{{ all_gating_jobs() | indent(6) }}
@@ -423,8 +432,7 @@ jobs:
passed:
{{ all_gating_jobs() | indent(6) }}
- put: concourse-metadata-resource
- - aggregate:
- {% for run_var in [{'title': '', 'flag':'', 'options': ''}, {'title': '_with_ssl', 'flag': '-PwithSsl', 'options': '--tests=*GetBenchmark --tests=*PutBenchmark'}, {'title': '_with_security_manager', 'flag': '-PwithSecurityManager', 'options': '--tests=Partitioned*'}] %}
+ - do:
- task: run_benchmarks{{ run_var.title }}
image: alpine-tools-image
config:
@@ -476,50 +484,50 @@ jobs:
- name: geode-benchmarks
- name: concourse-metadata-resource
- name: results
- {% endfor %}
+{% endfor %}
{% if repository.upstream_fork != "apache" or repository.branch == "develop" %}
- name: PublishArtifacts
public: true
plan:
- aggregate:
- get: geode-ci
- passed:
- - Benchmark
+ passed: &publish-passed-inputs
+ {% for flavor in (benchmarks.flavors) %}
+ - Benchmark{{flavor.title}}
+ {% endfor %}
- get: alpine-tools-image
- get: geode
- passed:
- - Benchmark
+ passed: *publish-passed-inputs
trigger: true
- aggregate:
- get: geode-build-version
- passed:
- - Benchmark
- - do:
- - put: concourse-metadata-resource
- {{ init_retry()|indent(6) }}
- - task: create_instance
- image: alpine-tools-image
- config:
- platform: linux
- params:
- {{ common_instance_params(publish_artifacts) | indent(12) }}
- GEODE_BRANCH: {{repository.branch}}
- GEODE_FORK: {{repository.fork}}
- JAVA_BUILD_VERSION: {{ java_build_version.version }}
- run:
- path: geode-ci/ci/scripts/create_instance.sh
- inputs:
- - name: concourse-metadata-resource
- - name: geode-ci
- - name: geode
- - name: attempts-log
- path: old
- outputs:
- - name: instance-data
- - name: attempts-log
- path: new
- timeout: 15m
- attempts: 10
+ trigger: true
+ passed: *publish-passed-inputs
+ - put: concourse-metadata-resource
+ {{ init_retry()|indent(4) }}
+ - task: create_instance
+ image: alpine-tools-image
+ config:
+ platform: linux
+ params:
+ {{ common_instance_params(publish_artifacts) | indent(8) }}
+ GEODE_BRANCH: {{repository.branch}}
+ GEODE_FORK: {{repository.fork}}
+ JAVA_BUILD_VERSION: {{ java_build_version.version }}
+ run:
+ path: geode-ci/ci/scripts/create_instance.sh
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode-ci
+ - name: geode
+ - name: attempts-log
+ path: old
+ outputs:
+ - name: instance-data
+ - name: attempts-log
+ path: new
+ timeout: 15m
+ attempts: 10
- task: rsync_code_up
image: alpine-tools-image
config:
diff --git a/ci/pipelines/shared/jinja.variables.yml b/ci/pipelines/shared/jinja.variables.yml
index 4b956b7..24fbcf3 100644
--- a/ci/pipelines/shared/jinja.variables.yml
+++ b/ci/pipelines/shared/jinja.variables.yml
@@ -19,6 +19,16 @@ benchmarks:
baseline_branch: ''
baseline_version: '1.10.0'
benchmark_branch: develop
+ flavors:
+ - title: '_base'
+ flag: ''
+ options: ''
+ - title: '_with_ssl'
+ flag: '-PwithSsl'
+ options: '--tests=*GetBenchmark --tests=*PutBenchmark'
+ - title: '_with_security_manager'
+ flag: '-PwithSecurityManager'
+ options: '--tests=Partitioned*'
build_test:
ARTIFACT_SLUG: build
[geode] 03/04: GEODE-7554: Add retry mechanism for failed tests
(#4461)
Posted by mh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mhanson pushed a commit to branch release/1.11.0
in repository https://gitbox.apache.org/repos/asf/geode.git
commit 6e7681ddeeb9f669b9c74941424676f6293c2b7e
Author: Helena Bales <hb...@pivotal.io>
AuthorDate: Mon Dec 16 11:16:55 2019 -0800
GEODE-7554: Add retry mechanism for failed tests (#4461)
* GEODE-7554: Add retry mechanism for failed tests
There are still some flaky benchmarks. Retry tests that have failed, up
to 5 times, to determine if the failure is legitimate or just a flaky
test. If the test fails 5 times in a row, we know that it is a
legitimate failure.
With each run of run_against_baseline (in the apache/geode-benchmarks
repo), the class names of all failed tests will be written to a file.
The run_benchmarks script called by CI will read the failed tests from
the file and run only those tests. Once the return code is 0, or once
we've tried 5 times, exit.
* build failedFile path dynamically
Signed-off-by: Helena A. Bales <hb...@pivotal.io>
* add print of iteration number for CI debugging of failures
Signed-off-by: Robert Houghton <rh...@pivotal.io>
* triggering CI again. Previous status never updated
Signed-off-by: Helena A. Bales <hb...@pivotal.io>
* refactor if statement for readability
* restarting CI
* remove hardcoded cluster tag
(cherry picked from commit 278c2470a5cac2c332d13914b935f0618b820a91)
---
ci/scripts/run_benchmarks.sh | 46 +++++++++++++++++++++++++++++++++++++++-----
1 file changed, 41 insertions(+), 5 deletions(-)
diff --git a/ci/scripts/run_benchmarks.sh b/ci/scripts/run_benchmarks.sh
index 92f072d..ae18968 100755
--- a/ci/scripts/run_benchmarks.sh
+++ b/ci/scripts/run_benchmarks.sh
@@ -41,15 +41,51 @@ pushd geode
GEODE_SHA=$(git rev-parse --verify HEAD)
popd
+input= "$(pwd)/results/failedTests"
+
pushd geode-benchmarks/infrastructure/scripts/aws/
./launch_cluster.sh -t ${CLUSTER_TAG} -c ${CLUSTER_COUNT} --ci
-if [ -z "${BASELINE_VERSION}" ]; then
- ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -B ${BASELINE_BRANCH} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci','benchmark_branch':'${BENCHMARK_BRANCH}','baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS}
-else
- ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} -V ${BASELINE_VERSION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci','benchmark_branch':'${BENCHMARK_BRANCH}','baseline_version':'${BASELINE_VERSION}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS}
-fi
+# test retry loop - Check if any tests have failed. If so, overwrite the TEST_OPTIONS with only the
+# failed tests. Test failures only result in an exit code of 1 when on the last iteration of loop.
+for i in {1..5}
+do
+ echo "This is ITERATION ${i} of benchmarking against baseline."
+
+ if [[ -f ${input} ]]; then
+ unset TEST_OPTIONS
+ TEST_OPTIONS=""
+ while IFS= read -r line; do
+ test=" --tests $line"
+ TEST_OPTIONS=${TEST_OPTIONS}${test}
+ done < ${input}
+
+ rm ${input}
+ fi
+
+ if [[ ${i} != 5 ]]; then
+ set +e
+ fi
+ if [ -z "${BASELINE_VERSION}" ]; then
+ BASELINE_OPTION="-B ${BASELINE_BRANCH}"
+ METADATA_BASELINE="'benchmark_branch':'${BASELINE_BRANCH}'"
+ else
+ BASELINE_OPTION="-V ${BASELINE_VERSION}"
+ METADATA_BASELINE="'benchmark_version':'${BASELINE_VERSION}'"
+ fi
+ ./run_on_cluster.sh -t ${CLUSTER_TAG} -- pkill -9 java
+ ./run_on_cluster.sh -t ${CLUSTER_TAG} -- rm /home/geode/locator10334view.dat;
+ ./run_against_baseline.sh -t ${CLUSTER_TAG} -b ${GEODE_SHA} ${BASELINE_OPTION} -e ${BENCHMARKS_BRANCH} -o ${RESULTS_DIR} -m "'source':'geode-ci',${METADATA_BASELINE},'baseline_branch':'${BASELINE_BRANCH}','geode_branch':'${GEODE_SHA}'" --ci -- ${FLAGS} ${TEST_OPTIONS}
+
+ if [[ $? -eq 0 ]]; then
+ break;
+ fi
+
+ if [[ i != 5 ]]; then
+ set -e
+ fi
+done
popd