You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by on...@apache.org on 2021/02/03 19:33:26 UTC
[geode] branch support/1.13 updated: GEODE-8914: remove defunct
metrics pipeline from ci (#5999)
This is an automated email from the ASF dual-hosted git repository.
onichols pushed a commit to branch support/1.13
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/support/1.13 by this push:
new 7eab046 GEODE-8914: remove defunct metrics pipeline from ci (#5999)
7eab046 is described below
commit 7eab046a1fc64e69462a6f06517a3d3874730358
Author: Owen Nichols <34...@users.noreply.github.com>
AuthorDate: Wed Feb 3 11:21:31 2021 -0800
GEODE-8914: remove defunct metrics pipeline from ci (#5999)
(cherry picked from commit aeb214c66d53aa68766f2772cdf4c0c46f36b638)
---
ci/pipelines/meta/deploy_meta.sh | 6 +-
ci/pipelines/meta/destroy_pipelines.sh | 2 +-
ci/pipelines/meta/jinja.template.yml | 51 ------------
ci/pipelines/metrics/deploy_metrics_pipeline.sh | 84 --------------------
ci/pipelines/metrics/jinja.template.yml | 101 ------------------------
5 files changed, 3 insertions(+), 241 deletions(-)
diff --git a/ci/pipelines/meta/deploy_meta.sh b/ci/pipelines/meta/deploy_meta.sh
index b093f08..0240327 100755
--- a/ci/pipelines/meta/deploy_meta.sh
+++ b/ci/pipelines/meta/deploy_meta.sh
@@ -256,15 +256,14 @@ set +x
if [[ "${GEODE_FORK}" != "${UPSTREAM_FORK}" ]]; then
echo "Disabling unnecessary jobs for forks."
pauseJobs ${META_PIPELINE} set-reaper-pipeline
- pauseNewJobs ${META_PIPELINE} set-metrics-pipeline
elif [[ "$GEODE_FORK" == "${UPSTREAM_FORK}" ]] && [[ "$GEODE_BRANCH" == "develop" ]]; then
echo "Disabling optional jobs for develop"
- pauseNewJobs ${META_PIPELINE} set-pr-pipeline set-metrics-pipeline set-examples-pipeline
+ pauseNewJobs ${META_PIPELINE} set-pr-pipeline set-examples-pipeline
else
echo "Disabling unnecessary jobs for support branches."
echo "*** DO NOT RE-ENABLE THESE META-JOBS ***"
pauseJobs ${META_PIPELINE} set-images-pipeline set-reaper-pipeline
- pauseNewJobs ${META_PIPELINE} set-pr-pipeline set-metrics-pipeline set-examples-pipeline
+ pauseNewJobs ${META_PIPELINE} set-pr-pipeline set-examples-pipeline
fi
unpausePipeline ${META_PIPELINE}
@@ -279,7 +278,6 @@ unpausePipeline ${PIPELINE_PREFIX}main
if [[ "$GEODE_FORK" == "${UPSTREAM_FORK}" ]]; then
if [[ "${PUBLIC}" == "true" ]]; then
exposePipelines ${PIPELINE_PREFIX}main ${PIPELINE_PREFIX}images
- enableFeature metrics
enableFeature examples
fi
if [[ "$GEODE_BRANCH" == "develop" ]]; then
diff --git a/ci/pipelines/meta/destroy_pipelines.sh b/ci/pipelines/meta/destroy_pipelines.sh
index 7203a85..3c6f761 100755
--- a/ci/pipelines/meta/destroy_pipelines.sh
+++ b/ci/pipelines/meta/destroy_pipelines.sh
@@ -80,5 +80,5 @@ function destroyPipelines {
done
}
-destroyPipelines ${PIPELINE_PREFIX}main ${PIPELINE_PREFIX}pr ${PIPELINE_PREFIX}images ${PIPELINE_PREFIX}reaper ${PIPELINE_PREFIX}metrics ${PIPELINE_PREFIX}examples ${PIPELINE_PREFIX}meta ${PIPELINE_PREFIX}rc
+destroyPipelines ${PIPELINE_PREFIX}main ${PIPELINE_PREFIX}pr ${PIPELINE_PREFIX}images ${PIPELINE_PREFIX}reaper ${PIPELINE_PREFIX}examples ${PIPELINE_PREFIX}meta ${PIPELINE_PREFIX}rc
echo "Destroyed ${CONCOURSE_URL}/teams/${CONCOURSE_TEAM}/pipelines/${PIPELINE_PREFIX}main and all related pipelines"
diff --git a/ci/pipelines/meta/jinja.template.yml b/ci/pipelines/meta/jinja.template.yml
index 5fd8ee3..f342f1a 100644
--- a/ci/pipelines/meta/jinja.template.yml
+++ b/ci/pipelines/meta/jinja.template.yml
@@ -94,13 +94,6 @@ resources:
- ci/pipelines/render.py
- ci/pipelines/meta/meta.properties
{% endif %}
-- name: geode-metrics-pipeline
- type: git
- source:
- {{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
- paths:
- - ci/pipelines/metrics/*
jobs:
- name: set-images-pipeline
@@ -274,50 +267,6 @@ jobs:
{% endif %}
-- name: set-metrics-pipeline
- serial: true
- public: ((!public-pipelines))
- plan:
- - get: geode-metrics-pipeline
- trigger: true
- - get: meta-mini-image
- trigger: true
- passed: [build-meta-mini-docker-image]
- - task: create-metrics-pipeline-yml
- image: meta-mini-image
- config:
- platform: linux
- inputs:
- - name: geode-metrics-pipeline
- outputs:
- - name: results
- params:
- OUTPUT_DIRECTORY: results
- GEODE_BRANCH: ((geode-build-branch))
- GEODE_FORK: ((geode-fork))
- GEODE_REPO_NAME: ((geode-repo-name))
- REPOSITORY_PUBLIC: {{ repository.public }}
- ARTIFACT_BUCKET: ((artifact-bucket))
- UPSTREAM_FORK: {{ repository.upstream_fork }}
- PIPELINE_PREFIX: ((pipeline-prefix))
- SANITIZED_GEODE_BRANCH: ((sanitized-geode-build-branch))
- SANITIZED_GEODE_FORK: ((sanitized-geode-fork))
- GCP_PROJECT: ((gcp-project))
- PUBLIC_PIPELINES: ((public-pipelines))
- CONCOURSE_URL: ((concourse-url))
- CONCOURSE_HOST: ((concourse-host))
- CONCOURSE_TEAM: ((concourse-team))
- run:
- path: geode-metrics-pipeline/ci/pipelines/metrics/deploy_metrics_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))metrics
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
-
- name: set-reaper-pipeline
serial: true
public: ((!public-pipelines))
diff --git a/ci/pipelines/metrics/deploy_metrics_pipeline.sh b/ci/pipelines/metrics/deploy_metrics_pipeline.sh
deleted file mode 100755
index 396e5cd..0000000
--- a/ci/pipelines/metrics/deploy_metrics_pipeline.sh
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-GEODEBUILDDIR="${SCRIPTDIR}/../geode-build"
-
-set -e
-
-if [ -z "${GEODE_BRANCH}" ]; then
- GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-fi
-
-if [ "${GEODE_BRANCH}" = "HEAD" ]; then
- echo "Unable to determine branch for deployment. Quitting..."
- exit 1
-fi
-
-
-echo "Sanitized Geode Fork = ${SANITIZED_GEODE_FORK}"
-echo "Sanitized Geode Branch = ${SANITIZED_GEODE_BRANCH}"
-
-
-#echo "DEBUG INFO *****************************"
-#echo "Pipeline prefix = ${PIPELINE_PREFIX}"
-#echo "Docker image prefix = ${DOCKER_IMAGE_PREFIX}"
-
-pushd ${SCRIPTDIR} 2>&1 > /dev/null
-
- cat > repository.yml <<YML
-repository:
- project: 'geode'
- fork: ${GEODE_FORK}
- branch: ${GEODE_BRANCH}
- upstream_fork: ${UPSTREAM_FORK}
- public: ${REPOSITORY_PUBLIC}
-YML
-
- cat > pipelineProperties.yml <<YML
-pipelineProperties:
- public: ${PUBLIC_PIPELINES}
-YML
-
- python3 ../render.py jinja.template.yml --variable-file ../shared/jinja.variables.yml repository.yml pipelineProperties.yml --environment ../shared/ --output ${SCRIPTDIR}/generated-pipeline.yml || exit 1
-
-popd 2>&1 > /dev/null
-cp ${SCRIPTDIR}/generated-pipeline.yml ${OUTPUT_DIRECTORY}/generated-pipeline.yml
-
-grep -n . ${OUTPUT_DIRECTORY}/generated-pipeline.yml
-
-cat > ${OUTPUT_DIRECTORY}/pipeline-vars.yml <<YML
-geode-build-branch: ${GEODE_BRANCH}
-geode-fork: ${GEODE_FORK}
-geode-repo-name: ${GEODE_REPO_NAME}
-upstream-fork: ${UPSTREAM_FORK}
-pipeline-prefix: "${PIPELINE_PREFIX}"
-public-pipelines: ${PUBLIC_PIPELINES}
-gcp-project: ${GCP_PROJECT}
-concourse-url: ${CONCOURSE_URL}
-concourse-team: ${CONCOURSE_TEAM}
-fly-target: ${CONCOURSE_HOST}-${CONCOURSE_TEAM}
-YML
-
-
diff --git a/ci/pipelines/metrics/jinja.template.yml b/ci/pipelines/metrics/jinja.template.yml
deleted file mode 100644
index 91573f8..0000000
--- a/ci/pipelines/metrics/jinja.template.yml
+++ /dev/null
@@ -1,101 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{% from 'shared_jinja.yml' import github_access with context %}
----
-
-resources:
-- name: once-a-day
- type: time
- source:
- interval: 24h
-- name: geode-ci
- type: git
- source:
- depth: 1
- uri: https://github.com/apache/geode.git
- branch: ((geode-build-branch))
- paths:
- - ci/*
- {{ github_access() | indent(4) }}
-- name: docker-geode-build-image
- type: docker-image
- source:
- username: ((docker-username))
- password: ((docker-password))
- repository: gcr.io/((gcp-project))/((pipeline-prefix))metric-tools
- tag: latest
-
-
-{%- macro metrics_job(job_name) %}
-- name: Geode{{job_name}}Metrics
- serial: true
- public: true
- plan:
- - aggregate:
- - get: docker-geode-build-image
- - get: geode-ci
- - get: once-a-day
- trigger: true
- - task: get_metrics
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode-ci
- outputs:
- - name: workspace
- platform: linux
- params:
- CONCOURSE_URL: ((concourse-url))
-{%- if repository.upstream_fork!="apache" %}
- CONCOURSE_USERNAME: ((concourse-username))
- CONCOURSE_PASSWORD: ((concourse-password))
-{%- endif %}
- MAIN_PIPELINE_NAME: ((pipeline-prefix))main
- JOB_NAME: {{job_name}}
- JOB_COUNT: 50
- CONCOURSE_TEAM: ((concourse-team))
- PUBLIC_PIPELINE: ((public-pipelines))
- UPSTREAM_FORK: ((upstream-fork))
- run:
- path: bash
- args:
- - -ecx
- - |-
- apk add jq &> /dev/null
- pip install yq &> /dev/null
- apk add curl &> /dev/null
- curl "${CONCOURSE_URL}/api/v1/cli?arch=amd64&platform=linux" --output fly
- chmod +x fly
- TARGET_NAME=concourse
-
- set +x
- ./fly -t ${TARGET_NAME} login -c "${CONCOURSE_URL}" -n "${CONCOURSE_TEAM}" -u "${CONCOURSE_USERNAME}" -p "${CONCOURSE_PASSWORD}"
- API_TOKEN=$(yq -r .targets.${TARGET_NAME}.token.value ~/.flyrc)
- PYTHON_ARGUMENTS="--number-of-builds ${JOB_COUNT} --team ${CONCOURSE_TEAM} --cookie-token {API_TOKEN} ${CONCOURSE_URL} ${MAIN_PIPELINE_NAME} ${JOB_NAME}"
- set -x
-
- python3 geode-ci/ci/bin/concourse_job_performance.py ${PYTHON_ARGUMENTS}
-
-{%- endmacro %}
-
-jobs:
-{{- metrics_job('Build') }}
- {%- for test in (tests) if not test.name=="StressNew" -%}
- {%- for java_test_version in (java_test_versions) %}
-{{ metrics_job(test.name+'Test'+java_test_version.name) }}
- {%- endfor -%}
- {%- endfor -%}