You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by sm...@apache.org on 2021/02/03 21:30:59 UTC
[geode] branch support/1.12 updated: GEODE-8915: pipeline changes
to support concourse 6.
This is an automated email from the ASF dual-hosted git repository.
smgoller pushed a commit to branch support/1.12
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/support/1.12 by this push:
new d1543d2 GEODE-8915: pipeline changes to support concourse 6.
d1543d2 is described below
commit d1543d2562cc590b3b42640c455d5484c2ee5083
Author: Sean Goller <se...@goller.net>
AuthorDate: Wed Feb 3 11:42:30 2021 -0800
GEODE-8915: pipeline changes to support concourse 6.
* Remove all references to ((!
* Use set_pipeline step instead of external
resource types to deploy pipelines internally
* use registry_mirror to pull from dockerhub
---
ci/pipelines/examples/jinja.template.yml | 12 ++-
ci/pipelines/geode-build/jinja.template.yml | 11 +-
ci/pipelines/images/jinja.template.yml | 2 +
ci/pipelines/meta/jinja.template.yml | 151 ++++++++++++++-------------
ci/pipelines/pull-request/jinja.template.yml | 14 +--
ci/pipelines/reaper/jinja.template.yml | 6 +-
6 files changed, 106 insertions(+), 90 deletions(-)
diff --git a/ci/pipelines/examples/jinja.template.yml b/ci/pipelines/examples/jinja.template.yml
index cd9ff8e..fb53e7b 100644
--- a/ci/pipelines/examples/jinja.template.yml
+++ b/ci/pipelines/examples/jinja.template.yml
@@ -29,7 +29,7 @@ RAM: {{ test.RAM }}
GRADLE_TASK_OPTIONS: {{ test.GRADLE_TASK_OPTIONS }}
{%- endif %}
MAINTENANCE_VERSION: ((geode-build-branch ))
-SERVICE_ACCOUNT: ((!concourse-gcp-account))
+SERVICE_ACCOUNT: ((concourse-gcp-account))
{%- endmacro %}
---
@@ -38,22 +38,28 @@ resource_types:
- name: gci
type: registry-image
source:
+ registry_mirror:
+ host: mirror.gcr.io
repository: smgoller/gci-resource
- name: maven-resource
type: registry-image
source:
+ registry_mirror:
+ host: mirror.gcr.io
repository: nulldriver/maven-resource
tag: latest
- name: concourse-metadata-resource
type: registry-image
source:
- password: ((!docker-password))
+ password: ((docker-password))
repository: gcr.io/((gcp-project))/((pipeline-prefix))concourse-metadata-resource
tag: latest
- username: ((!docker-username))
+ username: ((docker-username))
- name: gcs-resource
type: registry-image
source:
+ registry_mirror:
+ host: mirror.gcr.io
repository: frodenas/gcs-resource
- name: json-gcs-resource
type: docker-image
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 79ae842..a5be565 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -191,7 +191,7 @@ resources:
bucket: ((artifact-bucket))
driver: gcs
initial_version: {{ metadata.initial_version }}
- json_key: ((!concourse-gcp-key))
+ json_key: ((concourse-gcp-key))
key: semvers/((pipeline-prefix))((geode-build-branch))/number
- name: geode-passing-tokens
type: gcs-resource
@@ -226,6 +226,8 @@ resource_types:
- name: gci
type: registry-image
source:
+ registry_mirror:
+ host: mirror.gcr.io
repository: smgoller/gci-resource
- name: concourse-metadata-resource
type: docker-image
@@ -237,6 +239,7 @@ resource_types:
- name: gcs-resource
type: docker-image
source:
+ registry_mirror: mirror.gcr.io
repository: frodenas/gcs-resource
jobs:
{% for semverPiece in ['major', 'minor', 'patch'] -%}
@@ -379,7 +382,7 @@ jobs:
GRADLE_TASK: {{build_test.GRADLE_TASK}}
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
TAR_GEODE_BUILD_ARTIFACTS: true
run:
path: geode-ci/ci/scripts/archive_results.sh
@@ -591,7 +594,7 @@ jobs:
params:
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
JAVA_BUILD_VERSION: {{ java_build_version.version }}
MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
GRADLE_GLOBAL_ARGS: ((gradle-global-args))
@@ -727,7 +730,7 @@ jobs:
GRADLE_TASK: {{test.GRADLE_TASK}}
MAINTENANCE_VERSION: ((geode-build-branch))
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
run:
path: geode-ci/ci/scripts/archive_results.sh
inputs:
diff --git a/ci/pipelines/images/jinja.template.yml b/ci/pipelines/images/jinja.template.yml
index d0c6b7a..aebecd4 100644
--- a/ci/pipelines/images/jinja.template.yml
+++ b/ci/pipelines/images/jinja.template.yml
@@ -23,6 +23,8 @@ resource_types:
- name: gci
type: registry-image
source:
+ registry_mirror:
+ host: mirror.gcr.io
repository: smgoller/gci-resource
resources:
diff --git a/ci/pipelines/meta/jinja.template.yml b/ci/pipelines/meta/jinja.template.yml
index f342f1a..12a16ad 100644
--- a/ci/pipelines/meta/jinja.template.yml
+++ b/ci/pipelines/meta/jinja.template.yml
@@ -18,28 +18,12 @@
{% from 'shared_jinja.yml' import github_access with context %}
---
-resource_types:
-- name: concourse-pipeline
- type: docker-image
- source:
- repository: concourse/concourse-pipeline-resource
- tag: 2.2.0
-
resources:
-- name: concourse
- type: concourse-pipeline
- source:
- target: ((!concourse-url))
- insecure: "true"
- teams:
- - name: ((!concourse-team))
- username: ((!concourse-username))
- password: ((!concourse-password))
- name: geode-pipeline
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/pipelines/geode-build/*
- ci/pipelines/shared/*
@@ -49,35 +33,35 @@ resources:
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/pipelines/images/*
- name: geode-reaper-pipeline
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/pipelines/reaper/*
- name: meta-mini-dockerfile
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/images/meta-mini/*
- name: meta-mini-image
type: docker-image
source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/((!gcp-project))/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
+ username: ((docker-username))
+ password: ((docker-password))
+ repository: gcr.io/((gcp-project))/((sanitized-geode-fork))-((sanitized-geode-build-branch))-meta-img
{% if repository.fork == repository.upstream_fork %}
- name: geode-examples-pipeline
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/pipelines/examples/*
- ci/pipelines/shared/*
@@ -87,7 +71,7 @@ resources:
type: git
source:
{{ github_access() | indent(4) }}
- branch: ((!geode-build-branch))
+ branch: ((geode-build-branch))
paths:
- ci/pipelines/pull-request/*
- ci/pipelines/shared/*
@@ -98,7 +82,7 @@ resources:
jobs:
- name: set-images-pipeline
serial: true
- public: ((!public-pipelines))
+ public: ((public-pipelines))
plan:
- get: geode-images-pipeline
trigger: true
@@ -127,18 +111,13 @@ jobs:
PUBLIC_PIPELINES: ((public-pipelines))
run:
path: geode-images-pipeline/ci/pipelines/images/deploy_images_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))images
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
-
+ - set_pipeline: ((pipeline-prefix))images
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
- name: set-pipeline
serial: true
- public: ((!public-pipelines))
+ public: ((public-pipelines))
plan:
- get: geode-pipeline
trigger: true
@@ -171,20 +150,16 @@ jobs:
SEMVER_PRERELEASE_TOKEN: ((semver-prerelease-token))
run:
path: geode-pipeline/ci/pipelines/geode-build/deploy_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))main
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
+ - set_pipeline: ((pipeline-prefix))main
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
{% if repository.fork == repository.upstream_fork %}
- name: set-examples-pipeline
serial: true
- public: ((!public-pipelines))
+ public: ((public-pipelines))
plan:
- get: geode-examples-pipeline
trigger: true
@@ -216,18 +191,14 @@ jobs:
SEMVER_PRERELEASE_TOKEN: ((semver-prerelease-token))
run:
path: geode-examples-pipeline/ci/pipelines/examples/deploy_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))examples
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
+ - set_pipeline: ((pipeline-prefix))examples
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
- name: set-pr-pipeline
serial: true
- public: ((!public-pipelines))
+ public: ((public-pipelines))
plan:
- get: geode-pr-pipeline
trigger: true
@@ -244,8 +215,8 @@ jobs:
- name: results
params:
OUTPUT_DIRECTORY: results
- GEODE_BRANCH: ((!geode-build-branch))
- GEODE_FORK: ((!geode-fork))
+ GEODE_BRANCH: ((geode-build-branch))
+ GEODE_FORK: ((geode-fork))
GEODE_REPO_NAME: ((geode-repo-name))
REPOSITORY_PUBLIC: {{ repository.public }}
ARTIFACT_BUCKET: ((artifact-bucket))
@@ -256,20 +227,56 @@ jobs:
GRADLE_GLOBAL_ARGS: ((gradle-global-args))
run:
path: geode-pr-pipeline/ci/pipelines/pull-request/deploy_pr_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))pr
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
+ - set_pipeline: ((pipeline-prefix))pr
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
{% endif %}
+- name: set-metrics-pipeline
+ serial: true
+ public: ((public-pipelines))
+ plan:
+ - get: geode-metrics-pipeline
+ trigger: true
+ - get: meta-mini-image
+ trigger: true
+ passed: [build-meta-mini-docker-image]
+ - task: create-metrics-pipeline-yml
+ image: meta-mini-image
+ config:
+ platform: linux
+ inputs:
+ - name: geode-metrics-pipeline
+ outputs:
+ - name: results
+ params:
+ OUTPUT_DIRECTORY: results
+ GEODE_BRANCH: ((geode-build-branch))
+ GEODE_FORK: ((geode-fork))
+ GEODE_REPO_NAME: ((geode-repo-name))
+ REPOSITORY_PUBLIC: {{ repository.public }}
+ ARTIFACT_BUCKET: ((artifact-bucket))
+ UPSTREAM_FORK: {{ repository.upstream_fork }}
+ PIPELINE_PREFIX: ((pipeline-prefix))
+ SANITIZED_GEODE_BRANCH: ((sanitized-geode-build-branch))
+ SANITIZED_GEODE_FORK: ((sanitized-geode-fork))
+ GCP_PROJECT: ((gcp-project))
+ PUBLIC_PIPELINES: ((public-pipelines))
+ CONCOURSE_URL: ((concourse-url))
+ CONCOURSE_HOST: ((concourse-host))
+ CONCOURSE_TEAM: ((concourse-team))
+ run:
+ path: geode-metrics-pipeline/ci/pipelines/metrics/deploy_metrics_pipeline.sh
+ - set_pipeline: ((pipeline-prefix))metrics
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
+
- name: set-reaper-pipeline
serial: true
- public: ((!public-pipelines))
+ public: ((public-pipelines))
plan:
- get: geode-reaper-pipeline
trigger: true
@@ -298,17 +305,13 @@ jobs:
PUBLIC_PIPELINES: ((public-pipelines))
run:
path: geode-reaper-pipeline/ci/pipelines/reaper/deploy_reaper_pipeline.sh
- - put: concourse
- params:
- pipelines:
- - name: ((!pipeline-prefix))reaper
- team: ((!concourse-team))
- config_file: results/generated-pipeline.yml
- vars_files:
- - results/pipeline-vars.yml
+ - set_pipeline: ((pipeline-prefix))reaper
+ file: results/generated-pipeline.yml
+ var_files:
+ - results/pipeline-vars.yml
- name: build-meta-mini-docker-image
- public: ((!public-pipelines))
+ public: ((public-pipelines))
serial: true
plan:
- aggregate:
diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
index 4733eea..6605c94 100644
--- a/ci/pipelines/pull-request/jinja.template.yml
+++ b/ci/pipelines/pull-request/jinja.template.yml
@@ -72,10 +72,12 @@ resource_types:
- name: gci
type: registry-image
source:
+ registry_mirror: mirror.gcr.io
repository: smgoller/gci-resource
- name: gcs-resource
- type: docker-image
+ type: registry-image
source:
+ registry_mirror: mirror.gcr.io
repository: frodenas/gcs-resource
- name: pull-request
type: docker-image
@@ -84,10 +86,10 @@ resource_types:
- name: concourse-metadata-resource
type: docker-image
source:
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((pipeline-prefix))concourse-metadata-resource
+ password: ((docker-password))
+ repository: gcr.io/((gcp-project))/((pipeline-prefix))concourse-metadata-resource
tag: latest
- username: ((!docker-username))
+ username: ((docker-username))
jobs:
- name: {{build_test.name}}
public: true
@@ -217,7 +219,7 @@ jobs:
GRADLE_TASK: {{build_test.GRADLE_TASK}}
MAINTENANCE_VERSION: {{repository.branch}}
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
run:
path: geode-ci/ci/scripts/archive_results.sh
inputs:
@@ -385,7 +387,7 @@ jobs:
GRADLE_TASK: {{test.GRADLE_TASK}}
MAINTENANCE_VERSION: {{repository.branch}}
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
run:
path: geode-ci/ci/scripts/archive_results.sh
inputs:
diff --git a/ci/pipelines/reaper/jinja.template.yml b/ci/pipelines/reaper/jinja.template.yml
index 89d411c..949f5a0 100644
--- a/ci/pipelines/reaper/jinja.template.yml
+++ b/ci/pipelines/reaper/jinja.template.yml
@@ -22,9 +22,9 @@ resource_types:
- name: gce-instances
type: docker-image
source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/((gcp-project))/((!docker-image-prefix))gce-instances-resource
+ username: ((docker-username))
+ password: ((docker-password))
+ repository: gcr.io/((gcp-project))/((docker-image-prefix))gce-instances-resource
tag: latest
resources: