You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by jb...@apache.org on 2018/08/14 00:37:11 UTC
[geode] branch develop updated: GEODE-5578 Create small docker
image for meta-pipeline jobs (#2319)
This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new b2f8e2c GEODE-5578 Create small docker image for meta-pipeline jobs (#2319)
b2f8e2c is described below
commit b2f8e2cedac0dab85525f3b3baf7f949ce9f8b52
Author: Robert Houghton <rh...@pivotal.io>
AuthorDate: Mon Aug 13 17:37:07 2018 -0700
GEODE-5578 Create small docker image for meta-pipeline jobs (#2319)
Meta pipeline hosts a job that creates an image used in the other jobs.
Resolves chicken/egg issue with create-images-pipeline and meta itself.
Fix version of concourse-pipeline resource type. Improve pause behavior for meta pipeline
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Finn Southerland <fs...@pivotal.io>
---
ci/images/meta-mini/Dockerfile | 25 ++++++++++++++++
ci/pipelines/deploy_meta.sh | 28 ++++++++++++-----
ci/pipelines/meta.yml | 68 +++++++++++++++++++++++++++++++++---------
3 files changed, 99 insertions(+), 22 deletions(-)
diff --git a/ci/images/meta-mini/Dockerfile b/ci/images/meta-mini/Dockerfile
new file mode 100644
index 0000000..adcd5ea
--- /dev/null
+++ b/ci/images/meta-mini/Dockerfile
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM alpine:3.7
+
+RUN apk --no-cache add \
+ bash \
+ curl \
+ jq \
+ python3 \
+ && curl -L -o /usr/local/bin/spruce https://github.com/geofffranks/spruce/releases/download/$(curl --silent "https://api.github.com/repos/geofffranks/spruce/releases/latest" | jq -r .tag_name)/spruce-linux-amd64 \
+ && curl -L -o /usr/local/bin/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux" \
+ && chmod +x /usr/local/bin/*
diff --git a/ci/pipelines/deploy_meta.sh b/ci/pipelines/deploy_meta.sh
index a76ac66..5736b2c 100755
--- a/ci/pipelines/deploy_meta.sh
+++ b/ci/pipelines/deploy_meta.sh
@@ -19,6 +19,7 @@ GEODE_BRANCH=$(git rev-parse --abbrev-ref HEAD)
SANITIZED_GEODE_BRANCH=$(echo ${GEODE_BRANCH} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
TARGET=geode
GEODE_FORK=${1:-apache}
+SANITIZED_GEODE_FORK=$(echo ${GEODE_FORK} | tr "/" "-" | tr '[:upper:]' '[:lower:]')
TEAM=$(fly targets | grep ^${TARGET} | awk '{print $3}')
PUBLIC=true
@@ -41,19 +42,30 @@ fly -t ${TARGET} set-pipeline \
-p ${META_PIPELINE} \
-c meta.yml \
--var geode-build-branch=${GEODE_BRANCH} \
+ --var sanitized-geode-build-branch=${SANITIZED_GEODE_BRANCH} \
+ --var sanitized-geode-fork=${SANITIZED_GEODE_FORK} \
--var geode-fork=${GEODE_FORK} \
--var pipeline-prefix=${PIPELINE_PREFIX} \
--var concourse-team=${TEAM} \
- --yaml-var public-pipelines=${PUBLIC}
+ --yaml-var public-pipelines=${PUBLIC} 2>&1 |tee flyOutput.log
+
set +x
+
+if [[ "$(tail -n1 flyOutput.log)" == "bailing out" ]]; then
+ exit 1
+fi
+
if [[ "${GEODE_FORK}" != "apache" ]]; then
echo "Disabling unnecessary jobs for forks."
set -x
- fly -t ${TARGET} pause-job \
- -j ${META_PIPELINE}/set-pr-pipeline
- fly -t ${TARGET} pause-job \
- -j ${META_PIPELINE}/set-metrics-pipeline
- fly -t ${TARGET} pause-job \
- -j ${META_PIPELINE}/set-reaper-pipeline
+ for job in set set-pr set-images set-metrics set-reaper set-examples; do
+ fly -t ${TARGET} pause-job \
+ -j ${META_PIPELINE}/${job}-pipeline
+ done
+
+ fly -t ${TARGET} trigger-job \
+ -j ${META_PIPELINE}/build-meta-mini-docker-image
+ fly -t ${TARGET} unpause-pipeline \
+ -p ${META_PIPELINE}
set +x
-fi
\ No newline at end of file
+fi
diff --git a/ci/pipelines/meta.yml b/ci/pipelines/meta.yml
index 542df7a..d44bf84 100644
--- a/ci/pipelines/meta.yml
+++ b/ci/pipelines/meta.yml
@@ -21,6 +21,7 @@ resource_types:
type: docker-image
source:
repository: concourse/concourse-pipeline-resource
+ tag: 3
resources:
- name: apachegeode-concourse
@@ -75,12 +76,19 @@ resources:
branch: ((!geode-build-branch))
paths:
- ci/pipelines/reaper.yml
-- name: apachegeode-build-concourse-docker-image
+- name: meta-mini-dockerfile
+ type: git
+ source:
+ uri: https://github.com/((!geode-fork))/geode.git
+ branch: ((!geode-build-branch))
+ paths:
+ - ci/images/meta-mini/*
+- name: meta-mini-image
type: docker-image
source:
username: ((!docker-username))
password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/apachegeode-build-concourse
+ repository: gcr.io/apachegeode-ci/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
jobs:
- name: set-pipeline
@@ -89,15 +97,20 @@ jobs:
plan:
- get: geode-pipeline
trigger: true
- - get: apachegeode-build-concourse-docker-image
- task: deploy-build
- image: apachegeode-build-concourse-docker-image
config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ tag: latest
+ repository: gcr.io/apachegeode-ci/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
inputs:
- name: geode-pipeline
outputs:
- name: results
- platform: linux
params:
OUTPUT_DIRECTORY: results
GEODE_BRANCH: ((!geode-build-branch))
@@ -114,15 +127,20 @@ jobs:
plan:
- get: geode-examples-pipeline
trigger: true
- - get: apachegeode-build-concourse-docker-image
- task: deploy-build
- image: apachegeode-build-concourse-docker-image
config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ tag: latest
+ repository: gcr.io/apachegeode-ci/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
inputs:
- name: geode-examples-pipeline
outputs:
- name: results
- platform: linux
params:
OUTPUT_DIRECTORY: results
GEODE_BRANCH: ((!geode-build-branch))
@@ -139,15 +157,20 @@ jobs:
plan:
- get: geode-pr-pipeline
trigger: true
- - get: apachegeode-build-concourse-docker-image
- task: deploy-pr
- image: apachegeode-build-concourse-docker-image
config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ tag: latest
+ repository: gcr.io/apachegeode-ci/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
inputs:
- name: geode-pr-pipeline
outputs:
- name: results
- platform: linux
params:
OUTPUT_DIRECTORY: results
GEODE_BRANCH: ((!geode-build-branch))
@@ -164,15 +187,20 @@ jobs:
plan:
- get: geode-images-pipeline
trigger: true
- - get: apachegeode-build-concourse-docker-image
- task: deploy-images
- image: apachegeode-build-concourse-docker-image
config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ tag: latest
+ repository: gcr.io/apachegeode-ci/((!sanitized-geode-fork))-((!sanitized-geode-build-branch))-meta-img
inputs:
- name: geode-images-pipeline
outputs:
- name: results
- platform: linux
params:
OUTPUT_DIRECTORY: results
GEODE_BRANCH: ((!geode-build-branch))
@@ -229,3 +257,15 @@ jobs:
config_file: geode-reaper-pipeline/ci/pipelines/reaper.yml
vars_files:
- pipeline-vars/vars
+
+- name: build-meta-mini-docker-image
+ public: ((!public-pipelines))
+ serial: true
+ plan:
+ - aggregate:
+ - get: meta-mini-dockerfile
+ trigger: true
+ - put: meta-mini-image
+ params:
+ build: meta-mini-dockerfile/ci/images/meta-mini
+ tag_as_latest: true