You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by jb...@apache.org on 2018/08/15 18:01:06 UTC
[geode] branch develop updated: GEODE-5584: Converts all pipelines
to Jinja2 templates (#2331)
This is an automated email from the ASF dual-hosted git repository.
jbarrett pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new bac62ee GEODE-5584: Converts all pipelines to Jinja2 templates (#2331)
bac62ee is described below
commit bac62ee74c9e076ce9e61456f402e6791d46f84b
Author: Jacob Barrett <jb...@pivotal.io>
AuthorDate: Wed Aug 15 11:01:00 2018 -0700
GEODE-5584: Converts all pipelines to Jinja2 templates (#2331)
- Created new Jinja2 templates to generate pipeline.
- Deleted old spruce templates.
- Images updated to include Jinja and remove spruce.
Co-authored-by: Robert Houghton <rh...@pivotal.io>
Co-authored-by: Finn Southerland <fs...@pivotal.io>
Co-authored-by: Patrick Rhomberg <pr...@pivotal.io>
Co-authored-by: Jake Barrett <bj...@pivotal.io>
---
ci/docker/Dockerfile | 7 +-
ci/images/meta-mini/Dockerfile | 5 +-
ci/pipelines/.gitignore | 1 +
ci/pipelines/README.md | 14 +-
ci/pipelines/examples/deploy_pipeline.sh | 43 +--
.../examples/{examples.yml => jinja.template.yml} | 6 +-
ci/pipelines/geode-build/base.yml | 173 -----------
ci/pipelines/geode-build/deploy_pipeline.sh | 65 ++--
ci/pipelines/geode-build/jinja.template.yml | 344 +++++++++++++++++++++
ci/pipelines/geode-build/test-stubs/acceptance.yml | 31 --
.../geode-build/test-stubs/distributed.yml | 32 --
.../geode-build/test-stubs/integration.yml | 31 --
ci/pipelines/geode-build/test-stubs/upgrade.yml | 32 --
.../test-stubs/windows/windows-acceptance.yml | 23 --
.../test-stubs/windows/windows-distributed.yml | 24 --
.../test-stubs/windows/windows-integration.yml | 23 --
.../test-stubs/windows/windows-unit.yml | 23 --
ci/pipelines/geode-build/test-template.yml | 136 --------
ci/pipelines/geode-build/windows-test-template.yml | 89 ------
ci/pipelines/images/deploy_images_pipeline.sh | 10 -
ci/pipelines/jinja.variables.yml | 106 +++++++
ci/pipelines/pull-request/base.yml | 135 --------
ci/pipelines/pull-request/deploy_pr_pipeline.sh | 77 ++---
ci/pipelines/pull-request/jinja.template.yml | 262 ++++++++++++++++
ci/pipelines/pull-request/pr-template.yml | 172 -----------
ci/pipelines/pull-request/test-stubs/newtests.yml | 33 --
ci/pipelines/render.py | 73 +++++
ci/pipelines/shared_jinja.yml | 32 ++
28 files changed, 908 insertions(+), 1094 deletions(-)
diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile
index c7bf95f..c401cb6 100644
--- a/ci/docker/Dockerfile
+++ b/ci/docker/Dockerfile
@@ -17,19 +17,16 @@ FROM openjdk:8
ENTRYPOINT []
ARG CHROME_DRIVER_VERSION=2.35
-ENV SPRUCE_VERSION 1.17.0
WORKDIR /tmp/work
COPY --from=hashicorp/packer:latest /bin/packer /usr/local/bin/packer
-ADD https://github.com/geofffranks/spruce/releases/download/v${SPRUCE_VERSION}/spruce-linux-amd64 /usr/local/bin/spruce
ADD https://github.com/krallin/tini/releases/download/v0.14.0/tini-static-amd64 /usr/local/bin/tini
ADD tini-wrapper.go .
ADD ./initdocker /usr/local/bin/initdocker
ADD cache_dependencies.sh cache_dependencies.sh
RUN chmod +x /usr/local/bin/tini \
- && chmod +x /usr/local/bin/spruce \
&& chmod +x /usr/local/bin/initdocker \
&& chmod +x cache_dependencies.sh \
&& apt-get update \
@@ -48,8 +45,8 @@ RUN chmod +x /usr/local/bin/tini \
aptitude \
ca-certificates \
cgroupfs-mount \
- docker-compose \
docker-ce \
+ docker-compose \
golang \
google-chrome-stable \
google-cloud-sdk \
@@ -57,8 +54,10 @@ RUN chmod +x /usr/local/bin/tini \
jq \
python3 \
python3-pip \
+ python3-setuptools \
unzip \
vim \
+ && pip3 install Jinja2 PyYAML \
&& gcloud config set core/disable_usage_reporting true \
&& gcloud config set component_manager/disable_update_check true \
&& gcloud config set metrics/environment github_docker_image \
diff --git a/ci/images/meta-mini/Dockerfile b/ci/images/meta-mini/Dockerfile
index adcd5ea..4929c06 100644
--- a/ci/images/meta-mini/Dockerfile
+++ b/ci/images/meta-mini/Dockerfile
@@ -20,6 +20,7 @@ RUN apk --no-cache add \
curl \
jq \
python3 \
- && curl -L -o /usr/local/bin/spruce https://github.com/geofffranks/spruce/releases/download/$(curl --silent "https://api.github.com/repos/geofffranks/spruce/releases/latest" | jq -r .tag_name)/spruce-linux-amd64 \
&& curl -L -o /usr/local/bin/fly "https://concourse.apachegeode-ci.info/api/v1/cli?arch=amd64&platform=linux" \
- && chmod +x /usr/local/bin/*
+ && pip3 install --upgrade pip \
+ && pip3 install Jinja2 pyYAML \
+ && chmod +x /usr/local/bin/* \
diff --git a/ci/pipelines/.gitignore b/ci/pipelines/.gitignore
new file mode 100644
index 0000000..9ed6311
--- /dev/null
+++ b/ci/pipelines/.gitignore
@@ -0,0 +1 @@
+generated-pipeline.yml
diff --git a/ci/pipelines/README.md b/ci/pipelines/README.md
index 773b5a8..1f761ed 100644
--- a/ci/pipelines/README.md
+++ b/ci/pipelines/README.md
@@ -2,4 +2,16 @@
```bash
./deploy_meta.sh <github account name>
-```
\ No newline at end of file
+```
+
+# Generating Pipeline Templates Manually
+To generate a pipeline, using jinja:
+* Be in the pipelines directory or a subdirectory
+* With ${GEODE_BRANCH} and ${GEODE_FORK} correctly set in your environment
+(for the pipeline you want to create):
+
+```bash
+./render.py <path to jinja template> <path to jinja variables> <path to generated file>
+```
+
+The generated file should be named `generated-pipeline.yml`.
\ No newline at end of file
diff --git a/ci/pipelines/examples/deploy_pipeline.sh b/ci/pipelines/examples/deploy_pipeline.sh
index 825ea3c..32edefa 100755
--- a/ci/pipelines/examples/deploy_pipeline.sh
+++ b/ci/pipelines/examples/deploy_pipeline.sh
@@ -24,15 +24,14 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-if ! [ -x "$(command -v spruce)" ]; then
- echo "Spruce must be installed for pipeline deployment to work."
- echo "For macos: 'brew tap starkandwayne/cf; brew install spruce'"
- echo "For Ubuntu: follow the instructions at https://github.com/geofffranks/spruce"
+for cmd in Jinja2 PyYAML; do
+ if ! [[ $(pip3 list |grep ${cmd}) ]]; then
+ echo "${cmd} must be installed for pipeline deployment to work."
+ echo " 'pip3 install ${cmd}'"
echo ""
exit 1
-else
- SPRUCE=$(which spruce || true)
-fi
+ fi
+done
set -e
@@ -55,15 +54,6 @@ chmod +x ${BIN_DIR}/fly
PATH=${PATH}:${BIN_DIR}
-echo "Spruce branch-name into resources"
-${SPRUCE} merge --prune metadata \
- ${SCRIPTDIR}/examples.yml \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}"; \
- echo " ") > ${TMP_DIR}/final.yml
-
-
TARGET="geode"
TEAM="staging"
@@ -80,9 +70,20 @@ else
fi
PIPELINE_NAME="${PIPELINE_NAME}-examples"
-fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD}
-fly -t ${TARGET} set-pipeline --non-interactive \
- --pipeline ${PIPELINE_NAME} \
- --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
- --config ${TMP_DIR}/final.yml
+pushd ${SCRIPTDIR} 2>&1 > /dev/null
+ # Template and output share a directory with this script, but variables are shared in the parent directory.
+ python3 ../render.py jinja.template.yml ../jinja.variables.yml generated-pipeline.yml || exit 1
+
+ fly login -t ${TARGET} \
+ -n ${TEAM} \
+ -c https://concourse.apachegeode-ci.info \
+ -u ${CONCOURSE_USERNAME} \
+ -p ${CONCOURSE_PASSWORD}
+
+ fly -t ${TARGET} set-pipeline \
+ --non-interactive \
+ --pipeline ${PIPELINE_NAME} \
+ --config generated-pipeline.yml \
+ --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
+popd 2>&1 > /dev/null
diff --git a/ci/pipelines/examples/examples.yml b/ci/pipelines/examples/jinja.template.yml
similarity index 91%
rename from ci/pipelines/examples/examples.yml
rename to ci/pipelines/examples/jinja.template.yml
index 79dd2bd..2825ec4 100644
--- a/ci/pipelines/examples/examples.yml
+++ b/ci/pipelines/examples/jinja.template.yml
@@ -35,8 +35,8 @@ resources:
type: git
source:
depth: 1
- uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" ))
- branch: (( grab metadata.geode-build-branch ))
+ uri: https://github.com/{{repository.fork}}/geode.git
+ branch: {{ repository.branch }}
paths:
- ci/pipelines/geode-build/*
- ci/scripts/*
@@ -76,7 +76,7 @@ jobs:
- name: docker-geode-build-image
platform: linux
params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
+ MAINTENANCE_VERSION: {{repository.branch}}
SERVICE_ACCOUNT: ((!concourse-gcp-account))
PUBLIC_BUCKET: ((!public-bucket))
run:
diff --git a/ci/pipelines/geode-build/base.yml b/ci/pipelines/geode-build/base.yml
deleted file mode 100644
index 919ca01..0000000
--- a/ci/pipelines/geode-build/base.yml
+++ /dev/null
@@ -1,173 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
----
-
-resource_types:
- - name: gcs-resource
- type: docker-image
- source:
- repository: frodenas/gcs-resource
- - name: email
- type: docker-image
- source:
- repository: pcfseceng/email-resource
- - name: concourse-metadata-resource
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
- tag: latest
-
-resources:
-- name: docker-geode-build-image
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
- tag: latest
-- name: geode
- type: git
- source:
- depth: 1
- uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" ))
- branch: (( grab metadata.geode-build-branch ))
- ignore_paths:
- - ci/*
-- name: geode-ci
- type: git
- source:
- depth: 1
- uri: (( concat "https://github.com/" metadata.geode-fork "/geode.git" ))
- branch: (( grab metadata.geode-build-branch ))
- paths:
- - ci/pipelines/geode-build/*
- - ci/scripts/*
-- name: geode-build-version
- type: semver
- source:
- driver: gcs
- bucket: ((!concourse-bucket))
- key: (( concat metadata.geode-build-branch "/version" ))
- json_key: ((!concourse-gcp-key))
- initial_version: 1.3.0
-- name: geode-build-artifact
- type: gcs-resource
- source:
- bucket: ((!public-bucket))
- json_key: ((!concourse-gcp-key))
- regexp: (( concat "artifacts/" metadata.geode-build-branch "/geodefiles-(.*).tgz" ))
-- name: send-notification-email
- type: email
- source:
- smtp:
- host: ((!source-email-server))
- port: "587"
- username: ((!source-email-username))
- password: ((!source-email-password))
- from: ((!source-email-address))
- to: [ ((!notification-email-address)) ]
-- name: concourse-metadata-resource
- type: concourse-metadata-resource
-
-groups:
-- name: main
- jobs:
- - Build
- - DistributedTest
- - AcceptanceTest
- - IntegrationTest
- - WindowsUnitTest
- - WindowsIntegrationTest
- - WindowsAcceptanceTest
- - WindowsGfshDistributedTest
- - UpgradeTest
- - UpdatePassingRef
-
-jobs:
-- name: Build
- serial: true
- public: true
- plan:
- - get: geode
- trigger: true
- - get: geode-ci
- - get: geode-build-version
- params: {pre: build}
- - task: build
- config:
- platform: linux
- image_resource:
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
- tag: latest
- inputs:
- - name: geode
- - name: geode-ci
- - name: geode-build-version
- outputs:
- - name: built-geode
- - name: results
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- path: geode-ci/ci/scripts/build.sh
- on_failure:
- aggregate:
- - put: send-notification-email
- params:
- subject: results/subject
- body: results/body
- ensure:
- aggregate:
- - put: geode-build-artifact
- params:
- file: built-geode/geodefiles-*.tgz
- - put: geode-build-version
- params:
- file: results/number
-- name: UpdatePassingRef
- serial: true
- public: true
- plan:
- - get: geode
- passed: [AcceptanceTest, DistributedTest, IntegrationTest, UpgradeTest]
- trigger: true
- - get: geode-ci
- - get: docker-geode-build-image
- - task: updatepassingref
- image: docker-geode-build-image
- config:
- inputs:
- - name: geode
- - name: geode-ci
- platform: linux
- outputs:
- - name: results
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- path: geode-ci/ci/scripts/update-passing-ref.sh
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh
index 1be9812..9676ab3 100755
--- a/ci/pipelines/geode-build/deploy_pipeline.sh
+++ b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -24,15 +24,14 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-if ! [ -x "$(command -v spruce)" ]; then
- echo "Spruce must be installed for pipeline deployment to work."
- echo "For macos: 'brew tap starkandwayne/cf; brew install spruce'"
- echo "For Ubuntu: follow the instructions at https://github.com/geofffranks/spruce"
+for cmd in Jinja2 PyYAML; do
+ if ! [[ $(pip3 list |grep ${cmd}) ]]; then
+ echo "${cmd} must be installed for pipeline deployment to work."
+ echo " 'pip3 install ${cmd}'"
echo ""
exit 1
-else
- SPRUCE=$(which spruce || true)
-fi
+ fi
+done
set -e
@@ -55,38 +54,6 @@ chmod +x ${BIN_DIR}/fly
PATH=${PATH}:${BIN_DIR}
-for i in ${SCRIPTDIR}/test-stubs/*.yml; do
- X=$(basename $i)
- echo "Merging ${i} into ${TMP_DIR}/${X}"
- ${SPRUCE} merge --prune metadata \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}") \
- ${SCRIPTDIR}/test-template.yml \
- ${i} > ${TMP_DIR}/${X}
-done
-
-for i in ${SCRIPTDIR}/test-stubs/windows/*.yml; do
- X=$(basename $i)
- echo "Merging ${i} into ${TMP_DIR}/${X}"
- ${SPRUCE} merge --prune metadata \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}") \
- ${SCRIPTDIR}/windows-test-template.yml \
- ${i} > ${TMP_DIR}/${X}
-done
-
-echo "Spruce branch-name into resources"
-${SPRUCE} merge --prune metadata \
- ${SCRIPTDIR}/base.yml \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}"; \
- echo " ") \
- ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml
-
-
TARGET="geode"
TEAM="staging"
@@ -102,9 +69,19 @@ else
DOCKER_IMAGE_PREFIX="${PIPELINE_NAME}-"
fi
-fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD}
-fly -t ${TARGET} set-pipeline --non-interactive \
- --pipeline ${PIPELINE_NAME} \
- --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
- --config ${TMP_DIR}/final.yml
+pushd ${SCRIPTDIR} 2>&1 > /dev/null
+ # Template and output share a directory with this script, but variables are shared in the parent directory.
+ python3 ../render.py jinja.template.yml ../jinja.variables.yml generated-pipeline.yml || exit 1
+
+ fly login -t ${TARGET} \
+ -n ${TEAM} \
+ -c https://concourse.apachegeode-ci.info \
+ -u ${CONCOURSE_USERNAME} \
+ -p ${CONCOURSE_PASSWORD}
+
+ fly -t ${TARGET} set-pipeline --non-interactive \
+ --pipeline ${PIPELINE_NAME} \
+ --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
+ --config generated-pipeline.yml
+popd 2>&1 > /dev/null
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
new file mode 100644
index 0000000..29afa52
--- /dev/null
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -0,0 +1,344 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{% from 'shared_jinja.yml' import alpine_tools_config with context %}
+
+{% macro plan_resource_gets() %}
+- get: geode-ci
+- aggregate:
+ - get: geode
+ passed:
+ - Build
+ - get: geode-build-version
+ passed:
+ - Build
+ trigger: true
+{% endmacro %}
+
+{% macro common_test_params(test) %}
+CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
+MAINTENANCE_VERSION: {{repository.branch}}
+PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
+PUBLIC_BUCKET: ((!public-bucket))
+SERVICE_ACCOUNT: ((!concourse-gcp-account))
+{% endmacro %}
+
+groups:
+- name: main
+ jobs:
+ - Build
+ {% for test in tests + windowstests -%}
+ - {{test.name}}Test
+ {% endfor -%}
+ - UpdatePassingRef
+
+resources:
+- name: geode-build-artifact
+ type: gcs-resource
+ source:
+ bucket: ((!public-bucket))
+ json_key: ((!concourse-gcp-key))
+ regexp: artifacts/jinja/geodefiles-(.*).tgz
+- name: send-notification-email
+ type: email
+ source:
+ from: ((!source-email-address))
+ smtp:
+ host: ((!source-email-server))
+ password: ((!source-email-password))
+ port: "587"
+ username: ((!source-email-username))
+ to:
+ - ((!notification-email-address))
+- name: concourse-metadata-resource
+ type: concourse-metadata-resource
+ source: {}
+- name: docker-geode-build-image
+ type: docker-image
+ source:
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-((!docker-image-name))
+ tag: latest
+ username: ((!docker-username))
+- name: geode
+ type: git
+ source:
+ branch: {{repository.branch}}
+ depth: 1
+ ignore_paths:
+ - ci/*
+ uri: https://github.com/{{repository.fork}}/{{repository.project}}.git
+- name: geode-ci
+ type: git
+ source:
+ branch: {{repository.branch}}
+ depth: 1
+ paths:
+ - ci/pipelines/geode-build/*
+ - ci/scripts/*
+ uri: https://github.com/{{repository.fork}}/{{repository.project}}.git
+- name: geode-build-version
+ type: semver
+ source:
+ bucket: ((!concourse-bucket))
+ driver: gcs
+ initial_version: 1.3.0
+ json_key: ((!concourse-gcp-key))
+ key: {{repository.branch}}/version
+
+resource_types:
+- name: concourse-metadata-resource
+ type: docker-image
+ source:
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-concourse-metadata-resource
+ tag: latest
+ username: ((!docker-username))
+- name: gcs-resource
+ type: docker-image
+ source:
+ repository: frodenas/gcs-resource
+- name: email
+ type: docker-image
+ source:
+ repository: pcfseceng/email-resource
+
+jobs:
+- name: Build
+ public: true
+ serial: true
+ plan:
+ - get: geode
+ trigger: true
+ - get: geode-ci
+ - get: geode-build-version
+ params:
+ pre: build
+ - task: build
+ config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-((!docker-image-name))
+ tag: latest
+ username: ((!docker-username))
+ params:
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: geode-ci/ci/scripts/build.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ - name: geode-build-version
+ outputs:
+ - name: built-geode
+ - name: results
+ on_failure:
+ aggregate:
+ - put: send-notification-email
+ params:
+ body: results/body
+ subject: results/subject
+ ensure:
+ aggregate:
+ - put: geode-build-artifact
+ params:
+ file: built-geode/geodefiles-*.tgz
+ - put: geode-build-version
+ params:
+ file: results/number
+- name: UpdatePassingRef
+ public: true
+ serial: true
+ plan:
+ - get: geode
+ passed:
+ {% for test in tests -%}
+ - {{test.name}}Test
+ {% endfor %}
+ trigger: true
+ - get: geode-ci
+ - get: docker-geode-build-image
+ - task: updatepassingref
+ config:
+ platform: linux
+ params:
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: geode-ci/ci/scripts/update-passing-ref.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ outputs:
+ - name: results
+ image: docker-geode-build-image
+{% for test in tests %}
+- name: {{test.name}}Test
+ public: true
+ plan:
+ - do:
+ {{ plan_resource_gets() |indent(4) }}
+ - do:
+ - put: concourse-metadata-resource
+ - task: start_instance
+ {{ alpine_tools_config()|indent(10) }}
+ params:
+ CPUS: {{test.CPUS}}
+ GEODE_BRANCH: {{repository.branch}}
+ GEODE_FORK: {{repository.fork}}
+ RAM: {{test.RAM}}
+ run:
+ path: geode-ci/ci/scripts/start_instance.sh
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode-ci
+ outputs:
+ - name: instance-data
+ timeout: 15m
+ attempts: 100
+ - task: rsync_code_up
+ {{ alpine_tools_config()|indent(6) }}
+ run:
+ path: geode-ci/ci/scripts/rsync_code_up.sh
+ inputs:
+ - name: geode-ci
+ - name: geode
+ - name: instance-data
+ timeout: 5m
+ - task: execute_tests
+ {{ alpine_tools_config()|indent(6) }}
+ params:
+ ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}
+ GRADLE_TASK: {{test.GRADLE_TASK}}
+ GRADLE_TASK_OPTIONS: ""
+ {{ common_test_params(test) | indent(10) }}
+ run:
+ path: geode-ci/ci/scripts/execute_tests.sh
+ inputs:
+ - name: geode-ci
+ - name: geode
+ - name: instance-data
+ timeout: {{test.execute_test_timeout}}
+ ensure:
+ do:
+ - task: rsync_code_down
+ {{ alpine_tools_config()|indent(8) }}
+ run:
+ path: geode-ci/ci/scripts/rsync_code_down.sh
+ inputs:
+ - name: geode-ci
+ - name: instance-data
+ outputs:
+ - name: geode-results
+ timeout: 5m
+ ensure:
+ do:
+ - aggregate:
+ - task: archive_results
+ {{ alpine_tools_config()|indent(12) }}
+ params:
+ ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}
+ GRADLE_TASK: {{test.GRADLE_TASK}}
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: geode-ci/ci/scripts/archive_results.sh
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode-ci
+ - name: geode-build-version
+ - name: geode-results
+ timeout: 1h
+ - task: stop_instance
+ {{ alpine_tools_config()|indent(12) }}
+ run:
+ path: geode-ci/ci/scripts/stop_instance.sh
+ inputs:
+ - name: geode-ci
+ - name: instance-data
+ timeout: 1h
+{% endfor -%}
+{% for test in windowstests %}
+- name: {{test.name}}Test
+ public: true
+ plan:
+ {{ plan_resource_gets() |indent(2) }}
+ - task: cleanup-java-processes
+ config:
+ platform: windows
+ run:
+ path: powershell
+ args:
+ - -command
+ - |
+ gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
+ kill -name java -force
+ exit 0
+ tags:
+ - windows-{{test.tags}}-tests
+ - task: execute_tests
+ config:
+ platform: windows
+ params:
+ JAVA_HOME: C:\progra~1\java\jdk1.8.0_181
+ {{ common_test_params(test) | indent(8) }}
+ run:
+ path: bash
+ args:
+ - geode-ci/ci/scripts/windows/test-run.sh
+ - {{test.target_arg}}
+ - {{test.dir_arg}}
+ - {{test.package_arg}}
+ inputs:
+ - name: geode-ci
+ - name: geode
+ outputs:
+ - name: built-geode
+ tags:
+ - windows-{{test.tags}}-tests
+ ensure:
+ aggregate:
+ - task: archive_results
+ config:
+ platform: windows
+ params:
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: bash
+ args:
+ - geode-ci/ci/scripts/windows/test-archive.sh
+ - {{test.target_arg}}
+ - {{test.dir_arg}}
+ inputs:
+ - name: geode-ci
+ - name: geode-build-version
+ - name: built-geode
+ tags:
+ - windows-{{test.tags}}-tests
+ timeout: 1h
+ timeout: 4h
+{% endfor %}
diff --git a/ci/pipelines/geode-build/test-stubs/acceptance.yml b/ci/pipelines/geode-build/test-stubs/acceptance.yml
deleted file mode 100644
index 41f8d87..0000000
--- a/ci/pipelines/geode-build/test-stubs/acceptance.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: AcceptanceTest
- gradle_task: :geode-assembly:acceptanceTest geode-connectors:acceptanceTest
- artifact_slug: acceptancetestfiles
- dunit:
- parallel: true
- forks: 7
- cpus: 8
-# specified in Gigabytes.
- ram: 12
-# specified in seconds
- call_stack_timeout: 1800
- timeout: 45m
- size: []
diff --git a/ci/pipelines/geode-build/test-stubs/distributed.yml b/ci/pipelines/geode-build/test-stubs/distributed.yml
deleted file mode 100644
index aa5fd27..0000000
--- a/ci/pipelines/geode-build/test-stubs/distributed.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: DistributedTest
- gradle_task: distributedTest
- artifact_slug: distributedtestfiles
- dunit:
- parallel: true
-# max number of docker containers to run, generally cpus/2
- forks: 24
- cpus: 96
-# specified in Gigabytes.
- ram: 180
-# specified in seconds
- call_stack_timeout: 7200
- timeout: 2h15m
- size: []
diff --git a/ci/pipelines/geode-build/test-stubs/integration.yml b/ci/pipelines/geode-build/test-stubs/integration.yml
deleted file mode 100644
index 729252d..0000000
--- a/ci/pipelines/geode-build/test-stubs/integration.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: IntegrationTest
- gradle_task: integrationTest
- artifact_slug: integrationtestfiles
- dunit:
- parallel: true
- forks: 48
- cpus: 96
-# specified in Gigabytes.
- ram: 90
-# specified in seconds
- call_stack_timeout: 1500
- timeout: 40m
- size: []
diff --git a/ci/pipelines/geode-build/test-stubs/upgrade.yml b/ci/pipelines/geode-build/test-stubs/upgrade.yml
deleted file mode 100644
index f02afe7..0000000
--- a/ci/pipelines/geode-build/test-stubs/upgrade.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: UpgradeTest
- gradle_task: upgradeTest
- artifact_slug: upgradetestfiles
- dunit:
- parallel: true
-# max number of docker containers to run, generally cpus/2
- forks: 48
- cpus: 96
-# specified in Gigabytes.
- ram: 160
-# specified in seconds
- call_stack_timeout: 3000
- timeout: 1h
- size: []
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml
deleted file mode 100644
index a414f2e..0000000
--- a/ci/pipelines/geode-build/test-stubs/windows/windows-acceptance.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: WindowsAcceptanceTest
- gradle_task: :geode-assembly:acceptanceTest
- artifact_slug: windows-acceptancetestfiles
- call_stack_timeout: 1800
- tags: windows-acceptance-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml
deleted file mode 100644
index d719bbb..0000000
--- a/ci/pipelines/geode-build/test-stubs/windows/windows-distributed.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: WindowsGfshDistributedTest
- gradle_task: distributedTest
- artifact_slug: windows-gfshdistributedtest
- test_category: org.apache.geode.test.junit.categories.GfshTest
- call_stack_timeout: 1800
- tags: windows-distributed-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml
deleted file mode 100644
index 2c3b785..0000000
--- a/ci/pipelines/geode-build/test-stubs/windows/windows-integration.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: WindowsIntegrationTest
- gradle_task: integrationTest
- artifact_slug: windows-integrationtestfiles
- call_stack_timeout: 1800
- tags: windows-integration-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml b/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml
deleted file mode 100644
index 2b704f3..0000000
--- a/ci/pipelines/geode-build/test-stubs/windows/windows-unit.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: WindowsUnitTest
- gradle_task: test
- artifact_slug: windows-unittestfiles
- call_stack_timeout: 1800
- tags: windows-unit-tests
\ No newline at end of file
diff --git a/ci/pipelines/geode-build/test-template.yml b/ci/pipelines/geode-build/test-template.yml
deleted file mode 100644
index 6cb8779..0000000
--- a/ci/pipelines/geode-build/test-template.yml
+++ /dev/null
@@ -1,136 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-
-image_resource: &alpine-tools-image
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
- tag: latest
-
-jobs:
-- name: (( grab metadata.job.name ))
- serial: false
- public: true
- plan:
- - do:
- - get: geode-ci
- - aggregate:
- - get: geode
- passed: [Build]
- - get: geode-build-version
- passed: [Build]
- trigger: true
- - do:
- - put: concourse-metadata-resource
- - task: start_instance
- timeout: 15m
- attempts: 100
- config:
- inputs:
- - name: concourse-metadata-resource
- - name: geode-ci
- outputs:
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- params:
- CPUS: (( grab metadata.job.cpus ))
- RAM: (( grab metadata.job.ram ))
- GEODE_BRANCH: (( grab metadata.geode-build-branch ))
- GEODE_FORK: (( grab metadata.geode-fork ))
- run:
- path: geode-ci/ci/scripts/start_instance.sh
- - task: rsync_code_up
- timeout: 5m
- config:
- inputs:
- - name: geode-ci
- - name: geode
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/rsync_code_up.sh
- - task: execute_tests
- timeout: (( grab metadata.job.timeout ))
- config:
- inputs:
- - name: geode-ci
- - name: geode
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
- DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
- CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
- GRADLE_TASK: (( grab metadata.job.gradle_task ))
- GRADLE_TASK_OPTIONS: (( grab metadata.job.gradle_task_options || "" ))
- ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
- run:
- path: geode-ci/ci/scripts/execute_tests.sh
- ensure:
- do:
- - task: rsync_code_down
- timeout: 5m
- config:
- inputs:
- - name: geode-ci
- - name: instance-data
- outputs:
- - name: geode-results
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/rsync_code_down.sh
- ensure:
- do:
- - aggregate:
- - task: archive_results
- timeout: 1h
- config:
- inputs:
- - name: concourse-metadata-resource
- - name: geode-ci
- - name: geode-build-version
- - name: geode-results
- platform: linux
- image_resource: *alpine-tools-image
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- GRADLE_TASK: (( grab metadata.job.gradle_task ))
- ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
- run:
- path: geode-ci/ci/scripts/archive_results.sh
- - task: stop_instance
- timeout: 1h
- config:
- inputs:
- - name: geode-ci
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/stop_instance.sh
diff --git a/ci/pipelines/geode-build/windows-test-template.yml b/ci/pipelines/geode-build/windows-test-template.yml
deleted file mode 100644
index 15438e8..0000000
--- a/ci/pipelines/geode-build/windows-test-template.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-
-jobs:
-- name: (( grab metadata.job.name ))
- serial: true
- public: true
- plan:
- - get: geode-ci
- - aggregate:
- - get: geode
- passed: [Build]
- - get: geode-build-version
- passed: [Build]
- trigger: true
- - task: cleanup-java-processes
- tags: [(( grab metadata.job.tags ))]
- config:
- platform: windows
- run:
- path: powershell
- args:
- - -command
- - |
- gwmi win32_process -filter 'name = "java.exe"' | select commandline | format-list
- kill -name java -force
- exit 0
- - task: execute_tests
- timeout: 4h
- tags: [(( grab metadata.job.tags ))]
- config:
- inputs:
- - name: geode-ci
- - name: geode
- outputs:
- - name: built-geode
- platform: windows
- params:
- CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
- DUNIT_PARALLEL_FORKS: 0
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- PARALLEL_DUNIT: false
- PUBLIC_BUCKET: ((!public-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- JAVA_HOME: "C:\\progra~1\\java\\jdk1.8.0_181"
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-run.sh
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
- - (( grab metadata.job.test_category || "" ))
- ensure:
- aggregate:
- - task: archive_results
- timeout: 1h
- tags: [(( grab metadata.job.tags ))]
- config:
- inputs:
- - name: geode-ci
- - name: geode-build-version
- - name: built-geode
- platform: windows
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- path: bash
- args:
- - geode-ci/ci/scripts/windows/test-archive.sh
- - (( grab metadata.job.gradle_task ))
- - (( grab metadata.job.artifact_slug ))
-
diff --git a/ci/pipelines/images/deploy_images_pipeline.sh b/ci/pipelines/images/deploy_images_pipeline.sh
index bcbeb50..dd544fe 100755
--- a/ci/pipelines/images/deploy_images_pipeline.sh
+++ b/ci/pipelines/images/deploy_images_pipeline.sh
@@ -25,16 +25,6 @@ done
SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
GEODEBUILDDIR="${SCRIPTDIR}/../geode-build"
-if ! [ -x "$(command -v spruce)" ]; then
- echo "Spruce must be installed for pipeline deployment to work."
- echo "For macos: 'brew tap starkandwayne/cf; brew install spruce'"
- echo "For Ubuntu: follow the instructions at https://github.com/geofffranks/spruce"
- echo ""
- exit 1
-else
- SPRUCE=$(which spruce || true)
-fi
-
set -e
if [ -z "${GEODE_BRANCH}" ]; then
diff --git a/ci/pipelines/jinja.variables.yml b/ci/pipelines/jinja.variables.yml
new file mode 100644
index 0000000..fd4789e
--- /dev/null
+++ b/ci/pipelines/jinja.variables.yml
@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+repository:
+ project: geode
+
+platforms:
+- name: "Linux"
+ prefix: ""
+- name: "Windows"
+ prefix: "Windows"
+
+tests:
+- name: "Acceptance"
+ CPUS: "8"
+ RAM: "12"
+ ARTIFACT_SLUG: acceptancetestfiles
+ CALL_STACK_TIMEOUT: "1800"
+ DUNIT_PARALLEL_FORKS: "7"
+ GRADLE_TASK: :geode-assembly:acceptanceTest geode-connectors:acceptanceTest
+ execute_test_timeout: 45m
+ PARALLEL_DUNIT: "true"
+- name: "Distributed"
+ CPUS: "96"
+ RAM: "180"
+ ARTIFACT_SLUG: distributedtestfiles
+ CALL_STACK_TIMEOUT: "7200"
+ DUNIT_PARALLEL_FORKS: "24"
+ GRADLE_TASK: distributedTest
+ execute_test_timeout: 2h15m
+ PARALLEL_DUNIT: "true"
+- name: "Integration"
+ CPUS: "96"
+ RAM: "90"
+ ARTIFACT_SLUG: integrationtestfiles
+ CALL_STACK_TIMEOUT: "1500"
+ DUNIT_PARALLEL_FORKS: "48"
+ GRADLE_TASK: integrationTest
+ execute_test_timeout: 40m
+ PARALLEL_DUNIT: "true"
+- name: "Upgrade"
+ CPUS: "96"
+ RAM: "160"
+ ARTIFACT_SLUG: upgradetestfiles
+ CALL_STACK_TIMEOUT: "3000"
+ DUNIT_PARALLEL_FORKS: "48"
+ GRADLE_TASK: upgradeTest
+ execute_test_timeout: 1h
+ PARALLEL_DUNIT: "true"
+- name: "StressNew"
+ CPUS: "96"
+ RAM: "210"
+ ARTIFACT_SLUG: stressnewtestfiles
+ CALL_STACK_TIMEOUT: "7200"
+ DUNIT_PARALLEL_FORKS: "24"
+ GRADLE_TASK: repeatTest
+ execute_test_timeout: 2h15m
+ PARALLEL_DUNIT: "true"
+
+windowstests:
+- name: "WindowsAcceptance"
+ tags: "acceptance"
+ CALL_STACK_TIMEOUT: "1800"
+ DUNIT_PARALLEL_FORKS: "0"
+ PARALLEL_DUNIT: "false"
+ target_arg: :geode-assembly:acceptanceTest
+ dir_arg: windows-acceptancetestfiles
+ package_arg: ""
+- name: "WindowsGfshDistributed"
+ tags: "distributed"
+ CALL_STACK_TIMEOUT: "1800"
+ DUNIT_PARALLEL_FORKS: "0"
+ PARALLEL_DUNIT: "false"
+ target_arg: distributedTest
+ dir_arg: windows-gfshdistributedtest
+ package_arg: org.apache.geode.test.junit.categories.GfshTest
+- name: "WindowsIntegration"
+ tags: "integration"
+ CALL_STACK_TIMEOUT: "1800"
+ DUNIT_PARALLEL_FORKS: "0"
+ PARALLEL_DUNIT: "false"
+ target_arg: integrationTest
+ dir_arg: windows-integrationtestfiles
+ package_arg: ""
+- name: "WindowsUnit"
+ tags: "unit"
+ CALL_STACK_TIMEOUT: "1800"
+ DUNIT_PARALLEL_FORKS: "0"
+ PARALLEL_DUNIT: "false"
+ target_arg: test
+ dir_arg: windows-unittestfiles
+ package_arg: ""
diff --git a/ci/pipelines/pull-request/base.yml b/ci/pipelines/pull-request/base.yml
deleted file mode 100644
index a6e5d88..0000000
--- a/ci/pipelines/pull-request/base.yml
+++ /dev/null
@@ -1,135 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
----
-
-resource_types:
- - name: gcs-resource
- type: docker-image
- source:
- repository: frodenas/gcs-resource
- - name: pull-request
- type: docker-image
- source:
- repository: jtarchie/pr
- - name: concourse-metadata-resource
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))concourse-metadata-resource
- tag: latest
-
-resources:
-- name: docker-geode-build-image
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))((!docker-image-name))
- tag: latest
-- name: geode
- type: pull-request
- source:
- access_token: ((!github-pr-access-token))
- repo: (( concat metadata.geode-fork "/geode" ))
- base: develop
- ignore_paths:
- - geode-docs/*
- - geode-book/*
-- name: geode-ci
- type: git
- source:
- depth: 1
- uri: https://github.com/apache/geode.git
- branch: develop
- paths:
- - ci/pipelines/geode-build/*
- - ci/scripts/*
-- name: concourse-metadata-resource
- type: concourse-metadata-resource
-
-groups:
-- name: main
- jobs:
- - Build
- - DistributedTest
- - AcceptanceTest
- - IntegrationTest
- - UpgradeTest
- - StressNewTests
-
-jobs:
-- name: Build
- serial: false
- public: true
- plan:
- - aggregate:
- - get: geode
- trigger: true
- version: every
- params:
- fetch_merge: true
- git:
- depth: 100
- - get: geode-ci
- - get: docker-geode-build-image
- params:
- rootfs: true
- # Unmerged pull request SHA, for writing status into GitHub
- - get: geode-unmerged-request
- resource: geode
- version: every
- params:
- fetch_merge: false
- - aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: pending
-
- - task: build
- image: docker-geode-build-image
- config:
- platform: linux
- inputs:
- - name: geode
- - name: geode-ci
- outputs:
- - name: built-geode
- - name: results
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- run:
- path: geode-ci/ci/scripts/build.sh
- on_failure:
- aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- status: failure
- context: $BUILD_JOB_NAME
- on_success:
- aggregate:
- - put: geode
- params:
- path: geode-unmerged-request
- status: success
- context: $BUILD_JOB_NAME
diff --git a/ci/pipelines/pull-request/deploy_pr_pipeline.sh b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
index 9237711..ff32ed9 100755
--- a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
+++ b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
@@ -26,15 +26,14 @@ SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
GEODEBUILDDIR="${SCRIPTDIR}/../geode-build"
GEODE_FORK=${GEODE_FORK:-apache}
-if ! [ -x "$(command -v spruce)" ]; then
- echo "Spruce must be installed for pipeline deployment to work."
- echo "For macos: 'brew tap starkandwayne/cf; brew install spruce'"
- echo "For Ubuntu: follow the instructions at https://github.com/geofffranks/spruce"
+for cmd in Jinja2 PyYAML; do
+ if ! [[ $(pip3 list |grep ${cmd}) ]]; then
+ echo "${cmd} must be installed for pipeline deployment to work."
+ echo " 'pip3 install ${cmd}'"
echo ""
exit 1
-else
- SPRUCE=$(which spruce || true)
-fi
+ fi
+done
set -e
@@ -59,48 +58,10 @@ chmod +x ${BIN_DIR}/fly
PATH=${PATH}:${BIN_DIR}
-for i in ${GEODEBUILDDIR}/test-stubs/*.yml; do
- X=$(basename $i)
- echo "Merging ${i} into ${TMP_DIR}/${X}"
- ${SPRUCE} merge --prune metadata \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}"; \
- echo " ") \
- ${SCRIPTDIR}/pr-template.yml \
- ${i} > ${TMP_DIR}/${X}
-done
-
-for i in ${SCRIPTDIR}/test-stubs/*.yml; do
- X=pull-request-$(basename $i)
- echo "Merging ${i} into ${TMP_DIR}/${X}"
- ${SPRUCE} merge --prune metadata \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}"; \
- echo " ") \
- ${SCRIPTDIR}/pr-template.yml \
- ${i} > ${TMP_DIR}/${X}
-done
-
-echo "Spruce branch-name into resources"
-${SPRUCE} merge --prune metadata \
- ${SCRIPTDIR}/base.yml \
- <(echo "metadata:"; \
- echo " geode-build-branch: ${GEODE_BRANCH}"; \
- echo " geode-fork: ${GEODE_FORK}"; \
- echo " ") \
- ${TMP_DIR}/*.yml > ${TMP_DIR}/final.yml
-
-
TARGET="geode"
TEAM=${CONCOURSE_TEAM:-main}
-#if [[ "${GEODE_BRANCH}" == "develop" ]] || [[ ${GEODE_BRANCH} =~ ^release/* ]]; then
-# TEAM="main"
-#fi
-
if [[ "${GEODE_FORK}" == "apache" ]]; then
PIPELINE_PREFIX=""
DOCKER_IMAGE_PREFIX=""
@@ -109,11 +70,23 @@ else
DOCKER_IMAGE_PREFIX=${PIPELINE_PREFIX}
fi
-fly login -t ${TARGET} -n ${TEAM} -c https://concourse.apachegeode-ci.info -u ${CONCOURSE_USERNAME} -p ${CONCOURSE_PASSWORD}
-fly -t ${TARGET} set-pipeline \
- --non-interactive \
- --pipeline pr-${SANITIZED_GEODE_BRANCH} \
- --config ${TMP_DIR}/final.yml \
- --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
- --var concourse-team=${TEAM}
+pushd ${SCRIPTDIR} 2>&1 > /dev/null
+ # Template and output share a directory with this script, but variables are shared in the parent directory.
+ python3 ../render.py jinja.template.yml ../jinja.variables.yml generated-pipeline.yml || exit 1
+
+ fly login -t ${TARGET} \
+ -n ${TEAM} \
+ -c https://concourse.apachegeode-ci.info \
+ -u ${CONCOURSE_USERNAME} \
+ -p ${CONCOURSE_PASSWORD}
+
+ fly -t ${TARGET} set-pipeline \
+ --non-interactive \
+ --pipeline pr-${SANITIZED_GEODE_BRANCH} \
+ --config generated-pipeline.yml \
+ --var docker-image-prefix=${DOCKER_IMAGE_PREFIX} \
+ --var concourse-team=${TEAM}
+
+popd 2>&1 > /dev/null
+
diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
new file mode 100644
index 0000000..860cdec
--- /dev/null
+++ b/ci/pipelines/pull-request/jinja.template.yml
@@ -0,0 +1,262 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{% from 'shared_jinja.yml' import alpine_tools_config with context %}
+
+groups:
+- name: main
+ jobs:
+ - Build
+{%- for test in tests %}
+ - {{test.name}}Test
+{%- endfor %}
+resources:
+- name: docker-geode-build-image
+ type: docker-image
+ source:
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-((!docker-image-name))
+ tag: latest
+ username: ((!docker-username))
+- name: geode
+ type: pull-request
+ source:
+ access_token: ((!github-pr-access-token))
+ base: develop
+ ignore_paths:
+ - geode-docs/*
+ - geode-book/*
+ repo: {{repository.fork}}/geode
+- name: geode-ci
+ type: git
+ source:
+ branch: develop
+ depth: 1
+ paths:
+ - ci/pipelines/geode-build/*
+ - ci/scripts/*
+ uri: https://github.com/apache/geode.git
+- name: concourse-metadata-resource
+ type: concourse-metadata-resource
+ source: {}
+resource_types:
+- name: gcs-resource
+ type: docker-image
+ source:
+ repository: frodenas/gcs-resource
+- name: pull-request
+ type: docker-image
+ source:
+ repository: jtarchie/pr
+- name: concourse-metadata-resource
+ type: docker-image
+ source:
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-concourse-metadata-resource
+ tag: latest
+ username: ((!docker-username))
+jobs:
+- name: Build
+ public: true
+ plan:
+ - aggregate:
+ - get: geode
+ trigger: true
+ params:
+ fetch_merge: true
+ git:
+ depth: 100
+ version: every
+ - get: geode-ci
+ - get: docker-geode-build-image
+ params:
+ rootfs: true
+ - get: geode-unmerged-request
+ resource: geode
+ params:
+ fetch_merge: false
+ version: every
+ - aggregate:
+ - put: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: pending
+ - task: build
+ config:
+ platform: linux
+ params:
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: geode-ci/ci/scripts/build.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ outputs:
+ - name: built-geode
+ - name: results
+ image: docker-geode-build-image
+ on_failure:
+ aggregate:
+ - put: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: failure
+ on_success:
+ aggregate:
+ - put: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: success
+{% for test in tests %}
+- name: {{test.name}}Test
+ public: true
+ plan:
+ - do:
+ - aggregate:
+ - get: geode
+ trigger: true
+ params:
+ fetch_merge: true
+ git:
+ depth: 100
+ version: every
+ - get: geode-ci
+ - aggregate:
+ - do:
+ - get: geode-unmerged-request
+ resource: geode
+ params:
+ fetch_merge: false
+ version: every
+ - put: pull-request-job-pending
+ resource: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: pending
+ - do:
+ - put: concourse-metadata-resource
+ - task: start_instance
+ {{- alpine_tools_config()|indent(10) }}
+ params:
+ CPUS: {{test.CPUS}}
+ GEODE_BRANCH: {{repository.branch}}
+ GEODE_FORK: {{repository.fork}}
+ RAM: {{test.RAM}}
+ run:
+ path: geode-ci/ci/scripts/start_instance.sh
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode
+ - name: geode-ci
+ outputs:
+ - name: instance-data
+ timeout: 15m
+ attempts: 100
+ - task: rsync_code_up
+ {{- alpine_tools_config()|indent(6) }}
+ run:
+ path: geode-ci/ci/scripts/rsync_code_up.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ - name: instance-data
+ timeout: 5m
+ - task: execute_tests
+ {{- alpine_tools_config()|indent(6) }}
+ params:
+ ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}
+ CALL_STACK_TIMEOUT: {{test.CALL_STACK_TIMEOUT}}
+ DUNIT_PARALLEL_FORKS: {{test.DUNIT_PARALLEL_FORKS}}
+ GRADLE_TASK: {{test.GRADLE_TASK}}
+ GRADLE_TASK_OPTIONS: ""
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ {%- if test.name=="StressNew" %}
+ path: geode/ci/scripts/repeat-new-tests.sh
+ {%- else %}
+ path: geode-ci/ci/scripts/execute_tests.sh
+ {%- endif %}
+ inputs:
+ - name: geode
+ - name: geode-ci
+ - name: instance-data
+ timeout: {{test.execute_test_timeout}}
+ on_failure:
+ do:
+ - put: pull-request-job-failure
+ resource: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: failure
+ ensure:
+ do:
+ - task: rsync_code_down
+ {{- alpine_tools_config()|indent(8) }}
+ run:
+ path: geode-ci/ci/scripts/rsync_code_down.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ - name: instance-data
+ outputs:
+ - name: geode-results
+ timeout: 5m
+ ensure:
+ aggregate:
+ - task: archive-results
+ {{- alpine_tools_config()|indent(10) }}
+ params:
+ ARTIFACT_SLUG: {{test.ARTIFACT_SLUG}}
+ GRADLE_TASK: {{test.GRADLE_TASK}}
+ MAINTENANCE_VERSION: {{repository.branch}}
+ PUBLIC_BUCKET: ((!public-bucket))
+ SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ run:
+ path: geode-ci/ci/scripts/archive_results.sh
+ inputs:
+ - name: concourse-metadata-resource
+ - name: geode
+ - name: geode-ci
+ - name: geode-results
+ - task: stop_instance
+ {{- alpine_tools_config()|indent(10) }}
+ run:
+ path: geode-ci/ci/scripts/stop_instance.sh
+ inputs:
+ - name: geode
+ - name: geode-ci
+ - name: instance-data
+ timeout: 1h
+ on_success:
+ do:
+ - put: pull-request-job-success
+ resource: geode
+ params:
+ context: $BUILD_JOB_NAME
+ path: geode-unmerged-request
+ status: success
+{% endfor %}
diff --git a/ci/pipelines/pull-request/pr-template.yml b/ci/pipelines/pull-request/pr-template.yml
deleted file mode 100644
index 722bee8..0000000
--- a/ci/pipelines/pull-request/pr-template.yml
+++ /dev/null
@@ -1,172 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
----
-
-image_resource: &alpine-tools-image
- type: docker-image
- source:
- username: ((!docker-username))
- password: ((!docker-password))
- repository: gcr.io/apachegeode-ci/((!docker-image-prefix))alpine-tools
- tag: latest
-
-jobs:
-- name: (( grab metadata.job.name ))
- serial: false
- public: true
- plan:
- - do:
- - aggregate:
- # Merged pull request SHA, for building
- - get: geode
- trigger: true
- version: every
- params:
- fetch_merge: true
- git:
- depth: 100
- - get: geode-ci
- - aggregate:
- # Unmerged pull request SHA, for writing status into GitHub
- - do:
- - get: geode-unmerged-request
- resource: geode
- version: every
- params:
- fetch_merge: false
- - put: pull-request-job-pending
- resource: geode
- params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: pending
- # Start heavy lifter instance
- - do:
- - put: concourse-metadata-resource
- - task: start_instance
- timeout: 15m
- attempts: 100
- config:
- inputs:
- - name: concourse-metadata-resource
- - name: geode
- - name: geode-ci
- outputs:
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- params:
- CPUS: (( grab metadata.job.cpus ))
- RAM: (( grab metadata.job.ram ))
- GEODE_BRANCH: (( grab metadata.geode-build-branch ))
- GEODE_FORK: (( grab metadata.geode-fork ))
- run:
- path: geode-ci/ci/scripts/start_instance.sh
- - task: rsync_code_up
- timeout: 5m
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/rsync_code_up.sh
- - task: execute_tests
- timeout: (( grab metadata.job.timeout ))
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- PARALLEL_DUNIT: (( grab metadata.job.dunit.parallel ))
- DUNIT_PARALLEL_FORKS: (( grab metadata.job.dunit.forks ))
- CALL_STACK_TIMEOUT: (( grab metadata.job.call_stack_timeout ))
- GRADLE_TASK: (( grab metadata.job.gradle_task ))
- GRADLE_TASK_OPTIONS: (( grab metadata.job.gradle_task_options || "" ))
- ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
- run:
- path: (( grab metadata.job.execute_command || "geode/ci/scripts/execute_tests.sh" ))
- on_success:
- do:
- - put: pull-request-job-success
- resource: geode
- params:
- path: geode-unmerged-request
- context: $BUILD_JOB_NAME
- status: success
- on_failure:
- do:
- - put: pull-request-job-failure
- resource: geode
- params:
- path: geode-unmerged-request
- status: failure
- context: $BUILD_JOB_NAME
- ensure:
- do:
- - task: rsync_code_down
- timeout: 5m
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: instance-data
- outputs:
- - name: geode-results
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/rsync_code_down.sh
- ensure:
- aggregate:
- - task: archive-results
- config:
- inputs:
- - name: concourse-metadata-resource
- - name: geode
- - name: geode-ci
- - name: geode-results
- platform: linux
- image_resource: *alpine-tools-image
- params:
- MAINTENANCE_VERSION: (( grab metadata.geode-build-branch ))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- PUBLIC_BUCKET: ((!public-bucket))
- GRADLE_TASK: (( grab metadata.job.gradle_task ))
- ARTIFACT_SLUG: (( grab metadata.job.artifact_slug ))
- run:
- path: geode-ci/ci/scripts/archive_results.sh
- - task: stop_instance
- timeout: 1h
- config:
- inputs:
- - name: geode
- - name: geode-ci
- - name: instance-data
- platform: linux
- image_resource: *alpine-tools-image
- run:
- path: geode-ci/ci/scripts/stop_instance.sh
-
diff --git a/ci/pipelines/pull-request/test-stubs/newtests.yml b/ci/pipelines/pull-request/test-stubs/newtests.yml
deleted file mode 100644
index 33fd689..0000000
--- a/ci/pipelines/pull-request/test-stubs/newtests.yml
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-metadata:
- job:
- name: StressNewTests
- gradle_task: repeatTest
- artifact_slug: stressnewtestfiles
- execute_command: geode/ci/scripts/repeat-new-tests.sh
- dunit:
- parallel: true
-# max number of docker containers to run, generally cpus/2
- forks: 24
- cpus: 96
-# specified in Gigabytes.
- ram: 210
-# specified in seconds
- call_stack_timeout: 7200
- timeout: 2h15m
- size: []
diff --git a/ci/pipelines/render.py b/ci/pipelines/render.py
new file mode 100755
index 0000000..2754462
--- /dev/null
+++ b/ci/pipelines/render.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import argparse
+import logging
+
+import yaml
+from jinja2 import Environment, FileSystemLoader, Undefined
+import jinja2.exceptions
+import os
+
+def main(template_file, variables_file, output_file):
+ # TODO Delete this TODO
+ # Also TODO: Make the FileSystemLoader accept the script-dir, current-dir, and commons-dir more sensibly.
+ env = Environment(loader=FileSystemLoader(['.', '..']), undefined=RaiseExceptionIfUndefined)
+ template = env.get_template(template_file)
+
+ with open(variables_file, 'r') as variablesFromYml:
+ variables = yaml.load(variablesFromYml)
+
+ variables['repository']['branch'] = os.environ['GEODE_BRANCH']
+ variables['repository']['fork'] = os.environ['GEODE_FORK']
+
+ logging.debug(f"Variables = {variables}")
+
+ logging.info(template.render(variables))
+ with open(output_file, 'w') as pipeline_file:
+ pipeline_file.write(template.render(variables))
+
+
+class RaiseExceptionIfUndefined(Undefined):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ def __str__(self):
+ raise jinja2.exceptions.UndefinedError("Interpolating undefined variables has been disabled.")
+
+ def __iter__(self):
+ raise jinja2.exceptions.UndefinedError("Interpolating undefined variables has been disabled.")
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument("template", help="Jinja template file.")
+ parser.add_argument("variables", help="Jinja variables file.")
+ parser.add_argument("output", help="Output target.")
+ parser.add_argument("--debug", help="It's debug. If you have to ask, you'll never know.", action="store_true")
+
+ _args = parser.parse_args()
+
+ if _args.debug:
+ logging.getLogger().setLevel(logging.DEBUG)
+
+ logging.debug(f"cwd: {os.getcwd()}")
+
+ main(_args.template, _args.variables, _args.output)
+
diff --git a/ci/pipelines/shared_jinja.yml b/ci/pipelines/shared_jinja.yml
new file mode 100644
index 0000000..3aff042
--- /dev/null
+++ b/ci/pipelines/shared_jinja.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{%- macro alpine_tools_config() %}
+{{- docker_config() }}
+{%- endmacro %}
+
+{%- macro docker_config(repository_url) %}
+config:
+ platform: linux
+ image_resource:
+ type: docker-image
+ source:
+ username: ((!docker-username))
+ password: ((!docker-password))
+ repository: gcr.io/apachegeode-ci/{{repository.fork}}-{{repository.branch}}-alpine-tools
+ tag: latest
+{%- endmacro %}