You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by sm...@apache.org on 2018/10/24 22:47:12 UTC
[geode] branch develop updated: GEODE-5864 Refactor pipeline
deployment. (#2707)
This is an automated email from the ASF dual-hosted git repository.
smgoller pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/develop by this push:
new 7d9858e GEODE-5864 Refactor pipeline deployment. (#2707)
7d9858e is described below
commit 7d9858ee882f6bf06a7658b39911aac4fe72e4e1
Author: Sean Goller <se...@goller.net>
AuthorDate: Wed Oct 24 15:47:01 2018 -0700
GEODE-5864 Refactor pipeline deployment. (#2707)
* deploy_meta.sh now looks at the file meta.properties for defaults
* Users should create meta.properties.local file and override values
there. A common customization is to override GEODE_FORK with your
personal geode repository.
* GRADLE_GLOBAL_ARGS is a string of gradle arguments that will be passed
to every invocation of gradle.
* Change "start_instance.sh" to "create_instance.sh"
* Change "stop_instance.sh" to "delete_instance.sh"
* Remove unnecessary gcloud delete invocation
* Add on_failure block to rsync_code_up job to delete the instance.
* Make on_failure call a do block to be consistent with other on_failures.
* Add system property for excluding individual test classes.
* Adjust excludeTests to handle multiple tests
* Add maven snapshot bucket for storing published artifacts.
* Make GCP_PROJECT a property instead of deriving it.
* Make deploy_meta.sh more user-friendly by displaying values and
offering the option to abort.
* Forks should no longer be hard-coded.
* Remove UpdatePassingRef job.
* Remove unused scripts.
Co-authored-by: Sean Goller <sg...@pivotal.io>
Co-authored-by: Jacob Barrett <jb...@pivotal.io>
Co-authored-by: Dick Cavender <dc...@pivotal.io>
---
.gitignore | 1 +
.../windows-packer.json | 2 +-
ci/pipelines/geode-build/deploy_pipeline.sh | 2 +
ci/pipelines/geode-build/jinja.template.yml | 76 +++++++++-----------
ci/pipelines/meta/README.md | 80 +++++++++++++---------
ci/pipelines/meta/deploy_meta.sh | 49 ++++++++-----
ci/pipelines/meta/jinja.template.yml | 3 +
.../meta/meta.properties} | 34 +++------
ci/pipelines/pull-request/deploy_pr_pipeline.sh | 1 +
ci/pipelines/pull-request/jinja.template.yml | 11 +--
ci/scripts/build-examples.sh | 26 -------
.../{start_instance.sh => create_instance.sh} | 3 -
.../{stop_instance.sh => delete_instance.sh} | 0
ci/scripts/execute_build.sh | 1 +
ci/scripts/execute_publish.sh | 2 +-
ci/scripts/execute_tests.sh | 3 +-
ci/scripts/update-passing-ref.sh | 31 ---------
gradle/publish.gradle | 6 +-
gradle/rat.gradle | 3 +
gradle/test.gradle | 6 ++
20 files changed, 154 insertions(+), 186 deletions(-)
diff --git a/.gitignore b/.gitignore
index fc375df..f2987c1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ geode-pulse/screenshots/
.git-together
bin/
.metadata/
+/ci/pipelines/meta/meta.properties.local
diff --git a/ci/images/google-windows-geode-builder/windows-packer.json b/ci/images/google-windows-geode-builder/windows-packer.json
index f73732d..a185f08 100644
--- a/ci/images/google-windows-geode-builder/windows-packer.json
+++ b/ci/images/google-windows-geode-builder/windows-packer.json
@@ -43,7 +43,7 @@
"Invoke-WebRequest https://chocolatey.org/install.ps1 -UseBasicParsing | Invoke-Expression",
"choco install -y git rsync openjdk",
- "Move-Item \"C:\\Program Files\\OpenJDK\\jdk-11\" c:\\java11",
+ "Move-Item \"C:\\Program Files\\OpenJDK\\jdk-11*\" c:\\java11",
"choco install -y jdk8 -params 'installdir=c:\\\\java8tmp;source=false'",
"Move-Item \"C:\\java8tmp\" c:\\java8",
"choco install -y openssh /SSHServerFeature",
diff --git a/ci/pipelines/geode-build/deploy_pipeline.sh b/ci/pipelines/geode-build/deploy_pipeline.sh
index 0cce6ae..ba4a603 100755
--- a/ci/pipelines/geode-build/deploy_pipeline.sh
+++ b/ci/pipelines/geode-build/deploy_pipeline.sh
@@ -71,6 +71,8 @@ public-pipelines: ${PUBLIC_PIPELINES}
gcp-project: ${GCP_PROJECT}
version-bucket: ${VERSION_BUCKET}
artifact-bucket: ${ARTIFACT_BUCKET}
+gradle-global-args: ${GRADLE_GLOBAL_ARGS}
+maven-snapshot-bucket: ${MAVEN_SNAPSHOT_BUCKET}
YML
diff --git a/ci/pipelines/geode-build/jinja.template.yml b/ci/pipelines/geode-build/jinja.template.yml
index 5195a83..8d8f8d1 100644
--- a/ci/pipelines/geode-build/jinja.template.yml
+++ b/ci/pipelines/geode-build/jinja.template.yml
@@ -71,8 +71,9 @@ DUNIT_PARALLEL_FORKS: {{ test.DUNIT_PARALLEL_FORKS }}
MAINTENANCE_VERSION: ((geode-build-branch ))
PARALLEL_DUNIT: {{ test.PARALLEL_DUNIT }}
PARALLEL_GRADLE: {{ test.PARALLEL_GRADLE }}
-ARTIFACT_BUCKET: ((!artifact-bucket))
-SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ARTIFACT_BUCKET: ((artifact-bucket))
+SERVICE_ACCOUNT: ((concourse-gcp-account))
+GRADLE_GLOBAL_ARGS: ((gradle-global-args))
{%- endmacro %}
{% macro all_gating_jobs(test) %}
@@ -88,8 +89,7 @@ groups:
jobs:
- {{ build_test.name }}
{{ all_gating_jobs() | indent(2) }}
- {%- if repository.sanitized_fork == "apache" %}
- - UpdatePassingRef
+ {%- if repository.sanitized_fork == repository.upstream_fork %}
- PublishArtifacts
{%- endif %}
- name: complete
@@ -100,8 +100,7 @@ groups:
- {{test.name}}Test{{java_test_version.name}}
{%- endfor -%}
{%- endfor -%}
- {%- if repository.sanitized_fork == "apache" %}
- - UpdatePassingRef
+ {%- if repository.sanitized_fork == repository.upstream_fork %}
- PublishArtifacts
{%- endif %}
- name: linux
@@ -130,7 +129,7 @@ groups:
{%- endfor %}
resources:
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
- name: send-notification-email
type: email
source:
@@ -184,7 +183,7 @@ resource_types:
type: docker-image
source:
repository: frodenas/gcs-resource
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
- name: email
type: docker-image
source:
@@ -204,14 +203,14 @@ jobs:
pre: build
- do:
- put: concourse-metadata-resource
- - task: start_instance
+ - task: create_instance
{{ alpine_tools_config()|indent(8) }}
params:
{{ common_instance_params(build_test) | indent(12) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
run:
- path: geode-ci/ci/scripts/start_instance.sh
+ path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
@@ -228,6 +227,16 @@ jobs:
- name: geode
- name: instance-data
timeout: 5m
+ on_failure:
+ do:
+ - task: delete_instance
+ {{ alpine_tools_config()|indent(8) }}
+ run:
+ path: geode-ci/ci/scripts/delete_instance.sh
+ inputs:
+ - name: geode-ci
+ - name: instance-data
+ timeout: 1h
- task: build
{{ alpine_tools_config()|indent(4) }}
params:
@@ -244,7 +253,7 @@ jobs:
outputs:
- name: built-geode
- name: results
-{% if repository.sanitized_fork == "apache" %}
+{% if repository.sanitized_fork == repository.upstream_fork %}
on_failure:
aggregate:
- put: send-notification-email
@@ -289,38 +298,16 @@ jobs:
- name: geode-build-version
- name: geode-results
timeout: 1h
- - task: stop_instance
+ - task: delete_instance
{{ alpine_tools_config()|indent(12) }}
run:
- path: geode-ci/ci/scripts/stop_instance.sh
+ path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
timeout: 1h
-{% if repository.sanitized_fork == "apache" %}
-- name: UpdatePassingRef
- public: true
- serial: true
- plan:
- - get: geode
- passed:
- {{ all_gating_jobs() | indent(4) }}
- trigger: true
- - get: geode-ci
- - task: updatepassingref
- {{ alpine_tools_config()|indent(4) }}
- params:
- MAINTENANCE_VERSION: ((geode-build-branch))
- ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
- run:
- path: geode-ci/ci/scripts/update-passing-ref.sh
- inputs:
- - name: geode
- - name: geode-ci
- outputs:
- - name: results
+{% if repository.sanitized_fork == repository.upstream_fork %}
- name: PublishArtifacts
public: true
plan:
@@ -335,14 +322,14 @@ jobs:
pre: build
- do:
- put: concourse-metadata-resource
- - task: start_instance
+ - task: create_instance
{{ alpine_tools_config()|indent(8) }}
params:
{{ common_instance_params(publish_artifacts) | indent(12) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
run:
- path: geode-ci/ci/scripts/start_instance.sh
+ path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
@@ -366,6 +353,7 @@ jobs:
ARTIFACT_BUCKET: ((artifact-bucket))
SERVICE_ACCOUNT: ((!concourse-gcp-account))
JAVA_BUILD_VERSION: {{ java_build_version.version }}
+ MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
run:
path: geode-ci/ci/scripts/execute_publish.sh
inputs:
@@ -375,10 +363,10 @@ jobs:
- name: geode-build-version
ensure:
do:
- - task: stop_instance
+ - task: delete_instance
{{ alpine_tools_config()|indent(8) }}
run:
- path: geode-ci/ci/scripts/stop_instance.sh
+ path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
@@ -400,14 +388,14 @@ jobs:
{{ plan_resource_gets() |indent(4) }}
- do:
- put: concourse-metadata-resource
- - task: start_instance
+ - task: create_instance
{{ alpine_tools_config()|indent(10) }}
params:
{{ common_instance_params(parameters) | indent(14) }}
GEODE_BRANCH: {{repository.branch}}
GEODE_FORK: {{repository.fork}}
run:
- path: geode-ci/ci/scripts/start_instance.sh
+ path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode-ci
@@ -475,10 +463,10 @@ jobs:
- name: geode-build-version
- name: geode-results
timeout: 1h
- - task: stop_instance
+ - task: delete_instance
{{ alpine_tools_config()|indent(12) }}
run:
- path: geode-ci/ci/scripts/stop_instance.sh
+ path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode-ci
- name: instance-data
diff --git a/ci/pipelines/meta/README.md b/ci/pipelines/meta/README.md
index ec1672b..fe9208c 100644
--- a/ci/pipelines/meta/README.md
+++ b/ci/pipelines/meta/README.md
@@ -6,39 +6,57 @@ number of deployment scenarios it handles.
##### Prerequisites
You must be logged into the appropriate Google Cloud Platform project for this to work.
-## Deploying to Production
-If you run `deploy_meta.sh` with no arguments, its default behavior is to deploy to the [production
+### Deployment Configuration
+In this directory is a file called `meta.properties`. This file contains variables that change
+how Geode is deployed. These values should be overriden by creating a file called
+`meta.properties.local` and changing value definitions in that file. Note that
+`meta.properties.local` is ignored by git because those types of changes are solely related to
+a specific deployment scenario.
+
+## Configuration Variables
+
+### UPSTREAM_FORK
+The UPSTREAM_FORK value indicates the GitHub user that hosts the master repository for Geode. Its
+default is `apache`, which results in a repository URL of `https://github.com/apache/geode.git`.
+
+### GEODE_FORK
+The GEODE_FORK value indicates the github username that hosts the repository under test. For
+example, if the GitHub user `molly` wishes to deploy a pipeline testing Geode from her fork, she
+would set this value to `molly`. This results in the repository URL
+`https://github.com/molly/geode.git`.
+
+### GEODE_REPO_NAME
+The GEODE_REPO_NAME value indicates the name of the repository under test. This defaults to `geode`.
+
+### CONCOURSE_HOST
+The CONCOURSE_HOST value indicates the hostname of the concourse infrastructure you wish to deploy
+the Geode CI pipeline to. The default value is `concourse.apachegeode-ci.info`. You might wish to
+change this value if you are hosting your own CI infrastructure for Geode.
+
+### ARTIFACT_BUCKET
+The ARTIFACT_BUCKET value indicates the Google Cloud Storage bucket name in which Concourse will
+store build artifacts and test results. The default value is `files.apachegeode-ci.info`.
+
+### PUBLIC
+The PUBLIC value indicates whether the deployed pipeline should be public or not. The default value
+is `true`, though in certain situations this is overridden, such as a production deployment.
+
+### REPOSITORY_PUBLIC
+The REPOSITORY_PUBLIC value indicates whether the repository under test is public or not. The
+default value is `true`. If this value is set to false, the pipeline will expect credentials to
+be available in the vault deployment associated with the concourse deployment.
+
+### GRADLE_GLOBAL_ARGS
+The GRADLE_GLOBAL_ARGS value is for passing a set of arguments to gradle wherever it is invoked in
+CI. The default value is an empty string.
+
+## Deploying
+Once you have made your configuration choices, simply run `deploy_meta.sh` with no arguments.
+its default behavior is to deploy to the [production
environment](https://concourse.apachegeode-ci.info). It will autodetect your branch and deploy the
appropriate pipeline. This will assume the branch is present in the
[upstream repository](https://github.com/apache/geode). The resulting meta pipeline will be named
`apache-<branch>-meta`.
-## Deploying from your fork to Geode Concourse
-If you wish to deploy the meta pipeline using your fork as the source, you must specify your github
-username as the first argument: `./deploy_meta.sh <github username>`. The resulting meta pipeline
-will be named `<github username>-<branch>-meta`. The default value for this argument is `apache`.
-
-## Specifying a different repository
-If your repository is named something other than `geode`, you must specify it as a second argument:
-`./deploy_meta.sh apache geode-testing`. There are no optional arguments, so if you need
-to provide an argument beyond the first, you must manually specify earlier arguments even if
-you are using default values.
-
-## Specifying a different upstream
-If your upstream repository owner is not `apache` (you forked a fork) then you must specify it as the third argument:
-`./deploy_meta.sh <github username> geode-testing not-apache`.
-
-## Specifying a different concourse infrastructure
-If you would like to deploy the pipeline to a concourse infrastructure other than
-`concourse.apachegeode-ci.info`, you must provide its hostname as the fourth argument:
-`./deploy_meta.sh apache geode apache concourse.mydomain.com`
-
-## Specifying the GCS bucket for artifact and test result storage
-If you would like to use a bucket other than `files.apachegeode-ci.info` for storing test results
-and artifacts, you must specify it as the fifth argument:
-`./deploy_meta.sh apache geode apache concourse.apachegeode-ci.info <my-gcs-bucket-name>`
-
-## Changing whether the created pipelines are public or not
-If the upstream and fork names do not match, the pipeline will not be made public. Otherwise
-the pipelines will be public by default, but can be overridden with the fifth argument:
-`./deploy_meta.sh apache geode apache concourse.apachegeode-ci.info files.apachegeode-ci.info false`
+If you change the configuration to use your own fork of Geode, the resulting meta pipeline will be
+named `<your GitHub username>-<branch>-meta`.
diff --git a/ci/pipelines/meta/deploy_meta.sh b/ci/pipelines/meta/deploy_meta.sh
index 8cdc4b6..38171c4 100755
--- a/ci/pipelines/meta/deploy_meta.sh
+++ b/ci/pipelines/meta/deploy_meta.sh
@@ -34,28 +34,39 @@ for cmd in Jinja2 PyYAML; do
fi
done
-if [ -z $(command -v gcloud) ]; then
- echo "Install gcloud (try brew cask install google-cloud-sdk, or see https://cloud.google.com/sdk/docs/downloads-interactive#mac)"
- exit 1
+META_PROPERTIES=${SCRIPTDIR}/meta.properties
+LOCAL_META_PROPERTIES=${SCRIPTDIR}/meta.properties.local
+
+## Load default properties file
+source ${META_PROPERTIES}
+echo "**************************************************"
+echo "Default Environment variables for this deployment:"
+cat ${SCRIPTDIR}/meta.properties | grep -v "^#"
+echo "**************************************************"
+
+## Load local overrides properties file
+if [[ -f ${LOCAL_META_PROPERTIES} ]]; then
+ echo "Locally overridden environment variables for this:"
+ cat ${SCRIPTDIR}/meta.properties.local
+ echo "**************************************************"
+ source ${LOCAL_META_PROPERTIES}
fi
-GCP_PROJECT=${GCP_PROJECT:-$(gcloud info --format="value(config.project)")}
-echo "GCP_PROJECT=${GCP_PROJECT}"
-if [ -z ${GCP_PROJECT} ]; then
- echo "GCP_PROJECT not set. Quitting"
- exit 1
+source ${META_PROPERTIES}
+
+if [[ -f ${LOCAL_META_PROPERTIES} ]]; then
+ source ${LOCAL_META_PROPERTIES}
fi
+read -n 1 -s -r -p "Press any key to continue or x to abort" DEPLOY
+echo
+if [[ "${DEPLOY}" == "x" ]]; then
+ echo "x pressed, aborting deploy."
+ exit 0
+fi
set -e
set -x
-GEODE_FORK=${1:-"apache"}
-GEODE_REPO_NAME=${2:-"geode"}
-UPSTREAM_FORK=${3:-"apache"}
-CONCOURSE_HOST=${4:-"concourse.apachegeode-ci.info"}
-ARTIFACT_BUCKET=${5:-"files.apachegeode-ci.info"}
-PUBLIC=${6:-"true"}
-REPOSITORY_PUBLIC=${7:-"true"}
if [[ "${CONCOURSE_HOST}" == "concourse.apachegeode-ci.info" ]]; then
CONCOURSE_SCHEME=https
fi
@@ -95,6 +106,8 @@ pushd ${SCRIPTDIR} 2>&1 > /dev/null
--var geode-repo-name=${GEODE_REPO_NAME} \
--var upstream-fork=${UPSTREAM_FORK} \
--var pipeline-prefix=${PIPELINE_PREFIX} \
+ --var gradle-global-args="${GRADLE_GLOBAL_ARGS}" \
+ --var maven-snapshot-bucket="${MAVEN_SNAPSHOT_BUCKET}" \
--var concourse-team=main \
--yaml-var public-pipelines=${PUBLIC} 2>&1 |tee flyOutput.log
@@ -203,13 +216,13 @@ set +x
if [[ "${GEODE_FORK}" != "${UPSTREAM_FORK}" ]]; then
echo "Disabling unnecessary jobs for forks."
pauseJobs ${META_PIPELINE} set-images set-reaper
-elif [[ "$GEODE_FORK" == "apache" ]] && [[ "$GEODE_BRANCH" == "develop" ]]; then
+elif [[ "$GEODE_FORK" == "${UPSTREAM_FORK}" ]] && [[ "$GEODE_BRANCH" == "develop" ]]; then
echo "Disabling optional jobs for develop"
- pauseNewJobs set-pr set-images set-metrics set-examples
+ pauseNewJobs ${META_PIPELINE} set-pr set-images set-metrics set-examples
else
echo "Disabling unnecessary jobs for release branches."
echo "*** DO NOT RE-ENABLE THESE META-JOBS ***"
- pauseJobs set-pr set-images set-reaper set-metrics set-examples
+ pauseJobs ${META_PIPELINE} set-pr set-images set-reaper set-metrics set-examples
fi
unpausePipeline ${META_PIPELINE}
diff --git a/ci/pipelines/meta/jinja.template.yml b/ci/pipelines/meta/jinja.template.yml
index 1290997..19cfcc4 100644
--- a/ci/pipelines/meta/jinja.template.yml
+++ b/ci/pipelines/meta/jinja.template.yml
@@ -175,6 +175,8 @@ jobs:
SANITIZED_GEODE_FORK: ((sanitized-geode-fork))
GCP_PROJECT: ((gcp-project))
PUBLIC_PIPELINES: ((public-pipelines))
+ GRADLE_GLOBAL_ARGS: ((gradle-global-args))
+ MAVEN_SNAPSHOT_BUCKET: ((maven-snapshot-bucket))
run:
path: geode-pipeline/ci/pipelines/geode-build/deploy_pipeline.sh
- put: concourse
@@ -263,6 +265,7 @@ jobs:
PIPELINE_PREFIX: ((pipeline-prefix))
GCP_PROJECT: ((gcp-project))
PUBLIC_PIPELINES: ((public-pipelines))
+ GRADLE_GLOBAL_ARGS: ((gradle-global-args))
run:
path: geode-pr-pipeline/ci/pipelines/pull-request/deploy_pr_pipeline.sh
- put: concourse
diff --git a/ci/scripts/stop_instance.sh b/ci/pipelines/meta/meta.properties
old mode 100755
new mode 100644
similarity index 53%
copy from ci/scripts/stop_instance.sh
copy to ci/pipelines/meta/meta.properties
index 62b4697..a5894c8
--- a/ci/scripts/stop_instance.sh
+++ b/ci/pipelines/meta/meta.properties
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,25 +14,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-set -e
-
-BASE_DIR=$(pwd)
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-INSTANCE_NAME="$(cat instance-data/instance-name)"
-ZONE="$(cat instance-data/zone)"
-
-
-echo 'StrictHostKeyChecking no' >> /etc/ssh/ssh_config
-
-gcloud compute instances delete ${INSTANCE_NAME} \
- --zone=${ZONE} \
- --quiet
+UPSTREAM_FORK=apache
+GEODE_FORK=apache
+GEODE_REPO_NAME=geode
+CONCOURSE_HOST=concourse.apachegeode-ci.info
+GCP_PROJECT=apachegeode-ci
+ARTIFACT_BUCKET=files.apachegeode-ci.info
+PUBLIC=true
+REPOSITORY_PUBLIC=true
+GRADLE_GLOBAL_ARGS=""
+MAVEN_SNAPSHOT_BUCKET=maven.apachegeode-ci.info
diff --git a/ci/pipelines/pull-request/deploy_pr_pipeline.sh b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
index 9677714..89c8521 100755
--- a/ci/pipelines/pull-request/deploy_pr_pipeline.sh
+++ b/ci/pipelines/pull-request/deploy_pr_pipeline.sh
@@ -89,5 +89,6 @@ public-pipelines: ${PUBLIC_PIPELINES}
gcp-project: ${GCP_PROJECT}
version-bucket: ${VERSION_BUCKET}
artifact-bucket: ${ARTIFACT_BUCKET}
+gradle-global-args: ${GRADLE_GLOBAL_ARGS}
YML
diff --git a/ci/pipelines/pull-request/jinja.template.yml b/ci/pipelines/pull-request/jinja.template.yml
index 2335ad7..9bc0da0 100644
--- a/ci/pipelines/pull-request/jinja.template.yml
+++ b/ci/pipelines/pull-request/jinja.template.yml
@@ -158,7 +158,7 @@ jobs:
status: pending
- do:
- put: concourse-metadata-resource
- - task: start_instance
+ - task: create_instance
{{- alpine_tools_config()|indent(10) }}
params:
CPUS: {{test.CPUS}}
@@ -168,7 +168,7 @@ jobs:
GCP_PROJECT: ((gcp-project))
RAM: {{test.RAM}}
run:
- path: geode-ci/ci/scripts/start_instance.sh
+ path: geode-ci/ci/scripts/create_instance.sh
inputs:
- name: concourse-metadata-resource
- name: geode
@@ -200,7 +200,8 @@ jobs:
PARALLEL_DUNIT: {{test.PARALLEL_DUNIT}}
PARALLEL_GRADLE: {{test.PARALLEL_GRADLE}}
ARTIFACT_BUCKET: ((artifact-bucket))
- SERVICE_ACCOUNT: ((!concourse-gcp-account))
+ SERVICE_ACCOUNT: ((concourse-gcp-account))
+ GRADLE_GLOBAL_ARGS: ((gradle-global-args))
run:
{%- if test.name=="StressNew" %}
path: geode/ci/scripts/repeat-new-tests.sh
@@ -253,10 +254,10 @@ jobs:
- name: geode
- name: geode-ci
- name: geode-results
- - task: stop_instance
+ - task: delete_instance
{{- alpine_tools_config()|indent(10) }}
run:
- path: geode-ci/ci/scripts/stop_instance.sh
+ path: geode-ci/ci/scripts/delete_instance.sh
inputs:
- name: geode
- name: geode-ci
diff --git a/ci/scripts/build-examples.sh b/ci/scripts/build-examples.sh
deleted file mode 100755
index ae984fc..0000000
--- a/ci/scripts/build-examples.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/local/bin/tini-wrapper /bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-set -e -x
-export TERM=${TERM:-dumb}
-pushd geode-examples
- ./gradlew --no-daemon clean runAll
-popd
-
diff --git a/ci/scripts/start_instance.sh b/ci/scripts/create_instance.sh
similarity index 98%
rename from ci/scripts/start_instance.sh
rename to ci/scripts/create_instance.sh
index 6618b4c..8c69944 100755
--- a/ci/scripts/start_instance.sh
+++ b/ci/scripts/create_instance.sh
@@ -65,9 +65,6 @@ fi
ZONE=us-central1-f
INSTANCE_NAME="heavy-lifter-$(uuidgen -n @dns -s -N "${WINDOWS_PREFIX}${SANITIZED_GEODE_FORK}-${SANITIZED_GEODE_BRANCH}-${SANITIZED_BUILD_PIPELINE_NAME}-${SANITIZED_BUILD_JOB_NAME}-${SANITIZED_BUILD_NAME}")"
-gcloud compute instances delete ${INSTANCE_NAME} \
- --zone=${ZONE} \
- --quiet || true
MY_NAME=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
MY_ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google")
diff --git a/ci/scripts/stop_instance.sh b/ci/scripts/delete_instance.sh
similarity index 100%
rename from ci/scripts/stop_instance.sh
rename to ci/scripts/delete_instance.sh
diff --git a/ci/scripts/execute_build.sh b/ci/scripts/execute_build.sh
index fffadf0..09aa6e4 100755
--- a/ci/scripts/execute_build.sh
+++ b/ci/scripts/execute_build.sh
@@ -90,6 +90,7 @@ fi
GRADLE_COMMAND="./gradlew \
${DEFAULT_GRADLE_TASK_OPTIONS} \
+ ${GRADLE_GLOBAL_ARGS} \
build install -x test"
echo "${GRADLE_COMMAND}"
diff --git a/ci/scripts/execute_publish.sh b/ci/scripts/execute_publish.sh
index cc4d73a..9bd6e9d 100755
--- a/ci/scripts/execute_publish.sh
+++ b/ci/scripts/execute_publish.sh
@@ -67,7 +67,7 @@ SET_JAVA_HOME="export JAVA_HOME=/usr/lib/jvm/java-${JAVA_BUILD_VERSION}-openjdk-
GRADLE_COMMAND="./gradlew \
${DEFAULT_GRADLE_TASK_OPTIONS} \
- -PbuildId=${BUILD_ID} publish"
+ -PbuildId=${BUILD_ID} -PmavenSnapshotBucket=${MAVEN_SNAPSHOT_BUCKET} publish"
echo "${GRADLE_COMMAND}"
ssh ${SSH_OPTIONS} geode@${INSTANCE_IP_ADDRESS} "mkdir -p tmp && cd geode && ${SET_JAVA_HOME} && ${GRADLE_COMMAND}"
diff --git a/ci/scripts/execute_tests.sh b/ci/scripts/execute_tests.sh
index 11a5ff5..95ef72a 100755
--- a/ci/scripts/execute_tests.sh
+++ b/ci/scripts/execute_tests.sh
@@ -91,7 +91,8 @@ GRADLE_ARGS=" \
-PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}') \
${DEFAULT_GRADLE_TASK_OPTIONS} \
${GRADLE_TASK} \
- ${GRADLE_TASK_OPTIONS}"
+ ${GRADLE_TASK_OPTIONS} \
+ ${GRADLE_GLOBAL_ARGS}"
EXEC_COMMAND="bash -c 'echo Building with: $SEP ${JAVA_BUILD_PATH}/bin/java -version $SEP echo Testing with: $SEP ${JAVA_TEST_PATH}/bin/java -version $SEP cd geode $SEP cp gradlew gradlewStrict $SEP sed -e 's/JAVA_HOME/GRADLE_JVM/g' -i.bak gradlewStrict $SEP GRADLE_JVM=${JAVA_BUILD_PATH} ./gradlewStrict ${GRADLE_ARGS}'"
echo "${EXEC_COMMAND}"
diff --git a/ci/scripts/update-passing-ref.sh b/ci/scripts/update-passing-ref.sh
deleted file mode 100755
index bf6b894..0000000
--- a/ci/scripts/update-passing-ref.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-export BUILDROOT=$(pwd)
-REPOSITORY_DIR=$(pwd)/geode
-LOCAL_FILE=${BUILDROOT}/results/passing.txt
-DESTINATION_URL=gs://${ARTIFACT_BUCKET}/${MAINTENANCE_VERSION}/passing.txt
-pushd ${REPOSITORY_DIR}
-git rev-parse HEAD > ${LOCAL_FILE}
-popd
-echo "Updating passing reference file for ${MAINTENANCE_VERSION} to the following SHA:"
-cat ${LOCAL_FILE}
-gsutil -q -m cp ${LOCAL_FILE} ${DESTINATION_URL}
-gsutil setmeta -h "Cache-Control:no-cache" ${DESTINATION_URL}
diff --git a/gradle/publish.gradle b/gradle/publish.gradle
index 72a4597..b4d5e7f 100644
--- a/gradle/publish.gradle
+++ b/gradle/publish.gradle
@@ -108,7 +108,11 @@ subprojects {
} else {
// If testing from a non-GCE instance, then the shell needs service-account creds
// following the instructions at https://cloud.google.com/docs/authentication/production
- url = "gcs://maven.apachegeode-ci.info/snapshots"
+ if (project.hasProperty("mavenSnapshotBucket")) {
+ url = "gcs://$project.mavenSnapshotBucket/snapshots"
+ } else {
+ url = "gcs://maven.apachegeode-ci.info/snapshots"
+ }
}
}
if (! url.toString().startsWith("gcs:")) {
diff --git a/gradle/rat.gradle b/gradle/rat.gradle
index df7538b..1048d96 100644
--- a/gradle/rat.gradle
+++ b/gradle/rat.gradle
@@ -100,6 +100,9 @@ rat {
// other text files
'**/log4j*.xml',
+ // local configuration files that are ignored by git
+ 'ci/pipelines/meta/meta.properties.local',
+
// modules
'extensions/**/log4j.properties',
'extensions/**/tomcat-users.xml',
diff --git a/gradle/test.gradle b/gradle/test.gradle
index 7cbc027..f83b970 100644
--- a/gradle/test.gradle
+++ b/gradle/test.gradle
@@ -196,6 +196,12 @@ subprojects {
// default classpath works for this one.
}
+ configure([integrationTest, distributedTest, performanceTest, acceptanceTest, uiTest, upgradeTest]) {
+ if (project.hasProperty('excludeTest')) {
+ exclude project.getProperty('excludeTest').split(',')
+ }
+ }
+
configure([repeatDistributedTest, repeatIntegrationTest, repeatUpgradeTest, repeatUnitTest, repeatAcceptanceTest]) {
times = Integer.parseInt(repeat)
useJUnit {}