You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@cassandra.apache.org by ad...@apache.org on 2021/09/28 16:52:40 UTC
[cassandra] branch cassandra-3.0 updated: Add option for
environment variables to CircleCI config generation script
This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-3.0 by this push:
new b6678a0 Add option for environment variables to CircleCI config generation script
b6678a0 is described below
commit b6678a002cfa00be2068d499c5b7ce9c515bbc4e
Author: Andrés de la Peña <a....@gmail.com>
AuthorDate: Tue Sep 28 17:45:56 2021 +0100
Add option for environment variables to CircleCI config generation script
patch by Andrés de la Peña; reviewed by Berenguer Blasi, Ekaterina Dimitrova and Ruslan Fomkin for CASSANDRA-16989
---
.circleci/config-2_1.yml | 8 ++++
.circleci/generate.sh | 109 +++++++++++++++++++++++++++++++++++++++++------
.circleci/readme.md | 30 +++++++++++++
3 files changed, 135 insertions(+), 12 deletions(-)
diff --git a/.circleci/config-2_1.yml b/.circleci/config-2_1.yml
index 9595495..c5e2583 100644
--- a/.circleci/config-2_1.yml
+++ b/.circleci/config-2_1.yml
@@ -19,6 +19,14 @@
version: 2.1
default_env_vars: &default_env_vars
+
+ # The values of some of these environment variables are meant to be frequently changed by developers.
+ # The generate.sh script contains a list of accepted environment variables that should contain some of
+ # these variables. Also, some variables are mentioned in the documentation, at least in
+ # .circleci/readme.md and in doc/source/development/testing.rst.
+ # If you modify these variables, or if you add new variables whose values are meant to be changed frequently,
+ # please remember to modify the generate.sh script and the documentation accordingly.
+
JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
ANT_HOME: /usr/share/ant
LANG: en_US.UTF-8
diff --git a/.circleci/generate.sh b/.circleci/generate.sh
index db7d772..f9d5ddf 100755
--- a/.circleci/generate.sh
+++ b/.circleci/generate.sh
@@ -22,41 +22,109 @@ BASEDIR=`dirname $0`
die ()
{
echo "ERROR: $*"
- echo "Usage: $0 [-l|-m|-h]"
+ print_help
+ exit 1
+}
+
+print_help()
+{
+ echo "Usage: $0 [-l|-m|-h|-f|-e]"
+ echo " -a Generate the default config.yml using low resources and the three templates"
+ echo " (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES). Use this for"
+ echo " permanent changes in config-2_1.yml that will be committed to the main repo."
echo " -l Generate config.yml using low resources"
echo " -m Generate config.yml using mid resources"
echo " -h Generate config.yml using high resources"
- echo " No flags generates the default config.yml using low resources and the three"
- echo " templates (config.yml.LOWRES, config.yml.MIDRES and config.yml.HIGHRES)"
- exit 1
+ echo " -e <key=value> Environment variables to be used in the generated config.yml, e.g.:"
+ echo " -e DTEST_BRANCH=CASSANDRA-8272"
+ echo " -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git"
+ echo " -e REPEATED_UTEST_TARGET=testsome"
+ echo " -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest"
+ echo " -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable"
+ echo " -e REPEATED_UTEST_COUNT=100"
+ echo " -e REPEATED_UTEST_STOP_ON_FAILURE=false"
+ echo " -e REPEATED_DTEST_NAME=cqlsh_tests/test_cqlsh.py::TestCqlshSmoke"
+ echo " -e REPEATED_DTEST_VNODES=false"
+ echo " -e REPEATED_DTEST_COUNT=100"
+ echo " -e REPEATED_DTEST_STOP_ON_FAILURE=false"
+ echo " For the complete list of environment variables, please check the"
+ echo " list of examples in config-2_1.yml and/or the documentation."
+ echo " If you want to specify multiple environment variables simply add"
+ echo " multiple -e options. The flags -l/-m/-h should be used when using -e."
+ echo " -f Stop checking that the environment variables are known"
}
+all=false
lowres=false
midres=false
highres=false
-while getopts ":lmh" opt; do
+env_vars=""
+has_env_vars=false
+check_env_vars=true
+while getopts "e:almhf" opt; do
case $opt in
- l ) ($midres || $highres) && die "Cannot specify option -l after specifying options -m or -h"
- lowres=true
+ a ) all=true
+ ;;
+ l ) lowres=true
+ ;;
+ m ) midres=true
;;
- m ) ($lowres || $highres) && die "Cannot specify option -m after specifying options -l or -h"
- midres=true
+ h ) highres=true
;;
- h ) ($lowres || $midres) && die "Cannot specify option -h after specifying options -l or -m"
- highres=true
+ e ) if (!($has_env_vars)); then
+ env_vars="$OPTARG"
+ else
+ env_vars="$env_vars|$OPTARG"
+ fi
+ has_env_vars=true
+ ;;
+ f ) check_env_vars=false
;;
\?) die "Invalid option: -$OPTARG"
;;
esac
done
+shift $((OPTIND-1))
+if [ "$#" -ne 0 ]; then
+ die "Unexpected arguments"
+fi
+
+# validate environment variables
+if $has_env_vars && $check_env_vars; then
+ for entry in $(echo $env_vars | tr "|" "\n"); do
+ key=$(echo $entry | tr "=" "\n" | head -n 1)
+ if [ "$key" != "DTEST_REPO" ] &&
+ [ "$key" != "DTEST_BRANCH" ] &&
+ [ "$key" != "REPEATED_UTEST_TARGET" ] &&
+ [ "$key" != "REPEATED_UTEST_CLASS" ] &&
+ [ "$key" != "REPEATED_UTEST_METHODS" ] &&
+ [ "$key" != "REPEATED_UTEST_COUNT" ] &&
+ [ "$key" != "REPEATED_UTEST_STOP_ON_FAILURE" ] &&
+ [ "$key" != "REPEATED_DTEST_NAME" ] &&
+ [ "$key" != "REPEATED_DTEST_VNODES" ] &&
+ [ "$key" != "REPEATED_DTEST_COUNT" ] &&
+ [ "$key" != "REPEATED_DTEST_STOP_ON_FAILURE" ] &&
+ [ "$key" != "REPEATED_UPGRADE_DTEST_NAME" ] &&
+ [ "$key" != "REPEATED_UPGRADE_DTEST_COUNT" ] &&
+ [ "$key" != "REPEATED_UPGRADE_DTEST_STOP_ON_FAILURE" ] &&
+ [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_CLASS" ] &&
+ [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_METHODS" ] &&
+ [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_COUNT" ] &&
+ [ "$key" != "REPEATED_JVM_UPGRADE_DTEST_STOP_ON_FAILURE" ]; then
+ die "Unrecognised environment variable name: $key"
+ fi
+ done
+fi
if $lowres; then
+ ($all || $midres || $highres) && die "Cannot use option -l with options -a, -m or -h"
echo "Generating new config.yml file with low resources from config-2_1.yml"
circleci config process $BASEDIR/config-2_1.yml > $BASEDIR/config.yml.LOWRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.LOWRES.tmp > $BASEDIR/config.yml
rm $BASEDIR/config.yml.LOWRES.tmp
elif $midres; then
+ ($all || $lowres || $highres) && die "Cannot use option -m with options -a, -l or -h"
echo "Generating new config.yml file with middle resources from config-2_1.yml"
patch -o $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.mid_res.patch
circleci config process $BASEDIR/config-2_1.yml.MIDRES > $BASEDIR/config.yml.MIDRES.tmp
@@ -64,13 +132,15 @@ elif $midres; then
rm $BASEDIR/config-2_1.yml.MIDRES $BASEDIR/config.yml.MIDRES.tmp
elif $highres; then
+ ($all || $lowres || $midres) && die "Cannot use option -h with options -a, -l or -m"
echo "Generating new config.yml file with high resources from config-2_1.yml"
patch -o $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config-2_1.yml $BASEDIR/config-2_1.yml.high_res.patch
circleci config process $BASEDIR/config-2_1.yml.HIGHRES > $BASEDIR/config.yml.HIGHRES.tmp
cat $BASEDIR/license.yml $BASEDIR/config.yml.HIGHRES.tmp > $BASEDIR/config.yml
rm $BASEDIR/config-2_1.yml.HIGHRES $BASEDIR/config.yml.HIGHRES.tmp
-else
+elif $all; then
+ ($lowres || $midres || $highres || $has_env_vars) && die "Cannot use option -a with options -l, -m, -h or -e"
echo "Generating new config.yml file with low resources and LOWRES/MIDRES/HIGHRES templates from config-2_1.yml"
# setup lowres
@@ -92,7 +162,22 @@ else
# copy lower into config.yml to make sure this gets updated
cp $BASEDIR/config.yml.LOWRES $BASEDIR/config.yml
+
+elif (!($has_env_vars)); then
+ print_help
fi
+# replace environment variables
+if $has_env_vars; then
+ IFS='='
+ echo "$env_vars" | tr '|' '\n' | while read entry; do
+ set -- $entry
+ key=$1
+ val=$2
+ echo "Setting environment variable $key: $val"
+ sed -i.bak "s|- $key:.*|- $key: $val|" $BASEDIR/config.yml
+ done
+ unset IFS
+fi
diff --git a/.circleci/readme.md b/.circleci/readme.md
index e4e32f5..044adfc 100644
--- a/.circleci/readme.md
+++ b/.circleci/readme.md
@@ -42,6 +42,36 @@ This script validates and applies any changes to the `config-2_1.yml` file, and
requires the [CircleCI CLI](https://circleci.com/docs/2.0/local-cli/#install) to be
installed.
+## Setting environment variables
+Both `config-2_1.yml` and `config.yml` files contain a set of environment variables
+defining things like what dtest repo and branch to use, what tests could be repeatedly
+run, etc.
+
+These environment variables can be directly edited in the `config.yml` file, although if
+you do this you should take into account that the entire set of env vars is repeated on
+every job.
+
+A probably better approach is editing them in `config-2_1.yml` and then regenerate the
+`config.yml` file using the `generate.sh` script. You can also directly pass environment
+variable values to the `generate.sh` script with the `-e` flag. For example, to set the
+dtest repo and branch with MIDRES config you can run:
+
+```
+generate.sh -m \
+ -e DTEST_REPO=git://github.com/adelapena/cassandra-dtest.git \
+ -e DTEST_BRANCH=CASSANDRA-8272
+```
+
+Or you can set the test multiplexer for repeating a specific test with HIGHRES:
+
+```
+generate.sh -h \
+ -e REPEATED_UTEST_TARGET=testsome \
+ -e REPEATED_UTEST_CLASS=org.apache.cassandra.cql3.ViewTest \
+ -e REPEATED_UTEST_METHODS=testCompoundPartitionKey,testStaticTable \
+ -e REPEATED_UTEST_COUNT=100
+```
+
## Updating the config
For configuration changes meant to be permanent in the Apache repo you should never edit
the `config.yml` file manually. Instead, you should edit the `config-2_1.yml` file and then
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org