You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2020/05/12 09:25:57 UTC
[flink-docker] 01/02: [FLINK-17439] Introduce development branches
This is an automated email from the ASF dual-hosted git repository.
chesnay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
commit 8513a68143c61f10114fc94d09921a1193741dd7
Author: Chesnay Schepler <ch...@apache.org>
AuthorDate: Thu Apr 30 10:11:18 2020 +0200
[FLINK-17439] Introduce development branches
- remove generates/templates; these now reside in the development branches
- remove testing files; these now reside in the development branches
- update README.md to reflect new development/release workflow
---
.travis.yml | 9 --
Dockerfile-debian.template | 87 ---------------
README.md | 59 +++++++---
add-custom.sh | 49 ---------
add-version.sh | 113 -------------------
common.sh | 26 -----
docker-entrypoint.sh | 114 -------------------
testing/run_travis_tests.sh | 31 ------
testing/testing_lib.sh | 261 --------------------------------------------
9 files changed, 43 insertions(+), 706 deletions(-)
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index a4e4f1d..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-os: linux
-dist: bionic
-language: shell
-
-services:
-- docker
-
-script:
-- testing/run_travis_tests.sh
diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template
deleted file mode 100644
index 7222314..0000000
--- a/Dockerfile-debian.template
+++ /dev/null
@@ -1,87 +0,0 @@
-###############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
-
-FROM openjdk:8-jre
-
-# Install dependencies
-RUN set -ex; \
- apt-get update; \
- apt-get -y install libsnappy1v5 gettext-base; \
- rm -rf /var/lib/apt/lists/*
-
-# Grab gosu for easy step-down from root
-ENV GOSU_VERSION 1.11
-RUN set -ex; \
- wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \
- wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
- done && \
- gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
- chmod +x /usr/local/bin/gosu; \
- gosu nobody true
-
-# Configure Flink version
-ENV FLINK_TGZ_URL=%%BINARY_DOWNLOAD_URL%% \
- FLINK_ASC_URL=%%ASC_DOWNLOAD_URL%% \
- GPG_KEY=%%GPG_KEY%% \
- CHECK_GPG=%%CHECK_GPG%%
-
-# Prepare environment
-ENV FLINK_HOME=/opt/flink
-ENV PATH=$FLINK_HOME/bin:$PATH
-RUN groupadd --system --gid=9999 flink && \
- useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
-WORKDIR $FLINK_HOME
-
-# Install Flink
-RUN set -ex; \
- wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
- \
- if [ "$CHECK_GPG" = "true" ]; then \
- wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
- export GNUPGHOME="$(mktemp -d)"; \
- for server in ha.pool.sks-keyservers.net $(shuf -e \
- hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com \
- hkp://keyserver.ubuntu.com:80 \
- pgp.mit.edu) ; do \
- gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
- done && \
- gpg --batch --verify flink.tgz.asc flink.tgz; \
- gpgconf --kill all; \
- rm -rf "$GNUPGHOME" flink.tgz.asc; \
- fi; \
- \
- tar -xf flink.tgz --strip-components=1; \
- rm flink.tgz; \
- \
- chown -R flink:flink .;
-
-# Configure container
-COPY docker-entrypoint.sh /
-ENTRYPOINT ["/docker-entrypoint.sh"]
-EXPOSE 6123 8081
-CMD ["help"]
diff --git a/README.md b/README.md
index ff13dd7..ef2b814 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,24 @@ Flink Docker image lifecycle
https://github.com/docker-library/official-images/blob/master/library/flink).
+Development workflow
+----------------------------
+
+The `master` branch of this repository serves as a pure publishing area for releases.
+
+Development happens on the various `dev-X` branches.
+
+Pull requests for a specific version should be opened against the respective `dev-<version>` branch.
+Pull requests for all versions, or for the next minor Flink release, should be opened against the `dev-master` branch.
+
+### CI
+
+The `dev-master` branch is tested against nightly Flink snapshots for the next minor Flink version. This allows us to
+develop features in tandem with Flink.
+
+The `dev-1.x` branches are tested against the latest corresponding patch Flink release, to ensure any changes we make
+are compatible with the currently used Flink version.
+
Workflow for new Flink releases
-------------------------------
@@ -39,24 +57,30 @@ There are additional steps required when a new Flink minor version (x.y.0) is re
### Release workflow
-When a new release of Flink is available, the Dockerfiles in this repo should be updated and a new
+When a new release of Flink is available, the Dockerfiles in the `master` branch should be updated and a new
manifest sent to the Docker Library [`official-images`](
https://github.com/docker-library/official-images) repo.
-Updating the Dockerfiles involves 3 steps:
-
-1. Update `add-version.sh` with the GPG key ID of the key used to sign the new release
- * Commit this change with message `Add GPG key for x.y.z release` <sup>\[[example](
- https://github.com/apache/flink-docker/commit/94845f46c0f0f2de80d4a5ce309db49aff4655d0)]</sup>
-2. Remove any existing Dockerfiles from the same minor version
- * e.g. `rm -r 1.2`, if the new Flink version is `1.2.1`
-3. Run `add-version.sh` with the appropriate arguments (`-r flink-minor-version -f
- flink-full-version`)
- * e.g. `./add-version.sh -r 1.2 -f 1.2.1`
- * Commit these changes with message `Update Dockerfiles for x.y.z release` <sup>\[[example](
+The Dockerfiles are generated on the respective `dev-<version>` branches, and copied over to the `master` branch for
+publishing.
+
+Updating the Dockerfiles involves the following steps:
+
+1. Generate the Dockerfiles
+ * Checkout the `dev-x.y` branch of the respective release, e.g., dev-1.9
+ * Update `add-version.sh` with the GPG key ID of the key used to sign the new release
+ * Commit this change with message `Add GPG key for x.y.z release` <sup>\[[example](
+ https://github.com/apache/flink-docker/commit/94845f46c0f0f2de80d4a5ce309db49aff4655d0)]</sup>
+ * Create a pull request against the `dev-x.y` branch containing this commit.
+ * Run `add-version.sh` with the appropriate arguments (`-r flink-minor-version -f flink-full-version`)
+ * e.g. `./add-version.sh -r 1.2 -f 1.2.1`
+2. Update Dockerfiles on the the `master` branch
+ * Remove any existing Dockerfiles from the same minor version
+ * e.g. `rm -r 1.2`, if the new Flink version is `1.2.1`
+ * Copy the generated Dockerfiles from the `dev-x.y` branch to `master`
+ * Commit the changes with message `Update Dockerfiles for x.y.z release` <sup>\[[example](
https://github.com/apache/flink-docker/commit/5920fd775ca1a8d03ee959d79bceeb5d6e8f35a1)]</sup>
-
-A pull request can then be opened on this repo with the changes.
+ * Create a pull request against the `master` branch containing this commit.
Once the pull request has been merged, a new manifest should be generated and a pull request opened
on the Docker Library [`official-images`](https://github.com/docker-library/official-images) repo.
@@ -79,14 +103,17 @@ available shortly thereafter.
### Release checklist
+Checklist for the `dev` branch:
- [ ] The GPG key ID of the key used to sign the release has been added to `add-version.sh` and
committed with the message `Add GPG key for x.y.z release`
+- [ ] `./add-version.sh -r x.y -f x.y.z` has been run on the respective dev branch
+
+Checklist for the `master` branch:
- [ ] _(new minor releases only)_ any unsupported Flink minor version Dockerfiles have been removed
(only two `x.y/` directories should be present)
- [ ] _(new patch releases only)_ any existing generated files for the same minor version have been
removed
-- [ ] `./add-version.sh -r x.y -f x.y.z` has been run, and the updated Dockerfiles committed with
- the message `Update Dockerfiles for x.y.z release`
+- [ ] The updated Dockerfiles have been committed with the message `Update Dockerfiles for x.y.z release`
- [ ] _(new minor releases only)_ the `aliases` array in `generate-stackbrew-library.sh` has been
updated with `[x.y]='latest'` and committed with the message `Update latest image tag to x.y`
- [ ] A pull request with the above changes has been opened on this repo and merged
diff --git a/add-custom.sh b/add-custom.sh
deleted file mode 100755
index c863f3f..0000000
--- a/add-custom.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash -e
-
-# Use this script to build the Dockerfiles against an arbitrary
-# Flink distribution.
-# This is exlusively for development purposes.
-
-source "$(dirname "$0")"/common.sh
-
-function usage() {
- echo >&2 "usage: $0 -u binary-download-url [-n name]"
-}
-
-binary_download_url=
-name=custom
-
-while getopts u:n:h arg; do
- case "$arg" in
- u)
- binary_download_url=$OPTARG
- ;;
- n)
- name=$OPTARG
- ;;
- h)
- usage
- exit 0
- ;;
- \?)
- usage
- exit 1
- ;;
- esac
-done
-
-if [ -z "${binary_download_url}" ]; then
- usage
- exit 1
-fi
-
-mkdir -p "dev"
-
-echo -n >&2 "Generating Dockerfiles..."
-for source_variant in "${SOURCE_VARIANTS[@]}"; do
- dir="dev/${name}-${source_variant}"
- rm -rf "${dir}"
-
- generate "${dir}" "${binary_download_url}" "" "" false ${source_variant}
-done
-echo >&2 " done."
diff --git a/add-version.sh b/add-version.sh
deleted file mode 100755
index 329ed70..0000000
--- a/add-version.sh
+++ /dev/null
@@ -1,113 +0,0 @@
-#!/bin/bash -e
-
-# Use this script to rebuild the Dockerfiles and all variants for a particular
-# release. Before running this, you must first delete the existing release
-# directory.
-#
-# TODO: to conform with other similar setups, this likely needs to become
-# "update.sh" and be taught how to derive the latest version (e.g. 1.2.0) from
-# a given release (e.g. 1.2) and assemble a .travis.yml file dynamically.
-#
-# See other repos (e.g. httpd, cassandra) for update.sh examples.
-
-source "$(dirname "$0")"/common.sh
-
-function usage() {
- echo >&2 "usage: $0 -r flink-release -f flink-version"
-}
-
-function error() {
- local msg="$1"
- if [ -n "$2" ]; then
- local code="$2"
- else
- local code=1
- fi
- echo >&2 "$msg"
- exit "$code"
-}
-
-flink_release= # Like 1.2
-flink_version= # Like 1.2.0
-
-while getopts r:f:h arg; do
- case "$arg" in
- r)
- flink_release=$OPTARG
- ;;
- f)
- flink_version=$OPTARG
- ;;
- h)
- usage
- exit 0
- ;;
- \?)
- usage
- exit 1
- ;;
- esac
-done
-
-if [ -z "$flink_release" ] || [ -z "$flink_version" ]; then
- usage
- exit 1
-fi
-
-if [[ ! "$flink_version" =~ ^$flink_release\.+ ]]; then
- error "Flink release must be prefix of version"
-fi
-
-# Defaults, can vary between versions
-scala_versions=( 2.11 2.12 )
-gpg_key=
-
-# Version-specific variants (example)
-# if [ "$flink_release" = "x.y" ]; then
-# scala_versions=( 2.10 2.11 2.12 )
-# fi
-
-# No real need to cull old versions
-if [ "$flink_version" = "1.8.0" ]; then
- gpg_key="F2A67A8047499BBB3908D17AA8F4FD97121D7293"
-elif [ "$flink_version" = "1.8.1" ]; then
- gpg_key="8FEA1EE9D0048C0CCC70B7573211B0703B79EA0E"
-elif [ "$flink_version" = "1.8.2" ]; then
- gpg_key="E2C45417BED5C104154F341085BACB5AEFAE3202"
-elif [ "$flink_version" = "1.8.3" ]; then
- gpg_key="EF88474C564C7A608A822EEC3FF96A2057B6476C"
-elif [ "$flink_version" = "1.9.0" ]; then
- gpg_key="1C1E2394D3194E1944613488F320986D35C33D6A"
-elif [ "$flink_version" = "1.9.1" ]; then
- gpg_key="E2C45417BED5C104154F341085BACB5AEFAE3202"
-elif [ "$flink_version" = "1.9.2" ]; then
- gpg_key="EF88474C564C7A608A822EEC3FF96A2057B6476C"
-elif [ "$flink_version" = "1.9.3" ]; then
- gpg_key="6B6291A8502BA8F0913AE04DDEB95B05BF075300"
-elif [ "$flink_version" = "1.10.0" ]; then
- gpg_key="BB137807CEFBE7DD2616556710B12A1F89C115E8"
-else
- error "Missing GPG key ID for this release"
-fi
-
-if [ -d "$flink_release" ]; then
- error "Directory $flink_release already exists; delete before continuing"
-fi
-
-mkdir "$flink_release"
-
-echo -n >&2 "Generating Dockerfiles..."
-for source_variant in "${SOURCE_VARIANTS[@]}"; do
- for scala_version in "${scala_versions[@]}"; do
- dir="$flink_release/scala_${scala_version}-${source_variant}"
-
- flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz
-
- flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}"
- # Not all mirrors have the .asc files
- flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
-
- generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${source_variant}
- done
-done
-echo >&2 " done."
diff --git a/common.sh b/common.sh
deleted file mode 100644
index 0232930..0000000
--- a/common.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash -e
-
-# Defaults, can vary between versions
-export SOURCE_VARIANTS=( debian )
-
-function generate() {
- dir=$1
- binary_download_url=$2
- asc_download_url=$3
- gpg_key=$4
- check_gpg=$5
- source_variant=$6
-
- mkdir "$dir"
- cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
-
- # '&' has special semantics in sed replacement patterns
- escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
-
- sed \
- -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
- -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
- -e "s/%%GPG_KEY%%/$gpg_key/" \
- -e "s/%%CHECK_GPG%%/${check_gpg}/" \
- "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
-}
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
deleted file mode 100755
index 011ccea..0000000
--- a/docker-entrypoint.sh
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/bin/sh
-
-###############################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-###############################################################################
-
-# If unspecified, the hostname of the container is taken as the JobManager address
-JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
-CONF_FILE="${FLINK_HOME}/conf/flink-conf.yaml"
-
-drop_privs_cmd() {
- if [ $(id -u) != 0 ]; then
- # Don't need to drop privs if EUID != 0
- return
- elif [ -x /sbin/su-exec ]; then
- # Alpine
- echo su-exec flink
- else
- # Others
- echo gosu flink
- fi
-}
-
-copy_plugins_if_required() {
- if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then
- return 0
- fi
-
- echo "Enabling required built-in plugins"
- for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do
- echo "Linking ${target_plugin} to plugin directory"
- plugin_name=${target_plugin%.jar}
-
- mkdir -p "${FLINK_HOME}/plugins/${plugin_name}"
- if [ ! -e "${FLINK_HOME}/opt/${target_plugin}" ]; then
- echo "Plugin ${target_plugin} does not exist. Exiting."
- exit 1
- else
- ln -fs "${FLINK_HOME}/opt/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}"
- echo "Successfully enabled ${target_plugin}"
- fi
- done
-}
-
-set_config_option() {
- local option=$1
- local value=$2
-
- # escape periods for usage in regular expressions
- local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g")
-
- # either override an existing entry, or append a new one
- if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then
- sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}"
- else
- echo "${option}: ${value}" >> "${CONF_FILE}"
- fi
-}
-
-set_common_options() {
- set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS}
- set_config_option blob.server.port 6124
- set_config_option query.server.port 6125
-}
-
-if [ "$1" = "help" ]; then
- echo "Usage: $(basename "$0") (jobmanager|taskmanager|help)"
- exit 0
-elif [ "$1" = "jobmanager" ]; then
- shift 1
- echo "Starting Job Manager"
- copy_plugins_if_required
-
- set_common_options
-
- if [ -n "${FLINK_PROPERTIES}" ]; then
- echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
- fi
- envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
-
- exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
-elif [ "$1" = "taskmanager" ]; then
- shift 1
- echo "Starting Task Manager"
- copy_plugins_if_required
-
- TASK_MANAGER_NUMBER_OF_TASK_SLOTS=${TASK_MANAGER_NUMBER_OF_TASK_SLOTS:-$(grep -c ^processor /proc/cpuinfo)}
-
- set_common_options
- set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS}
-
- if [ -n "${FLINK_PROPERTIES}" ]; then
- echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
- fi
- envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
-
- exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@"
-fi
-
-exec "$@"
diff --git a/testing/run_travis_tests.sh b/testing/run_travis_tests.sh
deleted file mode 100755
index f00890e..0000000
--- a/testing/run_travis_tests.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash -e
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-
-. "${SCRIPT_DIR}/testing_lib.sh"
-
-IS_PULL_REQUEST=
-if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then
- IS_PULL_REQUEST=1
-fi
-
-BRANCH="$TRAVIS_BRANCH"
-
-if [ -n "$IS_PULL_REQUEST" ]; then
- changed_files="$(git diff --name-only $BRANCH...HEAD)"
-
- echo "Changed files in this pull request:"
- echo "${changed_files}"
- echo
-fi
-
-if [ -z "$IS_PULL_REQUEST" ] && [ "$BRANCH" = "master" ]; then
- # Test all images on master
- smoke_test_all_images
- smoke_test_non_root
-else
- # For pull requests and branches, test one image
- smoke_test_one_image
-fi
-
-# vim: et ts=2 sw=2
diff --git a/testing/testing_lib.sh b/testing/testing_lib.sh
deleted file mode 100644
index 36dfe32..0000000
--- a/testing/testing_lib.sh
+++ /dev/null
@@ -1,261 +0,0 @@
-#!/bin/bash -e
-
-# This test script discovers all image variants, builds them, then runs a
-# simple smoke test for each:
-# - start up a jobmanager
-# - wait for the /overview web UI endpoint to return successfully
-# - start up a taskmanager
-# - wait for the /overview web UI endpoint to report 1 connected taskmanager
-
-CURL_TIMEOUT=1
-CURL_COOLDOWN=1
-CURL_MAX_TRIES=30
-CURL_ENDPOINT=localhost:8081/overview
-IMAGE_REPO=docker-flink-test
-NETWORK_NAME=docker-flink-test-net
-
-function image_tag() {
- local dockerfile
- dockerfile="$1"
-
- local variant minor_version
- variant="$(basename "$(dirname "$dockerfile")")"
- minor_version="$(basename "$(dirname "$(dirname "$dockerfile")")")"
-
- echo "${minor_version}-${variant}"
-}
-
-function image_name() {
- local image_tag
- image_tag="$1"
-
- echo "${IMAGE_REPO}:${image_tag}"
-}
-
-function build_image() {
- local dockerfile
- dockerfile="$1"
-
- local image_tag image_name dockerfile_dir
- image_tag="$(image_tag "$dockerfile")"
- image_name="$(image_name "$image_tag")"
- dockerfile_dir="$(dirname "$dockerfile")"
-
- echo >&2 "===> Building ${image_tag} image..."
- docker build -t "$image_name" "$dockerfile_dir"
-}
-
-function internal_run() {
- local dockerfile="$1"
- local docker_run_command="$2"
- local args="$3"
-
- local image_tag image_name
- image_tag="$(image_tag "$dockerfile")"
- image_name="$(image_name "$image_tag")"
-
- echo >&2 "===> Starting ${image_tag} ${args}..."
-
- eval "docker run --rm --detach --network $NETWORK_NAME -e JOB_MANAGER_RPC_ADDRESS=jobmanager ${docker_run_command} $image_name ${args}"
-}
-
-function internal_run_jobmanager() {
- internal_run "$1" "--name jobmanager --publish 6123:6123 --publish 8081:8081 $2" jobmanager
-}
-
-function run_jobmanager() {
- internal_run_jobmanager "$1" ""
-}
-
-function run_jobmanager_non_root() {
- internal_run_jobmanager "$1" "--user flink"
-}
-
-function internal_run_taskmanager() {
- internal_run "$1" "--name taskmanager $2" "taskmanager"
-}
-
-function run_taskmanager() {
- internal_run_taskmanager "$1" ""
-}
-
-function run_taskmanager_non_root() {
- internal_run_taskmanager "$1" "--user flink"
-}
-
-function wait_for_jobmanager() {
- local dockerfile
- dockerfile="$1"
-
- local image_tag
- image_tag="$(image_tag "$dockerfile")"
-
- i=0
- echo >&2 "===> Waiting for ${image_tag} jobmanager to be ready..."
- while true; do
- i=$((i+1))
-
- set +e
-
- curl \
- --silent \
- --max-time "$CURL_TIMEOUT" \
- "$CURL_ENDPOINT" \
- > /dev/null
-
- result=$?
-
- set -e
-
- if [ "$result" -eq 0 ]; then
- break
- fi
-
- if [ "$i" -gt "$CURL_MAX_TRIES" ]; then
- echo >&2 "===> \$CURL_MAX_TRIES exceeded waiting for jobmanager to be ready"
- return 1
- fi
-
- sleep "$CURL_COOLDOWN"
- done
-
- echo >&2 "===> ${image_tag} jobmanager is ready."
-}
-
-function test_image() {
- local dockerfile
- dockerfile="$1"
-
- local image_tag
- image_tag="$(image_tag "$dockerfile")"
-
- i=0
- echo >&2 "===> Waiting for ${image_tag} taskmanager to connect..."
- while true; do
- i=$((i+1))
-
- set +e
-
- local overview
- overview="$(curl \
- --silent \
- --max-time "$CURL_TIMEOUT" \
- "$CURL_ENDPOINT")"
-
- num_taskmanagers="$(echo "$overview" | jq .taskmanagers)"
-
- set -e
-
- if [ "$num_taskmanagers" = "1" ]; then
- break
- fi
-
- if [ "$i" -gt "$CURL_MAX_TRIES" ]; then
- echo >&2 "===> \$CURL_MAX_TRIES exceeded for taskmanager to connect"
- return 1
- fi
-
- sleep "$CURL_COOLDOWN"
- done
- echo >&2 "===> ${image_tag} taskmanager connected."
-}
-
-function create_network() {
- docker network create "$NETWORK_NAME" > /dev/null
-}
-
-# Find and kill any remaining containers attached to the network, then remove
-# the network and any images produced by the build.
-function cleanup() {
- local containers
- containers="$(docker ps --quiet --filter network="$NETWORK_NAME")"
-
- if [ -n "$containers" ]; then
- echo >&2 -n "==> Killing $(echo -n "$containers" | grep -c '^') orphaned container(s)..."
- echo "$containers" | xargs docker kill > /dev/null
- echo >&2 " done."
- fi
-
- docker network rm "$NETWORK_NAME" > /dev/null 2>&1 || true
-
- local images
- images="$(docker images --quiet --filter reference="$IMAGE_REPO")"
-
- if [ -n "$images" ]; then
- echo >&2 -n "==> Deleting $(echo -n "$images" | grep -c '^') image(s)..."
- echo "$images" | xargs docker rmi > /dev/null
- echo >&2 " done."
- fi
-}
-
-# For each image, run a jobmanager and taskmanager and verify they start up and connect to each
-# other successfully.
-function smoke_test_all_images() {
- create_network
- trap cleanup EXIT RETURN
-
- local jobmanager_container_id
- local taskmanager_container_id
- local dockerfiles
- dockerfiles="$(ls ./*/*/Dockerfile)"
-
- echo >&2 "==> Test all images"
-
- for dockerfile in $dockerfiles; do
- build_image "$dockerfile"
- jobmanager_container_id="$(run_jobmanager "$dockerfile")"
- taskmanager_container_id="$(run_taskmanager "$dockerfile")"
- wait_for_jobmanager "$dockerfile"
- test_image "$dockerfile"
- docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null
- done
-}
-
-# Same as smoke_test_all_images, but test only the last image alphabetically (presumed to be the
-# most recent).
-function smoke_test_one_image() {
- create_network
- trap cleanup EXIT RETURN
-
- local jobmanager_container_id
- local taskmanager_container_id
- local dockerfiles
- dockerfiles="$dockerfiles $(ls ./*/*/Dockerfile | tail -n 1)"
-
- echo >&2 "==> Test one image"
-
- for dockerfile in $dockerfiles; do
- build_image "$dockerfile"
- jobmanager_container_id="$(run_jobmanager "$dockerfile")"
- taskmanager_container_id="$(run_taskmanager "$dockerfile")"
- wait_for_jobmanager "$dockerfile"
- test_image "$dockerfile"
- docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null
- done
-}
-
-# Similar to smoke_test_one_image, but test one debian image and one alpine image running as a
-# non-root user.
-function smoke_test_non_root() {
- create_network
- trap cleanup EXIT RETURN
-
- local jobmanager_container_id
- local taskmanager_container_id
- local dockerfiles
- dockerfiles="$dockerfiles $(ls ./*/*-debian/Dockerfile | tail -n 1)"
- dockerfiles="$dockerfiles $(ls ./*/*-alpine/Dockerfile | tail -n 1)"
-
- echo >&2 "==> Test images running as non-root"
-
- for dockerfile in $dockerfiles; do
- build_image "$dockerfile"
- jobmanager_container_id="$(run_jobmanager_non_root "$dockerfile")"
- taskmanager_container_id="$(run_taskmanager_non_root "$dockerfile")"
- wait_for_jobmanager "$dockerfile"
- test_image "$dockerfile"
- docker kill "$jobmanager_container_id" "$taskmanager_container_id" > /dev/null
- done
-}
-
-# vim: ts=4 sw=4 et