You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by rm...@apache.org on 2020/07/30 11:57:45 UTC
[flink-docker] 01/01: [FLINK-16260] Generate release.metadata file
This is an automated email from the ASF dual-hosted git repository.
rmetzger pushed a commit to branch dev-master
in repository https://gitbox.apache.org/repos/asf/flink-docker.git
commit 15491755aef85694adafe673a98196e7754e628b
Author: Robert Metzger <rm...@apache.org>
AuthorDate: Mon Jul 27 15:28:24 2020 +0200
[FLINK-16260] Generate release.metadata file
This closes #32
---
add-custom.sh | 14 ++++---
add-version.sh | 20 ++++++----
common.sh | 35 ------------------
generator.sh | 90 +++++++++++++++++++++++++++++++++++++++++++++
testing/run_travis_tests.sh | 19 +++++-----
5 files changed, 122 insertions(+), 56 deletions(-)
diff --git a/add-custom.sh b/add-custom.sh
index c863f3f..e8e2a5e 100755
--- a/add-custom.sh
+++ b/add-custom.sh
@@ -4,16 +4,17 @@
# Flink distribution.
# This is exlusively for development purposes.
-source "$(dirname "$0")"/common.sh
+source "$(dirname "$0")"/generator.sh
function usage() {
- echo >&2 "usage: $0 -u binary-download-url [-n name]"
+ echo >&2 "usage: $0 -u binary-download-url [-n name] [-j java_version]"
}
binary_download_url=
name=custom
+java_version=8
-while getopts u:n:h arg; do
+while getopts u:n:j:h arg; do
case "$arg" in
u)
binary_download_url=$OPTARG
@@ -21,6 +22,9 @@ while getopts u:n:h arg; do
n)
name=$OPTARG
;;
+ j)
+ java_version=$OPTARG
+ ;;
h)
usage
exit 0
@@ -43,7 +47,7 @@ echo -n >&2 "Generating Dockerfiles..."
for source_variant in "${SOURCE_VARIANTS[@]}"; do
dir="dev/${name}-${source_variant}"
rm -rf "${dir}"
-
- generate "${dir}" "${binary_download_url}" "" "" false ${source_variant}
+ mkdir "$dir"
+ generateDockerfile "${dir}" "${binary_download_url}" "" "" false ${java_version} ${source_variant}
done
echo >&2 " done."
diff --git a/add-version.sh b/add-version.sh
index 48acc79..fd07d92 100755
--- a/add-version.sh
+++ b/add-version.sh
@@ -10,7 +10,6 @@
#
# See other repos (e.g. httpd, cassandra) for update.sh examples.
-source "$(dirname "$0")"/common.sh
function usage() {
echo >&2 "usage: $0 -r flink-release -f flink-version"
@@ -60,6 +59,7 @@ fi
# Defaults, can vary between versions
scala_versions=( 2.11 2.12 )
+java_versions=( 8 11 )
gpg_key=
# Version-specific variants (example)
@@ -98,18 +98,24 @@ fi
mkdir "$flink_release"
+source "$(dirname "$0")"/generator.sh
+
echo -n >&2 "Generating Dockerfiles..."
for source_variant in "${SOURCE_VARIANTS[@]}"; do
for scala_version in "${scala_versions[@]}"; do
- dir="$flink_release/scala_${scala_version}-${source_variant}"
+ for java_version in "${java_versions[@]}"; do
+ dir="$flink_release/scala_${scala_version}-java${java_version}-${source_variant}"
- flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz
+ flink_url_file_path=flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz
- flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}"
- # Not all mirrors have the .asc files
- flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
+ flink_tgz_url="https://www.apache.org/dyn/closer.cgi?action=download&filename=${flink_url_file_path}"
+ # Not all mirrors have the .asc files
+ flink_asc_url=https://www.apache.org/dist/${flink_url_file_path}.asc
- generate "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${source_variant}
+ mkdir "$dir"
+ generateDockerfile "${dir}" "${flink_tgz_url}" "${flink_asc_url}" ${gpg_key} true ${java_version} ${source_variant}
+ generateReleaseMetadata "${dir}" ${flink_release} ${flink_version} ${scala_version} ${java_version} ${source_variant}
+ done
done
done
echo >&2 " done."
diff --git a/common.sh b/common.sh
deleted file mode 100644
index dcf5052..0000000
--- a/common.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-# Defaults, can vary between versions
-export SOURCE_VARIANTS=(java11-debian debian )
-
-function generate() {
- dir=$1
- binary_download_url=$2
- asc_download_url=$3
- gpg_key=$4
- check_gpg=$5
- source_variant=$6
-
- from_docker_image="openjdk:8-jre"
- if [[ $source_variant =~ "java11" ]] ; then
- from_docker_image="openjdk:11-jre"
- fi
-
- source_file="Dockerfile-debian"
-
- mkdir "$dir"
- cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
-
- # '&' has special semantics in sed replacement patterns
- escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
-
- # generate Dockerfile
- sed \
- -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
- -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
- -e "s/%%GPG_KEY%%/$gpg_key/" \
- -e "s/%%CHECK_GPG%%/${check_gpg}/" \
- -e "s/%%FROM_IMAGE%%/${from_docker_image}/" \
- "$source_file.template" > "$dir/Dockerfile"
-}
diff --git a/generator.sh b/generator.sh
new file mode 100644
index 0000000..8e492f2
--- /dev/null
+++ b/generator.sh
@@ -0,0 +1,90 @@
+#!/bin/bash -e
+
+export SOURCE_VARIANTS=(debian )
+
+export DEFAULT_SCALA="2.12"
+export DEFAULT_JAVA="8"
+
+function generateDockerfile {
+ # define variables
+ dir=$1
+ binary_download_url=$2
+ asc_download_url=$3
+ gpg_key=$4
+ check_gpg=$5
+ java_version=$6
+ source_variant=$7
+
+ from_docker_image="openjdk:${java_version}-jre"
+
+ cp docker-entrypoint.sh "$dir/docker-entrypoint.sh"
+
+ # '&' has special semantics in sed replacement patterns
+ escaped_binary_download_url=$(echo "$binary_download_url" | sed 's/&/\\\&/')
+
+ # generate Dockerfile
+ sed \
+ -e "s,%%BINARY_DOWNLOAD_URL%%,${escaped_binary_download_url}," \
+ -e "s,%%ASC_DOWNLOAD_URL%%,$asc_download_url," \
+ -e "s/%%GPG_KEY%%/$gpg_key/" \
+ -e "s/%%CHECK_GPG%%/${check_gpg}/" \
+ -e "s/%%FROM_IMAGE%%/${from_docker_image}/" \
+ "Dockerfile-$source_variant.template" > "$dir/Dockerfile"
+}
+
+function generateReleaseMetadata {
+ dir=$1
+ flink_release=$2
+ flink_version=$3
+ scala_version=$4
+ java_version=$5
+
+ # docker image tags:
+ java_suffix="-java${java_version}"
+
+ # example "1.2.0-scala_2.11-java11"
+ full_tag=${flink_version}-scala_${scala_version}${java_suffix}
+
+ # example "1.2-scala_2.11-java11"
+ short_tag=${flink_release}-scala_${scala_version}${java_suffix}
+
+ # example "scala_2.12-java11"
+ scala_tag="scala_${scala_version}${java_suffix}"
+
+ tags="$full_tag, $short_tag, $scala_tag"
+
+ if [[ "$java_version" == "$DEFAULT_JAVA" ]]; then
+ # example "1.2.0-scala_2.11"
+ full_tag=${flink_version}-scala_${scala_version}
+
+ # example "1.2-scala_2.11"
+ short_tag=${flink_release}-scala_${scala_version}
+
+ # example "scala_2.12"
+ scala_tag="scala_${scala_version}"
+
+ tags="$tags, $full_tag, $short_tag, $scala_tag"
+ fi
+
+
+ if [[ "$scala_version" == "$DEFAULT_SCALA" ]]; then
+ # we are generating the image for the latest scala version, add:
+ # "1.2.0-java11"
+ # "1.2-java11"
+ # "latest-java11"
+ tags="$tags, ${flink_version}${java_suffix}, ${flink_release}${java_suffix}, latest${java_suffix}"
+
+ if [[ "$java_version" == "$DEFAULT_JAVA" ]]; then
+ # we are generating the image for the default java version, add tags w/o java tag:
+ # "1.2.0"
+ # "1.2"
+ # "latest"
+ tags="$tags, ${flink_version}, ${flink_release}, latest"
+ fi
+ fi
+
+ echo "Tags: $tags" >> $dir/release.metadata
+
+ # We currently only support amd64 with Flink.
+ echo "Architectures: amd64" >> $dir/release.metadata
+}
diff --git a/testing/run_travis_tests.sh b/testing/run_travis_tests.sh
index a1ae8e4..0ee578a 100755
--- a/testing/run_travis_tests.sh
+++ b/testing/run_travis_tests.sh
@@ -11,14 +11,15 @@ fi
BRANCH="$TRAVIS_BRANCH"
-./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz"
-
-if [ -z "$IS_PULL_REQUEST" ] && [ "$BRANCH" = "dev-master" ]; then
- smoke_test_all_images
- smoke_test_one_image_non_root
-else
- # For pull requests and branches, test one image
- smoke_test_one_image
-fi
+./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz" -n test-java8
+
+# test Flink with Java11 image as well
+./add-custom.sh -u "https://s3.amazonaws.com/flink-nightly/flink-1.11-SNAPSHOT-bin-hadoop2.tgz" -j 11 -n test-java11
+
+smoke_test_all_images
+smoke_test_one_image_non_root
+
+
+echo "Test successfully finished"
# vim: et ts=2 sw=2