You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ta...@apache.org on 2023/09/16 17:09:40 UTC
[airflow] branch main updated: Use latest LTS clients of Oracle MySQL and MariaDB (#33722)
This is an automated email from the ASF dual-hosted git repository.
taragolis pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 18716c8b4f Use latest LTS clients of Oracle MySQL and MariaDB (#33722)
18716c8b4f is described below
commit 18716c8b4ff50fcc8c0def4aa20f7c5628d2189d
Author: Andrey Anshin <An...@taragol.is>
AuthorDate: Sat Sep 16 21:09:33 2023 +0400
Use latest LTS clients of Oracle MySQL and MariaDB (#33722)
---
Dockerfile | 87 ++++++++++++++++++++++++++-------------
Dockerfile.ci | 87 ++++++++++++++++++++++++++-------------
scripts/docker/install_mysql.sh | 91 ++++++++++++++++++++++++++++-------------
3 files changed, 180 insertions(+), 85 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 7b896a0a94..2e4a61c2c5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -180,53 +180,67 @@ COPY <<"EOF" /install_mysql.sh
set -euo pipefail
declare -a packages
-MYSQL_VERSION="8.0"
-readonly MYSQL_VERSION
-MARIADB_VERSION="10.5"
-readonly MARIADB_VERSION
+MYSQL_LTS_VERSION="8.0"
+MARIADB_LTS_VERSION="10.11"
+readonly MYSQL_LTS_VERSION
+readonly MARIADB_LTS_VERSION
COLOR_BLUE=$'\e[34m'
readonly COLOR_BLUE
COLOR_YELLOW=$'\e[1;33m'
readonly COLOR_YELLOW
+COLOR_RED=$'\e[1;31m'
+readonly COLOR_RED
COLOR_RESET=$'\e[0m'
readonly COLOR_RESET
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+export_key() {
+ local key="${1}"
+ local name="${2:-mysql}"
+
+ echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}"
+ GNUPGHOME="$(mktemp -d)"
+ export GNUPGHOME
+ set +e
+ for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
+ do
+ gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
+ done
+ set -e
+ gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg"
+ gpgconf --kill all
+ rm -rf "${GNUPGHOME}"
+ unset GNUPGHOME
+}
+
install_mysql_client() {
if [[ "${1}" == "dev" ]]; then
packages=("libmysqlclient-dev" "mysql-client")
elif [[ "${1}" == "prod" ]]; then
+ # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example
+ # 18 - MySQL 5.6.48
+ # 20 - MySQL 5.7.42
+ # 21 - MySQL 8.0.34
+ # 22 - MySQL 8.1
packages=("libmysqlclient21" "mysql-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "467B942D3A79BD29" "mysql"
+
echo
- echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
echo
- local key="467B942D3A79BD29"
- readonly key
-
- GNUPGHOME="$(mktemp -d)"
- export GNUPGHOME
- set +e
- for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
- do
- gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
- done
- set -e
- gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
- gpgconf --kill all
- rm -rf "${GNUPGHOME}"
- unset GNUPGHOME
- echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list
+ echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \
+ /etc/apt/sources.list.d/mysql.list
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
@@ -234,21 +248,38 @@ install_mysql_client() {
}
install_mariadb_client() {
+ # List of compatible package Oracle MySQL -> MariaDB:
+ # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
+ # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`
+ # `libmysqlclient-dev` -> `libmariadb-dev-compat`
+ #
+ # Different naming against Debian repo which we used before
+ # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:
+ # `libmariadb-dev` -> `libmariadb-dev-compat`
+ # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)
if [[ "${1}" == "dev" ]]; then
- packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb-dev-compat" "mariadb-client")
elif [[ "${1}" == "prod" ]]; then
- packages=("mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb3-compat" "mariadb-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "0xF1656F24C74CD1D8" "mariadb"
+
echo
- echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}"
- echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"
echo
+
+ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+ echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \
+ /etc/apt/sources.list.d/mariadb.list
+ # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies
+ printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 55fc05cc0e..af344a877a 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -140,53 +140,67 @@ COPY <<"EOF" /install_mysql.sh
set -euo pipefail
declare -a packages
-MYSQL_VERSION="8.0"
-readonly MYSQL_VERSION
-MARIADB_VERSION="10.5"
-readonly MARIADB_VERSION
+MYSQL_LTS_VERSION="8.0"
+MARIADB_LTS_VERSION="10.11"
+readonly MYSQL_LTS_VERSION
+readonly MARIADB_LTS_VERSION
COLOR_BLUE=$'\e[34m'
readonly COLOR_BLUE
COLOR_YELLOW=$'\e[1;33m'
readonly COLOR_YELLOW
+COLOR_RED=$'\e[1;31m'
+readonly COLOR_RED
COLOR_RESET=$'\e[0m'
readonly COLOR_RESET
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+export_key() {
+ local key="${1}"
+ local name="${2:-mysql}"
+
+ echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}"
+ GNUPGHOME="$(mktemp -d)"
+ export GNUPGHOME
+ set +e
+ for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
+ do
+ gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
+ done
+ set -e
+ gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg"
+ gpgconf --kill all
+ rm -rf "${GNUPGHOME}"
+ unset GNUPGHOME
+}
+
install_mysql_client() {
if [[ "${1}" == "dev" ]]; then
packages=("libmysqlclient-dev" "mysql-client")
elif [[ "${1}" == "prod" ]]; then
+ # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example
+ # 18 - MySQL 5.6.48
+ # 20 - MySQL 5.7.42
+ # 21 - MySQL 8.0.34
+ # 22 - MySQL 8.1
packages=("libmysqlclient21" "mysql-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "467B942D3A79BD29" "mysql"
+
echo
- echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
echo
- local key="467B942D3A79BD29"
- readonly key
-
- GNUPGHOME="$(mktemp -d)"
- export GNUPGHOME
- set +e
- for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
- do
- gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
- done
- set -e
- gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
- gpgconf --kill all
- rm -rf "${GNUPGHOME}"
- unset GNUPGHOME
- echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list
+ echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \
+ /etc/apt/sources.list.d/mysql.list
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
@@ -194,21 +208,38 @@ install_mysql_client() {
}
install_mariadb_client() {
+ # List of compatible package Oracle MySQL -> MariaDB:
+ # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
+ # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`
+ # `libmysqlclient-dev` -> `libmariadb-dev-compat`
+ #
+ # Different naming against Debian repo which we used before
+ # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:
+ # `libmariadb-dev` -> `libmariadb-dev-compat`
+ # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)
if [[ "${1}" == "dev" ]]; then
- packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb-dev-compat" "mariadb-client")
elif [[ "${1}" == "prod" ]]; then
- packages=("mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb3-compat" "mariadb-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "0xF1656F24C74CD1D8" "mariadb"
+
echo
- echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}"
- echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"
echo
+
+ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+ echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \
+ /etc/apt/sources.list.d/mariadb.list
+ # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies
+ printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh
index 89f37c2873..82e5cc420f 100644
--- a/scripts/docker/install_mysql.sh
+++ b/scripts/docker/install_mysql.sh
@@ -19,53 +19,69 @@
set -euo pipefail
declare -a packages
-MYSQL_VERSION="8.0"
-readonly MYSQL_VERSION
-MARIADB_VERSION="10.5"
-readonly MARIADB_VERSION
+# https://dev.mysql.com/blog-archive/introducing-mysql-innovation-and-long-term-support-lts-versions/
+MYSQL_LTS_VERSION="8.0"
+# https://mariadb.org/about/#maintenance-policy
+MARIADB_LTS_VERSION="10.11"
+readonly MYSQL_LTS_VERSION
+readonly MARIADB_LTS_VERSION
COLOR_BLUE=$'\e[34m'
readonly COLOR_BLUE
COLOR_YELLOW=$'\e[1;33m'
readonly COLOR_YELLOW
+COLOR_RED=$'\e[1;31m'
+readonly COLOR_RED
COLOR_RESET=$'\e[0m'
readonly COLOR_RESET
: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+export_key() {
+ local key="${1}"
+ local name="${2:-mysql}"
+
+ echo "${COLOR_BLUE}Verify and export GPG public key ${key}${COLOR_RESET}"
+ GNUPGHOME="$(mktemp -d)"
+ export GNUPGHOME
+ set +e
+ for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
+ keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
+ do
+ gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
+ done
+ set -e
+ gpg --export "${key}" > "/etc/apt/trusted.gpg.d/${name}.gpg"
+ gpgconf --kill all
+ rm -rf "${GNUPGHOME}"
+ unset GNUPGHOME
+}
+
install_mysql_client() {
if [[ "${1}" == "dev" ]]; then
packages=("libmysqlclient-dev" "mysql-client")
elif [[ "${1}" == "prod" ]]; then
+ # `libmysqlclientXX` where XX is number, and it should be increased every new GA MySQL release, for example
+ # 18 - MySQL 5.6.48
+ # 20 - MySQL 5.7.42
+ # 21 - MySQL 8.0.34
+ # 22 - MySQL 8.1
packages=("libmysqlclient21" "mysql-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "467B942D3A79BD29" "mysql"
+
echo
- echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing Oracle MySQL client version ${MYSQL_LTS_VERSION}: ${1}${COLOR_RESET}"
echo
- local key="467B942D3A79BD29"
- readonly key
-
- GNUPGHOME="$(mktemp -d)"
- export GNUPGHOME
- set +e
- for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
- keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
- do
- gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
- done
- set -e
- gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
- gpgconf --kill all
- rm -rf "${GNUPGHOME}"
- unset GNUPGHOME
- echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_VERSION}" > /etc/apt/sources.list.d/mysql.list
+ echo "deb http://repo.mysql.com/apt/debian/ $(lsb_release -cs) mysql-${MYSQL_LTS_VERSION}" > \
+ /etc/apt/sources.list.d/mysql.list
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
@@ -73,21 +89,38 @@ install_mysql_client() {
}
install_mariadb_client() {
+ # List of compatible package Oracle MySQL -> MariaDB:
+ # `mysql-client` -> `mariadb-client` or `mariadb-client-compat` (11+)
+ # `libmysqlclientXX` (where XX is a number) -> `libmariadb3-compat`
+ # `libmysqlclient-dev` -> `libmariadb-dev-compat`
+ #
+ # Different naming against Debian repo which we used before
+ # that some of packages might contains `-compat` suffix, Debian repo -> MariaDB repo:
+ # `libmariadb-dev` -> `libmariadb-dev-compat`
+ # `mariadb-client-core` -> `mariadb-client` or `mariadb-client-compat` (11+)
if [[ "${1}" == "dev" ]]; then
- packages=("libmariadb-dev" "mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb-dev-compat" "mariadb-client")
elif [[ "${1}" == "prod" ]]; then
- packages=("mariadb-client-core-${MARIADB_VERSION}")
+ packages=("libmariadb3-compat" "mariadb-client")
else
echo
- echo "Specify either prod or dev"
+ echo "${COLOR_RED}Specify either prod or dev${COLOR_RESET}"
echo
exit 1
fi
+ export_key "0xF1656F24C74CD1D8" "mariadb"
+
echo
- echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_VERSION}: ${1}${COLOR_RESET}"
- echo "${COLOR_YELLOW}MariaDB client binary compatible with MySQL client.${COLOR_RESET}"
+ echo "${COLOR_BLUE}Installing MariaDB client version ${MARIADB_LTS_VERSION}: ${1}${COLOR_RESET}"
+ echo "${COLOR_YELLOW}MariaDB client protocol-compatible with MySQL client.${COLOR_RESET}"
echo
+
+ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+ echo "deb [arch=amd64,arm64] https://archive.mariadb.org/mariadb-${MARIADB_LTS_VERSION}/repo/debian/ $(lsb_release -cs) main" > \
+ /etc/apt/sources.list.d/mariadb.list
+ # Make sure that dependencies from MariaDB repo are preferred over Debian dependencies
+ printf "Package: *\nPin: release o=MariaDB\nPin-Priority: 999\n" > /etc/apt/preferences.d/mariadb
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
apt-get autoremove -yqq --purge
@@ -96,7 +129,7 @@ install_mariadb_client() {
# Install MySQL client only if it is not disabled.
# For amd64 (x86_64) install MySQL client from Oracle repositories.
-# For arm64 install MariaDB client from Debian repository, see:
+# For arm64 install MariaDB client from MariaDB repository, see:
# https://mariadb.com/kb/en/mariadb-clientserver-tcp-protocol/
if [[ ${INSTALL_MYSQL_CLIENT:="true"} == "true" ]]; then
if [[ $(uname -m) == "arm64" || $(uname -m) == "aarch64" ]]; then