You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/05/31 17:47:16 UTC
[airflow] branch master updated: Uses bind volume instead of docker
volume for MSSQL docker in tmpfs (#16159)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 352fefa Uses bind volume instead of docker volume for MSSQL docker in tmpfs (#16159)
352fefa is described below
commit 352fefaef1712bf5e60e3e79a86214279be69e16
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon May 31 19:46:59 2021 +0200
Uses bind volume instead of docker volume for MSSQL docker in tmpfs (#16159)
Seems that MSSQL is not able to use data volume when it is mounted
from tmpfs filesystem. See https://github.com/microsoft/mssql-docker/issues/13
In such case, instead of mounting docker-created volume we mount
a volume mounted from home directory of the user which is unlikely
to be a tmpfs volume.
---
breeze | 23 ++++++++++++++++++
...end-mssql.yml => backend-mssql-bind-volume.yml} | 28 ++++++----------------
...d-mssql.yml => backend-mssql-docker-volume.yml} | 20 ----------------
scripts/ci/docker-compose/backend-mssql.yml | 2 --
scripts/ci/libraries/_initialization.sh | 14 ++++++++++-
.../ci_run_single_airflow_test_in_docker.sh | 26 +++++++++++++++++++-
6 files changed, 68 insertions(+), 45 deletions(-)
diff --git a/breeze b/breeze
index a257ee7..792e7cf 100755
--- a/breeze
+++ b/breeze
@@ -673,6 +673,22 @@ function breeze::prepare_command_files() {
local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
+ if [[ ${BACKEND} == "mssql" ]]; then
+ local docker_filesystem
+ docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker || echo "unknown")
+ if [[ ${docker_filesystem} == "tmpfs" ]]; then
+ # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+ # O_DIRECT parameter for direct writing to the filesystem
+ # https://github.com/microsoft/mssql-docker/issues/13
+ # so we need to mount an external volume for its db location
+ # specified by MSSQL_DATA_VOLUME
+ backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-bind-volume.yml"
+ else
+ backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml"
+ fi
+ fi
+
+
local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
local compose_prod_file=${main_prod_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
@@ -1422,6 +1438,13 @@ function breeze::parse_arguments() {
INTEGRATIONS+=("${INTEGRATION}")
fi
done
+ # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+ # O_DIRECT parameter for direct writing to the filesystem
+ # https://github.com/microsoft/mssql-docker/issues/13
+ # so we need to mount an external volume for its db location
+ # the external db must allow for parallel testing so external volume is mapped
+ # to the data volume. Stop should also clean the volume
+ rm -rf "${MSSQL_DATA_VOLUME:?"MSSQL_DATA_VOLUME should never be empty!"}"/*
shift
;;
restart)
diff --git a/scripts/ci/docker-compose/backend-mssql.yml b/scripts/ci/docker-compose/backend-mssql-bind-volume.yml
similarity index 50%
copy from scripts/ci/docker-compose/backend-mssql.yml
copy to scripts/ci/docker-compose/backend-mssql-bind-volume.yml
index b4574ef..7c827a4 100644
--- a/scripts/ci/docker-compose/backend-mssql.yml
+++ b/scripts/ci/docker-compose/backend-mssql-bind-volume.yml
@@ -17,26 +17,12 @@
---
version: "2.2"
services:
- airflow:
- environment:
- - BACKEND=mssql
- - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mssql+pyodbc://sa:Airflow123@mssql:1433/master?driver=ODBC+Driver+17+for+SQL+Server
- - AIRFLOW__CELERY__RESULT_BACKEND=db+mssql+pyodbc://sa:Airflow123@mssql:1433/master?driver=ODBC+Driver+17+for+SQL+Server
- - AIRFLOW__CORE__EXECUTOR=LocalExecutor
- depends_on:
- mssql:
- condition: service_healthy
mssql:
- image: mcr.microsoft.com/mssql/server:${MSSQL_VERSION}
- environment:
- - ACCEPT_EULA=Y
- - SA_PASSWORD=Airflow123
volumes:
- - mssql-db-volume:/var/opt/mssql
- healthcheck:
- test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost",
- "-U", "sa", "-P", "Airflow123", "-Q", "SELECT 1"]
- interval: 10s
- timeout: 10s
- retries: 10
- restart: always
+ # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+ # O_DIRECT parameter for direct writing to the filesystem
+ # https://github.com/microsoft/mssql-docker/issues/13
+ # so we need to mount an external volume for its db location
+ # the external db must allow for parallel testing so external volume is mapped
+ # to the data volume
+ - ${MSSQL_DATA_VOLUME}:/var/opt/mssql
diff --git a/scripts/ci/docker-compose/backend-mssql.yml b/scripts/ci/docker-compose/backend-mssql-docker-volume.yml
similarity index 53%
copy from scripts/ci/docker-compose/backend-mssql.yml
copy to scripts/ci/docker-compose/backend-mssql-docker-volume.yml
index b4574ef..9368d56 100644
--- a/scripts/ci/docker-compose/backend-mssql.yml
+++ b/scripts/ci/docker-compose/backend-mssql-docker-volume.yml
@@ -17,26 +17,6 @@
---
version: "2.2"
services:
- airflow:
- environment:
- - BACKEND=mssql
- - AIRFLOW__CORE__SQL_ALCHEMY_CONN=mssql+pyodbc://sa:Airflow123@mssql:1433/master?driver=ODBC+Driver+17+for+SQL+Server
- - AIRFLOW__CELERY__RESULT_BACKEND=db+mssql+pyodbc://sa:Airflow123@mssql:1433/master?driver=ODBC+Driver+17+for+SQL+Server
- - AIRFLOW__CORE__EXECUTOR=LocalExecutor
- depends_on:
- mssql:
- condition: service_healthy
mssql:
- image: mcr.microsoft.com/mssql/server:${MSSQL_VERSION}
- environment:
- - ACCEPT_EULA=Y
- - SA_PASSWORD=Airflow123
volumes:
- mssql-db-volume:/var/opt/mssql
- healthcheck:
- test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost",
- "-U", "sa", "-P", "Airflow123", "-Q", "SELECT 1"]
- interval: 10s
- timeout: 10s
- retries: 10
- restart: always
diff --git a/scripts/ci/docker-compose/backend-mssql.yml b/scripts/ci/docker-compose/backend-mssql.yml
index b4574ef..06648f4 100644
--- a/scripts/ci/docker-compose/backend-mssql.yml
+++ b/scripts/ci/docker-compose/backend-mssql.yml
@@ -31,8 +31,6 @@ services:
environment:
- ACCEPT_EULA=Y
- SA_PASSWORD=Airflow123
- volumes:
- - mssql-db-volume:/var/opt/mssql
healthcheck:
test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost",
"-U", "sa", "-P", "Airflow123", "-Q", "SELECT 1"]
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 99e6393..1fec58a 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -41,12 +41,24 @@ function initialization::create_directories() {
# As well as hashes of the important files, but also we generate build scripts there that are
# Used to execute the commands for breeze
export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
- export BUILD_CACHE_DIR
readonly BUILD_CACHE_DIR
+ # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+ # O_DIRECT parameter for direct writing to the filesystem
+ # https://github.com/microsoft/mssql-docker/issues/13
+ # so we need to mount an external volume for its db location
+ # the external db must allow for parallel testing so external volume is mapped
+ # to the data volume
+ export MSSQL_DATA_VOLUME="${BUILD_CACHE_DIR}/tmp_mssql_volume"
+
# Create those folders above in case they do not exist
mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
mkdir -p "${FILES_DIR}" >/dev/null
+ mkdir -p "${MSSQL_DATA_VOLUME}" >/dev/null
+ # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable
+ # This is a bit scary and we could get by making it group-writeable but the group would have
+ # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo
+ chmod a+rwx "${MSSQL_DATA_VOLUME}"
# By default we are not in CI environment GitHub Actions sets CI to "true"
export CI="${CI="false"}"
diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
index 012493d..45bbb9b 100755
--- a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
+++ b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
@@ -82,6 +82,29 @@ function run_airflow_testing_in_docker() {
echo
echo "Semaphore grabbed. Running tests for ${TEST_TYPE}"
echo
+ local backend_docker_compose=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml")
+ if [[ ${BACKEND} == "mssql" ]]; then
+ local docker_filesystem
+ docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker || echo "unknown")
+ if [[ ${docker_filesystem} == "tmpfs" ]]; then
+ # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+ # O_DIRECT parameter for direct writing to the filesystem
+ # https://github.com/microsoft/mssql-docker/issues/13
+ # so we need to mount an external volume for its db location
+ # the external db must allow for parallel testing so TEST_TYPE
+ # is added to the volume name
+ export MSSQL_DATA_VOLUME="${HOME}/tmp-mssql-volume-${TEST_TYPE}"
+ mkdir -p "${MSSQL_DATA_VOLUME}"
+ # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable
+ # This is a bit scary and we could get by making it group-writeable but the group would have
+ # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo
+ chmod a+rwx "${MSSQL_DATA_VOLUME}"
+ backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-bind-volume.yml")
+ else
+ backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml")
+ fi
+ fi
+
for try_num in {1..5}
do
echo
@@ -90,13 +113,14 @@ function run_airflow_testing_in_docker() {
echo
echo "Making sure docker-compose is down and remnants removed"
echo
+
docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
--project-name "airflow-${TEST_TYPE}-${BACKEND}" \
down --remove-orphans \
--volumes --timeout 10
docker-compose --log-level INFO \
-f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
- -f "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml" \
+ "${backend_docker_compose[@]}" \
"${INTEGRATIONS[@]}" \
"${DOCKER_COMPOSE_LOCAL[@]}" \
--project-name "airflow-${TEST_TYPE}-${BACKEND}" \