You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/11/28 20:38:06 UTC
(airflow) branch main updated: Remove workaround for pymssql failing compilation with new Cython (#35924)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4f060a482c Remove workaround for pymssql failing compilation with new Cython (#35924)
4f060a482c is described below
commit 4f060a482c3233504e7905b3ab2d00fe56ea43cd
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Nov 28 21:37:58 2023 +0100
Remove workaround for pymssql failing compilation with new Cython (#35924)
Recent Cython release caused pymssql package failures when they
were installed on ARM platform. This had been workarounded in
the #32748, but since pymssql as of 2.1.8 supports new Cython, we
can remove the workaround and bump the minimum version of pymsssql.
This also makes it possible to remove the whole MSSQL client section
from the image if we decide to - because this section will only install
the odbc client that has been pre-installed to support MSSQL as
metadata DB for Airflow core.
---
Dockerfile | 42 -------------------------
Dockerfile.ci | 36 ---------------------
airflow/providers/microsoft/mssql/provider.yaml | 2 +-
generated/provider_dependencies.json | 2 +-
scripts/docker/install_mssql.sh | 39 -----------------------
5 files changed, 2 insertions(+), 119 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index b9b358d0c7..7a7cb89225 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -347,8 +347,6 @@ COPY <<"EOF" /install_mssql.sh
#!/usr/bin/env bash
set -euo pipefail
-. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
-
: "${AIRFLOW_PIP_VERSION:?Should be set}"
: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
@@ -384,40 +382,6 @@ function install_mssql_client() {
rm -rf /var/lib/apt/lists/*
apt-get autoremove -yqq --purge
apt-get clean && rm -rf /var/lib/apt/lists/*
-
- # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of
- # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using
- # setuptools, there is no easy way to fix version of Cython used to compile packages.
- #
- # This triggers a problem with newer `pip` versions that have build isolation enabled by default because
- # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have
- # limit on build dependencies, it will use the latest version of them to build that particular package.
- #
- # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to
- # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing
- # the package. Which is precisely what we are doing here.
- #
- # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to
- # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation
- # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip
- # launches a subprocess `pip` to build the package.
- #
- # This is a temporary solution until the issue is resolved in pymssql or Cython
- # Issues/discussions that track it:
- #
- # * https://github.com/cython/cython/issues/5541
- # * https://github.com/pymssql/pymssql/pull/827
- # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833
- #
- # TODO: Remove this workaround when the issue is resolved.
- # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib
- # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove)
- if [[ "${1}" == "dev" ]]; then
- common::install_pip_version
- echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt
- PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql
- rm /tmp/mssql-constraints.txt
- fi
}
install_mssql_client "${@}"
@@ -1272,12 +1236,6 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
# scripts which are needed much later will not invalidate the docker layer here
COPY --from=scripts install_mysql.sh install_mssql.sh install_postgres.sh /scripts/docker/
-# THE 3 LINES ARE ONLY NEEDED IN ORDER TO MAKE PYMSSQL BUILD WORK WITH LATEST CYTHON
-# AND SHOULD BE REMOVED WHEN WORKAROUND IN install_mssql.sh IS REMOVED
-ARG AIRFLOW_PIP_VERSION=23.3.1
-ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION}
-COPY --from=scripts common.sh /scripts/docker/
-
RUN bash /scripts/docker/install_mysql.sh dev && \
bash /scripts/docker/install_mssql.sh dev && \
bash /scripts/docker/install_postgres.sh dev
diff --git a/Dockerfile.ci b/Dockerfile.ci
index c3e5e2e3a0..a22c363fa5 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -307,8 +307,6 @@ COPY <<"EOF" /install_mssql.sh
#!/usr/bin/env bash
set -euo pipefail
-. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
-
: "${AIRFLOW_PIP_VERSION:?Should be set}"
: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
@@ -344,40 +342,6 @@ function install_mssql_client() {
rm -rf /var/lib/apt/lists/*
apt-get autoremove -yqq --purge
apt-get clean && rm -rf /var/lib/apt/lists/*
-
- # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of
- # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using
- # setuptools, there is no easy way to fix version of Cython used to compile packages.
- #
- # This triggers a problem with newer `pip` versions that have build isolation enabled by default because
- # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have
- # limit on build dependencies, it will use the latest version of them to build that particular package.
- #
- # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to
- # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing
- # the package. Which is precisely what we are doing here.
- #
- # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to
- # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation
- # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip
- # launches a subprocess `pip` to build the package.
- #
- # This is a temporary solution until the issue is resolved in pymssql or Cython
- # Issues/discussions that track it:
- #
- # * https://github.com/cython/cython/issues/5541
- # * https://github.com/pymssql/pymssql/pull/827
- # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833
- #
- # TODO: Remove this workaround when the issue is resolved.
- # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib
- # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove)
- if [[ "${1}" == "dev" ]]; then
- common::install_pip_version
- echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt
- PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql
- rm /tmp/mssql-constraints.txt
- fi
}
install_mssql_client "${@}"
diff --git a/airflow/providers/microsoft/mssql/provider.yaml b/airflow/providers/microsoft/mssql/provider.yaml
index ce4bb0ceb3..934a809171 100644
--- a/airflow/providers/microsoft/mssql/provider.yaml
+++ b/airflow/providers/microsoft/mssql/provider.yaml
@@ -48,7 +48,7 @@ versions:
dependencies:
- apache-airflow>=2.5.0
- apache-airflow-providers-common-sql>=1.3.1
- - pymssql>=2.1.5
+ - pymssql>=2.1.8
integrations:
- integration-name: Microsoft SQL Server (MSSQL)
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index 89ae8735bc..0ff1e0b32e 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -600,7 +600,7 @@
"deps": [
"apache-airflow-providers-common-sql>=1.3.1",
"apache-airflow>=2.5.0",
- "pymssql>=2.1.5"
+ "pymssql>=2.1.8"
],
"cross-providers-deps": [
"common.sql"
diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh
index 248ca68307..b5c74e44bc 100644
--- a/scripts/docker/install_mssql.sh
+++ b/scripts/docker/install_mssql.sh
@@ -18,11 +18,6 @@
# shellcheck shell=bash
set -euo pipefail
-# REMOVE THOSE 3 LINES BELOW TOGETHER WITH PYMSSQL INSTALLATION BELOW AFTER PYMSSQL
-# INSTALLATION WITH CYTHON 3.0.0 IS FIXED SEE BELOW FOR MORE DETAILS
-# shellcheck source=scripts/docker/common.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
-
: "${AIRFLOW_PIP_VERSION:?Should be set}"
: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
@@ -58,40 +53,6 @@ function install_mssql_client() {
rm -rf /var/lib/apt/lists/*
apt-get autoremove -yqq --purge
apt-get clean && rm -rf /var/lib/apt/lists/*
-
- # Workaround an issue with installing pymssql on ARM architecture triggered by Cython 3.0.0 release as of
- # 18 July 2023. The problem is that pip uses latest Cython to compile pymssql and since we are using
- # setuptools, there is no easy way to fix version of Cython used to compile packages.
- #
- # This triggers a problem with newer `pip` versions that have build isolation enabled by default because
- # There is no (easy) way to pin build dependencies for dependent packages. If a package does not have
- # limit on build dependencies, it will use the latest version of them to build that particular package.
- #
- # The workaround to the problem suggest in the last thread by Pradyun Gedam - pip maintainer - is to
- # use PIP_CONSTRAINT environment variable and constraint the version of Cython used while installing
- # the package. Which is precisely what we are doing here.
- #
- # Note that it does not work if we pass ``--constraint`` option to pip because it will not be passed to
- # the package being build in isolation. The fact that the PIP_CONSTRAINT env variable works in the isolation
- # is a bit of side-effect on how env variables work and that they are passed to subprocesses as pip
- # launches a subprocess `pip` to build the package.
- #
- # This is a temporary solution until the issue is resolved in pymssql or Cython
- # Issues/discussions that track it:
- #
- # * https://github.com/cython/cython/issues/5541
- # * https://github.com/pymssql/pymssql/pull/827
- # * https://discuss.python.org/t/no-way-to-pin-build-dependencies/29833
- #
- # TODO: Remove this workaround when the issue is resolved.
- # ALSO REMOVE THE TOP LINES ABOVE WITH common.sh IMPORT AS WELL AS COPYING common.sh ib
- # Dockerfile AND Dockerfile.ci (look for capital PYMSSQL - there are several places to remove)
- if [[ "${1}" == "dev" ]]; then
- common::install_pip_version
- echo "Cython==0.29.36" >> /tmp/mssql-constraints.txt
- PIP_CONSTRAINT=/tmp/mssql-constraints.txt pip install pymssql
- rm /tmp/mssql-constraints.txt
- fi
}
install_mssql_client "${@}"