You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/03/03 09:37:48 UTC
[airflow] 22/28: Remove reinstalling azure-storage steps from CI /
Breeze (#14102)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cac851c59dbc787b9f96c44767cbb7510d740fc6
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Mon Feb 8 21:56:46 2021 +0000
Remove reinstalling azure-storage steps from CI / Breeze (#14102)
Since https://github.com/apache/airflow/pull/12188 was merged I
don't think we need this steps.
This step also caused the docker build step for 2.0.1rc2 to fail
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
(cherry picked from commit 3ffd21745d25e6239254fe3f5688b34f5f6f77e8)
---
scripts/docker/install_airflow.sh | 7 +------
scripts/in_container/_in_container_utils.sh | 22 ++++------------------
scripts/in_container/entrypoint_ci.sh | 4 ++--
scripts/in_container/run_ci_tests.sh | 2 --
.../run_install_and_test_provider_packages.sh | 5 ++---
.../run_prepare_provider_documentation.sh | 1 -
setup.py | 18 +++++-------------
7 files changed, 14 insertions(+), 45 deletions(-)
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
index bfe88be..5f1e9d9 100755
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -66,9 +66,7 @@ function install_airflow() {
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
- # Work around to install azure-storage-blob
- pip uninstall azure-storage azure-storage-blob azure-storage-file --yes
- pip install azure-storage-blob azure-storage-file
+
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
@@ -85,9 +83,6 @@ function install_airflow() {
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
- # Work around to install azure-storage-blob
- pip uninstall azure-storage azure-storage-blob azure-storage-file --yes
- pip install azure-storage-blob azure-storage-file
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 0a9db95..1e9a192 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -275,7 +275,7 @@ function install_airflow_from_wheel() {
>&2 echo
exit 4
fi
- pip install "${airflow_package}${1}"
+ pip install "${airflow_package}${extras}"
}
function install_airflow_from_sdist() {
@@ -292,20 +292,7 @@ function install_airflow_from_sdist() {
>&2 echo
exit 4
fi
- pip install "${airflow_package}${1}"
-}
-
-function reinstall_azure_storage_blob() {
- group_start "Reinstalls azure-storage-blob (temporary workaround)"
- # Reinstall azure-storage-blob here until https://github.com/apache/airflow/pull/12188 is solved
- # Azure-storage-blob need to be reinstalled to overwrite azure-storage-blob installed by old version
- # of the `azure-storage` library
- echo
- echo "Reinstalling azure-storage-blob"
- echo
- pip uninstall azure-storage azure-storage-blob azure-storage-file --yes
- pip install azure-storage-blob azure-storage-file --no-deps --force-reinstall
- group_end
+ pip install "${airflow_package}${extras}"
}
function install_remaining_dependencies() {
@@ -338,13 +325,12 @@ function uninstall_airflow_and_providers() {
function install_released_airflow_version() {
local version="${1}"
- local extras="${2}"
echo
- echo "Installing released ${version} version of airflow with extras ${extras}"
+ echo "Installing released ${version} version of airflow without extras"
echo
rm -rf "${AIRFLOW_SOURCES}"/*.egg-info
- pip install --upgrade "apache-airflow${extras}==${version}"
+ pip install --upgrade "apache-airflow==${version}"
}
function install_local_airflow_with_eager_upgrade() {
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index 3761a3b..b99cdc1 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -98,9 +98,9 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then
uninstall_providers
else
echo
- echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras"
+ echo "Install airflow from PyPI without extras"
echo
- install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]"
+ install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}"
fi
if [[ ${INSTALL_PACKAGES_FROM_DIST=} == "true" ]]; then
echo
diff --git a/scripts/in_container/run_ci_tests.sh b/scripts/in_container/run_ci_tests.sh
index 43be453..ca3c41d 100755
--- a/scripts/in_container/run_ci_tests.sh
+++ b/scripts/in_container/run_ci_tests.sh
@@ -18,8 +18,6 @@
# shellcheck source=scripts/in_container/_in_container_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-reinstall_azure_storage_blob
-
echo
echo "Starting the tests with those pytest arguments:" "${@}"
echo
diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh
index 9b951c7..76d41e4 100755
--- a/scripts/in_container/run_install_and_test_provider_packages.sh
+++ b/scripts/in_container/run_install_and_test_provider_packages.sh
@@ -67,9 +67,9 @@ function install_airflow_as_specified() {
uninstall_providers
else
echo
- echo "Install airflow from PyPI including [${AIRFLOW_EXTRAS}] extras"
+ echo "Install airflow from PyPI without extras"
echo
- install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[${AIRFLOW_EXTRAS}]"
+ install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}"
uninstall_providers
fi
group_end
@@ -197,7 +197,6 @@ setup_provider_packages
verify_parameters
install_airflow_as_specified
install_remaining_dependencies
-reinstall_azure_storage_blob
install_provider_packages
import_all_provider_classes
diff --git a/scripts/in_container/run_prepare_provider_documentation.sh b/scripts/in_container/run_prepare_provider_documentation.sh
index e88cdfc..1a0bfa8 100755
--- a/scripts/in_container/run_prepare_provider_documentation.sh
+++ b/scripts/in_container/run_prepare_provider_documentation.sh
@@ -100,7 +100,6 @@ install_supported_pip_version
# install extra packages missing in devel_ci
# TODO: remove it when devel_all == devel_ci
install_remaining_dependencies
-reinstall_azure_storage_blob
if [[ ${BACKPORT_PACKAGES} != "true" ]]; then
import_all_provider_classes
diff --git a/setup.py b/setup.py
index cd38ef2..a752d82 100644
--- a/setup.py
+++ b/setup.py
@@ -219,6 +219,8 @@ azure = [
'azure-mgmt-containerinstance>=1.5.0,<2.0',
'azure-mgmt-datalake-store>=0.5.0',
'azure-mgmt-resource>=2.2.0',
+ 'azure-storage-blob>=12.7.0',
+ 'azure-storage-common>=2.1.0',
'azure-storage-file>=2.1.0',
]
cassandra = [
@@ -423,19 +425,9 @@ slack = [
'slack_sdk>=3.0.0,<4.0.0',
]
snowflake = [
- # The `azure` provider uses legacy `azure-storage` library, where `snowflake` uses the
- # newer and more stable versions of those libraries. Most of `azure` operators and hooks work
- # fine together with `snowflake` because the deprecated library does not overlap with the
- # new libraries except the `blob` classes. So while `azure` works fine for most cases
- # blob is the only exception
- # Solution to that is being worked on in https://github.com/apache/airflow/pull/12188
- # once it is merged, we can move those two back to `azure` extra.
- 'azure-core>=1.10.0',
- 'azure-storage-blob',
- 'azure-storage-common',
- # Snowflake conector > 2.3.8 is needed because it has vendored urrllib3 and requests libraries which
- # are monkey-patched. In earlier versions of the library, monkeypatching the libraries by snowflake
- # caused other providers to fail (Google, Amazon etc.)
+ # Snowflake connector > 2.3.8 is needed because it has vendored-in, patched urllib and requests libraries
+ # In earlier versions of the snowflake library, monkey-patching the libraries caused other
+ # providers to fail (Google, Amazon etc.)
'snowflake-connector-python>=2.3.8',
'snowflake-sqlalchemy>=1.1.0',
]