[airflow] 27/29: Fixes "development" and "rc" cross dependencies between providers (#17023)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 61dc4cacacf1478076ab0a2fb775df684591c1b0
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu Jul 15 16:17:18 2021 +0200

    Fixes "development" and "rc" cross dependencies between providers (#17023)
    
    In case we have additional dependencies between providers released
    at the same time (for example we need to release sftp and ssh
    packages now where sftp package depends on release of ssh
    at the same time) we have to add suffix to the version of the
    additional_dependency.
    
    PIP does not take into account unfortunately that development
    dependencies should likely be considered as fulfilling the
    requirement of >=. For example if you have:
    
    sftp depends on ssh>=2.1.0 and you release ssh 2.1.0.dev0 at
    the same time the ssh>=2.1.0 condition is not fulfilled.
    
    Same case will be with rc1. Therefore we need to add the suffix in such
    cross-provider dependencies to be able to install them in CI
    and in rc candidates.
    
    In the future we might ask PIP to change behaviour in such case.
    
    (cherry picked from commit 07e0a67bf641d25b0237b457f2585788fa4c8ada)
---
 dev/provider_packages/prepare_provider_packages.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 16bd046..e36fcf8 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -357,11 +357,12 @@ def get_long_description(provider_package_id: str) -> str:
     return long_description
 
 
-def get_install_requirements(provider_package_id: str) -> List[str]:
+def get_install_requirements(provider_package_id: str, version_suffix: str) -> List[str]:
     """
     Returns install requirements for the package.
 
     :param provider_package_id: id of the provider package
+    :param version_suffix: optional version suffix for packages
 
     :return: install requirements of the package
     """
@@ -369,7 +370,21 @@ def get_install_requirements(provider_package_id: str) -> List[str]:
     provider_yaml = get_provider_yaml(provider_package_id)
     install_requires = []
     if "additional-dependencies" in provider_yaml:
-        install_requires = provider_yaml['additional-dependencies']
+        additional_dependencies = provider_yaml['additional-dependencies']
+        if version_suffix:
+            # In case we are preparing "rc" or dev0 packages, we should also
+            # make sure that cross-dependency with Airflow or Airflow Providers will
+            # contain the version suffix, otherwise we will have conflicting dependencies.
+            # For example if (in sftp) we have ssh>=2.0.1 and release ssh==2.0.1
+            # we want to turn this into ssh>=2.0.1.dev0 if we build dev0 version of the packages
+            # or >=2.0.1rc1 if we build rc1 version of the packages.
+            for dependency in additional_dependencies:
+                if dependency.startswith("apache-airflow") and ">=" in dependency:
+                    dependency = dependency + version_suffix
+                install_requires.append(dependency)
+        else:
+            install_requires.extend(additional_dependencies)
+
     install_requires.extend(dependencies)
     return install_requires
 
@@ -1513,7 +1528,7 @@ def get_provider_jinja_context(
         "PROVIDERS_FOLDER": "providers",
         "PROVIDER_DESCRIPTION": provider_details.provider_description,
         "INSTALL_REQUIREMENTS": get_install_requirements(
-            provider_package_id=provider_details.provider_package_id
+            provider_package_id=provider_details.provider_package_id, version_suffix=version_suffix
         ),
         "SETUP_REQUIREMENTS": get_setup_requirements(),
         "EXTRAS_REQUIREMENTS": get_package_extras(provider_package_id=provider_details.provider_package_id),