[airflow] branch main updated: Dev: Allow easy repacking of Providers (#22018)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 4e05bbc  Dev: Allow easy repacking of Providers (#22018)
4e05bbc is described below

commit 4e05bbc92557f5eafaa37509387b2fb2b0ab3d4a
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Mon Mar 7 02:07:56 2022 +0000

    Dev: Allow easy repacking of Providers (#22018)
    
    * Dev: Allow easy repacking of Providers
    
    This PR adds `--skip-tag-check` to easily repackage a provider by running:
    
    ```
    rm -rf provider_packages/airflow
    cp -r airflow provider_packages
    mkdir dist
    python dev/provider_packages/prepare_provider_packages.py generate-setup-files --skip-tag-check "cncf.kubernetes"
    python dev/provider_packages/prepare_provider_packages.py build-provider-packages --skip-tag-check "cncf.kubernetes"
    ```
    
    This allow us to repackage providers without using Docker containers/breeze or git.
---
 dev/provider_packages/prepare_provider_packages.py | 42 +++++++++++++---------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 2b44dd9..152ae42 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -104,7 +104,6 @@ sys.path.insert(0, str(SOURCE_DIR_PATH))
 # those imports need to come after the above sys.path.insert to make sure that Airflow
 # sources are importable without having to add the airflow sources to the PYTHONPATH before
 # running the script
-import tests.deprecated_classes  # noqa # isort:skip
 from dev.import_all_classes import import_all_classes  # noqa # isort:skip
 from setup import PROVIDERS_REQUIREMENTS, PREINSTALLED_PROVIDERS  # noqa # isort:skip
 
@@ -123,6 +122,13 @@ def cli():
     ...
 
 
+option_skip_tag_check = click.option(
+    "--skip-tag-check/--no-skip-tag-check",
+    default=False,
+    is_flag=True,
+    help="Skip checking if the tag already exists in the remote repository",
+)
+
 option_git_update = click.option(
     '--git-update/--no-git-update',
     default=True,
@@ -1880,7 +1886,10 @@ def tag_exists_for_version(provider_package_id: str, current_tag: str, verbose:
 @option_git_update
 @argument_package_id
 @option_verbose
-def generate_setup_files(version_suffix: str, git_update: bool, package_id: str, verbose: bool):
+@option_skip_tag_check
+def generate_setup_files(
+    version_suffix: str, git_update: bool, package_id: str, verbose: bool, skip_tag_check: bool
+):
     """
     Generates setup files for the package.
 
@@ -1888,20 +1897,17 @@ def generate_setup_files(version_suffix: str, git_update: bool, package_id: str,
     """
     provider_package_id = package_id
     with with_group(f"Generate setup files for '{provider_package_id}'"):
-        current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
-        if tag_exists_for_version(provider_package_id, current_tag, verbose):
-            console.print(f"[yellow]The tag {current_tag} exists. Not preparing the package.[/]")
-            # Returns 1 in case of skipped package
-            sys.exit(1)
+        if not skip_tag_check:
+            current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
+            if tag_exists_for_version(provider_package_id, current_tag, verbose):
+                console.print(f"[yellow]The tag {current_tag} exists. Not preparing the package.[/]")
+                # Returns 1 in case of skipped package
+                sys.exit(1)
+        if update_setup_files(provider_package_id, version_suffix):
+            console.print(f"[green]Generated regular package setup files for {provider_package_id}[/]")
         else:
-            if update_setup_files(
-                provider_package_id,
-                version_suffix,
-            ):
-                console.print(f"[green]Generated regular package setup files for {provider_package_id}[/]")
-            else:
-                # Returns 64 in case of skipped package
-                sys.exit(64)
+            # Returns 64 in case of skipped package
+            sys.exit(64)
 
 
 def get_current_tag(provider_package_id: str, suffix: str, git_update: bool, verbose: bool):
@@ -1952,12 +1958,14 @@ def verify_setup_py_prepared(provider_package):
 @option_version_suffix
 @argument_package_id
 @option_verbose
+@option_skip_tag_check
 def build_provider_packages(
     package_format: str,
     git_update: bool,
     version_suffix: str,
     package_id: str,
     verbose: bool,
+    skip_tag_check: bool,
 ):
     """
     Builds provider package.
@@ -1974,7 +1982,7 @@ def build_provider_packages(
     try:
         provider_package_id = package_id
         with with_group(f"Prepare provider package for '{provider_package_id}'"):
-            if version_suffix.startswith("rc") or version_suffix == "":
+            if not skip_tag_check and (version_suffix.startswith("rc") or version_suffix == ""):
                 # For RC and official releases we check if the "officially released" version exists
                 # and skip the released if it was. This allows to skip packages that have not been
                 # marked for release. For "dev" suffixes, we always build all packages
@@ -1982,7 +1990,7 @@ def build_provider_packages(
                 if tag_exists_for_version(provider_package_id, released_tag, verbose):
                     console.print(f"[yellow]The tag {released_tag} exists. Skipping the package.[/]")
                     return False
-            console.print(f"Changing directory to ${TARGET_PROVIDER_PACKAGES_PATH}")
+            console.print(f"Changing directory to {TARGET_PROVIDER_PACKAGES_PATH}")
             os.chdir(TARGET_PROVIDER_PACKAGES_PATH)
             cleanup_remnants(verbose)
             provider_package = package_id