You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/11/15 02:39:29 UTC
(airflow) branch move-prepare-providers-packages-to-breeze updated (bd6b41067a -> b43046438f)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch move-prepare-providers-packages-to-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit bd6b41067a Move prepare-provider-packages to be run entirely in Breeze Python
omit 9416085808 Move "prepare-provider-documentation" to Breeze
add 24aca11b52 Change terminal mode to cbreak in `execute_interactive` and handle `SIGINT` (#35602)
add e755b799c2 Move "prepare-provider-documentation" to Breeze (#35586)
add 83257a7b35 Remove all code related to Qubole provider (#35636)
new b43046438f Move prepare-provider-packages to be run entirely in Breeze Python
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (bd6b41067a)
\
N -- N -- N refs/heads/move-prepare-providers-packages-to-breeze (b43046438f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../airflow_providers_bug_report.yml | 1 -
.github/boring-cyborg.yml | 6 -
INTHEWILD.md | 1 -
airflow/contrib/hooks/__init__.py | 6 -
airflow/contrib/operators/__init__.py | 9 -
airflow/contrib/sensors/__init__.py | 5 -
.../providers/qubole/.latest-doc-only-change.txt | 1 -
airflow/providers/qubole/CHANGELOG.rst | 341 ------------------
airflow/providers/qubole/__init__.py | 43 ---
airflow/providers/qubole/hooks/__init__.py | 17 -
airflow/providers/qubole/hooks/qubole.py | 304 ----------------
airflow/providers/qubole/hooks/qubole_check.py | 122 -------
airflow/providers/qubole/operators/__init__.py | 17 -
airflow/providers/qubole/operators/qubole.py | 303 ----------------
airflow/providers/qubole/operators/qubole_check.py | 210 -----------
airflow/providers/qubole/provider.yaml | 85 -----
airflow/providers/qubole/sensors/__init__.py | 17 -
airflow/providers/qubole/sensors/qubole.py | 117 -------
airflow/utils/db.py | 8 -
airflow/utils/process_utils.py | 7 +-
dev/README_RELEASE_PROVIDER_PACKAGES.md | 20 +-
docs/apache-airflow-providers-qubole/changelog.rst | 18 -
docs/apache-airflow-providers-qubole/commits.rst | 390 ---------------------
docs/apache-airflow-providers-qubole/index.rst | 112 ------
.../installing-providers-from-sources.rst | 18 -
.../operators/index.rst | 32 --
.../operators/qubole.rst | 184 ----------
.../operators/qubole_check.rst | 45 ---
docs/apache-airflow-providers-qubole/security.rst | 19 -
docs/apache-airflow/extra-packages-ref.rst | 2 -
docs/integration-logos/qubole/Qubole.png | Bin 1892 -> 0 bytes
setup.py | 1 -
tests/providers/qubole/__init__.py | 17 -
tests/providers/qubole/hooks/__init__.py | 17 -
tests/providers/qubole/hooks/test_qubole.py | 80 -----
tests/providers/qubole/hooks/test_qubole_check.py | 42 ---
tests/providers/qubole/operators/__init__.py | 17 -
tests/providers/qubole/operators/test_qubole.py | 179 ----------
.../qubole/operators/test_qubole_check.py | 192 ----------
tests/providers/qubole/sensors/__init__.py | 17 -
tests/providers/qubole/sensors/test_qubole.py | 80 -----
tests/system/providers/qubole/__init__.py | 16 -
tests/system/providers/qubole/example_qubole.py | 235 -------------
.../providers/qubole/example_qubole_sensors.py | 94 -----
44 files changed, 15 insertions(+), 3432 deletions(-)
delete mode 100644 airflow/providers/qubole/.latest-doc-only-change.txt
delete mode 100644 airflow/providers/qubole/CHANGELOG.rst
delete mode 100644 airflow/providers/qubole/__init__.py
delete mode 100644 airflow/providers/qubole/hooks/__init__.py
delete mode 100644 airflow/providers/qubole/hooks/qubole.py
delete mode 100644 airflow/providers/qubole/hooks/qubole_check.py
delete mode 100644 airflow/providers/qubole/operators/__init__.py
delete mode 100644 airflow/providers/qubole/operators/qubole.py
delete mode 100644 airflow/providers/qubole/operators/qubole_check.py
delete mode 100644 airflow/providers/qubole/provider.yaml
delete mode 100644 airflow/providers/qubole/sensors/__init__.py
delete mode 100644 airflow/providers/qubole/sensors/qubole.py
delete mode 100644 docs/apache-airflow-providers-qubole/changelog.rst
delete mode 100644 docs/apache-airflow-providers-qubole/commits.rst
delete mode 100644 docs/apache-airflow-providers-qubole/index.rst
delete mode 100644 docs/apache-airflow-providers-qubole/installing-providers-from-sources.rst
delete mode 100644 docs/apache-airflow-providers-qubole/operators/index.rst
delete mode 100644 docs/apache-airflow-providers-qubole/operators/qubole.rst
delete mode 100644 docs/apache-airflow-providers-qubole/operators/qubole_check.rst
delete mode 100644 docs/apache-airflow-providers-qubole/security.rst
delete mode 100644 docs/integration-logos/qubole/Qubole.png
delete mode 100644 tests/providers/qubole/__init__.py
delete mode 100644 tests/providers/qubole/hooks/__init__.py
delete mode 100644 tests/providers/qubole/hooks/test_qubole.py
delete mode 100644 tests/providers/qubole/hooks/test_qubole_check.py
delete mode 100644 tests/providers/qubole/operators/__init__.py
delete mode 100644 tests/providers/qubole/operators/test_qubole.py
delete mode 100644 tests/providers/qubole/operators/test_qubole_check.py
delete mode 100644 tests/providers/qubole/sensors/__init__.py
delete mode 100644 tests/providers/qubole/sensors/test_qubole.py
delete mode 100644 tests/system/providers/qubole/__init__.py
delete mode 100644 tests/system/providers/qubole/example_qubole.py
delete mode 100644 tests/system/providers/qubole/example_qubole_sensors.py
(airflow) 01/01: Move prepare-provider-packages to be run entirely in Breeze Python
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch move-prepare-providers-packages-to-breeze
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b43046438fcc14bdd4b9a237ed97ed5f29da056f
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 12 20:40:44 2023 +0100
Move prepare-provider-packages to be run entirely in Breeze Python
This is a follow-up after #35586 and it depends on this one. It
moves the whole functionality of preparing provider packages to
breeze, removing the need of doing it in the Breeze CI image.
Since we have Python breeze with its own environment managed via
`pipx` we can now make sure that all the necessary packages are
installed in this environment and run package building in the
same environment Breeze uses.
Previously we have been running all the package building inside the
CI image for two reasons:
* we could rely on the same version of build tools (wheel/setuptools)
being installed in the CI image
* security of the provider package preparation that used setuptools
pre PEP-517 way of building packages that executed setup.py code
In order to isolate execution of potentially arbitrary code
in setup.py from the HOST environment in CI - where the host
environment might have access to secrets and tokens that would allow
it to break out of the sandbox for PRs coming from forks. The setup.py
file has been prepared by breeze using JINJA templates but it was
potentially possible to manipulate provider package directory structure
and get "Python" injection into generated setup.py, so it was safer
to run it in the isolated Breeze CI environment.
This PR makes it secure to run it in the Host environment,
because instead of generating setup.cfg and setup.py we generate
pyproject.toml with all the necessary information and we are using
PEP-517 compliant way of building provider packages - no arbitrary
code executed via setup.py is possible this way on the host,
so we can safely build provider packages in the host. We are
generating declarative pyproject.toml for that rather than imperative
setup.py, so we are safe to run the build process in the host without
being afraid of executing arbitrary code.
We are using flit as build tool - this is one of the popular build
tools - created by Python Packaging team. It is simple and not
too opinionated, it supports PEP-517 as well as PEP-621, so most of
the project mnetadata in pyproject toml can be added to PEP-621
compliant "project" section of pyproject.toml.
Together with the change we improves the process of generation of the
extracted sources for the providers. Originally we copied the whole
sources of Airflow to a single directory (provider_packages) and run
sequentially provider packages building from that single directory,
however it made it impossible to parallelise such builds - all
providers had to be built sequentially.
We change the approach now - instead of copying all airflow
sources once to the single directory, we build providers in separate
subdirectories of files/provider_packages/PROVIDER_ID and we only
copy there relevant sources (i.e. only provider's subfolder from
the "airflow/providers". This is quite a bit faster (each provider
only gets built using only its own sources so just scanning the
directory is faster) but it also allows to run package preparation
in parallel because each provider is fully isolated from others.
This PR also excludes not-needed `prepare_providers_package.py`
and unneded `provider_packages` folder used to prepare providers
before as well as bash script to build the providers and some
unused bash functions.
---
.rat-excludes | 3 +
NOTICE | 2 +-
airflow/www/webpack.config.js | 2 +-
chart/NOTICE | 2 +-
dev/breeze/README.md | 2 +-
dev/breeze/setup.cfg | 13 +-
.../commands/release_management_commands.py | 226 ++--
.../commands/release_management_commands_config.py | 4 +-
.../prepare_providers/provider_documentation.py | 191 +--
.../prepare_providers/provider_packages.py | 245 ++++
.../templates/CHANGELOG_TEMPLATE.rst.jinja2 | 27 +-
.../PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 | 11 +-
.../templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 | 15 +-
.../templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2 | 30 +-
.../templates/PROVIDER_README_TEMPLATE.rst.jinja2 | 12 +-
.../PROVIDER__INIT__PY_TEMPLATE.py.jinja2 | 7 +-
.../templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 | 6 -
.../get_provider_info_TEMPLATE.py.jinja2 | 23 +-
.../templates/pyproject_TEMPLATE.toml.jinja2 | 109 ++
dev/breeze/src/airflow_breeze/utils/packages.py | 217 +++-
dev/breeze/src/airflow_breeze/utils/versions.py | 6 +
dev/breeze/tests/test_packages.py | 61 +-
dev/breeze/tests/test_provider_documentation.py | 60 +-
dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2 | 36 -
.../PROVIDER_README_TEMPLATE.rst.jinja2 | 109 --
dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2 | 84 --
dev/provider_packages/SETUP_TEMPLATE.py.jinja2 | 47 -
dev/provider_packages/prepare_provider_packages.py | 1277 --------------------
...elease-management_prepare-provider-packages.svg | 46 +-
...elease-management_prepare-provider-packages.txt | 2 +-
provider_packages/.gitignore | 7 -
provider_packages/INSTALL | 15 -
provider_packages/LICENSE | 201 ---
provider_packages/NOTICE | 6 -
provider_packages/dist | 1 -
provider_packages/pyproject.toml | 1 -
scripts/in_container/_in_container_utils.sh | 96 --
.../in_container/run_prepare_provider_packages.sh | 149 ---
38 files changed, 915 insertions(+), 2436 deletions(-)
diff --git a/.rat-excludes b/.rat-excludes
index b37f97594c..751742b1af 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -142,3 +142,6 @@ doap_airflow.rdf
# nvm (Node Version Manager)
.nvmrc
+
+# PKG-INFO file
+PKG-INFO
diff --git a/NOTICE b/NOTICE
index 84c77cd4fc..33371e44a7 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Airflow
-Copyright 2016-2021 The Apache Software Foundation
+Copyright 2016-2023 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js
index fc5c6a6497..6ac1f3a208 100644
--- a/airflow/www/webpack.config.js
+++ b/airflow/www/webpack.config.js
@@ -40,7 +40,7 @@ const BUILD_DIR = path.resolve(__dirname, "./static/dist");
// Convert licenses json into a standard format for LICENSES.txt
const formatLicenses = (packages) => {
let text = `Apache Airflow
-Copyright 2016-2021 The Apache Software Foundation
+Copyright 2016-2023 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
diff --git a/chart/NOTICE b/chart/NOTICE
index 3f68897ba6..ff6e647c9e 100644
--- a/chart/NOTICE
+++ b/chart/NOTICE
@@ -1,5 +1,5 @@
Apache Airflow
-Copyright 2016-2021 The Apache Software Foundation
+Copyright 2016-2023 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
diff --git a/dev/breeze/README.md b/dev/breeze/README.md
index 1421958fa9..1f4e7c65c9 100644
--- a/dev/breeze/README.md
+++ b/dev/breeze/README.md
@@ -66,6 +66,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT.
---------------------------------------------------------------------------------------------------------
-Package config hash: 7b512fa3a81a967c22fc4ccccf052a4c4dbcafd5c014adea775d45f0034d03e1c63d7d1e3df723e93724924ed3cfa92a5848c994c247dfd326c0a6300e282f88
+Package config hash: 4b93f639e3365c7420475b6e7850624c74f24bc167c9cb5ce41971720942d2f1195ca4806cbd23b34dd4991a020b9549cdf4143e7c6173c42bda52b2911850ad
---------------------------------------------------------------------------------------------------------
diff --git a/dev/breeze/setup.cfg b/dev/breeze/setup.cfg
index db53c94ebb..e81ec3aa9d 100644
--- a/dev/breeze/setup.cfg
+++ b/dev/breeze/setup.cfg
@@ -55,29 +55,28 @@ package_dir=
=src
packages = find:
install_requires =
+ PyGithub>=2.1.1
black>=23.11.0
click>=8.1.7
filelock>=3.13.0
+ flit>=3.9.0
+ gitpython>=3.1.40
inputimeout>=1.0.4
jinja2>=3.1.0
+ jsonschema>=4.19.1
packaging>=23.2
pendulum>=2.1.2,<3
pre-commit>=3.5.0
psutil>=5.9.6
- pytest>=7.4.0
pytest-xdist>=3.3.1
+ pytest>=7.4.0
pyyaml>=6.0.1
- PyGithub>=2.1.1
requests>=2.30.0
- rich>=13.6.0
rich-click>=1.7.1
- gitpython>=3.1.40
+ rich>=13.6.0
semver>=3.0.2
tabulate>=0.9.0
twine>=4.0.2
- wheel>=0.41.3
- setuptools>=68.2.2
- jsonschema>=4.19.1
[options.packages.find]
where=src
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index 73330b51a0..e3cc6140ee 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -27,7 +27,7 @@ from copy import deepcopy
from datetime import datetime
from pathlib import Path
from subprocess import DEVNULL
-from typing import IO, Generator, NamedTuple
+from typing import IO, Any, Generator, NamedTuple
import click
from rich.progress import Progress
@@ -45,6 +45,18 @@ from airflow_breeze.global_constants import (
MULTI_PLATFORM,
)
from airflow_breeze.params.shell_params import ShellParams
+from airflow_breeze.prepare_providers.provider_packages import (
+ PrepareReleasePackageErrorBuildingPackageException,
+ PrepareReleasePackageTagExistException,
+ PrepareReleasePackageWrongSetupException,
+ build_provider_package,
+ cleanup_build_remnants,
+ copy_provider_sources_to_target,
+ generate_build_files,
+ get_packages_list_to_act_on,
+ move_built_packages_and_cleanup,
+ should_skip_the_package,
+)
from airflow_breeze.utils.add_back_references import (
start_generating_back_references,
)
@@ -90,12 +102,13 @@ from airflow_breeze.utils.docker_command_utils import (
)
from airflow_breeze.utils.github import download_constraints_file, get_active_airflow_versions
from airflow_breeze.utils.packages import (
+ PackageSuspendedException,
expand_all_provider_packages,
find_matching_long_package_names,
get_available_packages,
get_provider_details,
get_provider_packages_metadata,
- get_removed_provider_ids,
+ make_sure_remote_apache_exists_and_fetch,
)
from airflow_breeze.utils.parallel import (
GenericRegexpProgressMatcher,
@@ -226,9 +239,9 @@ def prepare_airflow_packages(
sys.exit(result_command.returncode)
-def provider_documentation_summary(documentation: str, message_type: MessageType, packages: list[str]):
+def provider_action_summary(description: str, message_type: MessageType, packages: list[str]):
if packages:
- get_console().print(f"{documentation}: {len(packages)}\n")
+ get_console().print(f"{description}: {len(packages)}\n")
get_console().print(f"[{message_type.value}]{' '.join(packages)}")
get_console().print()
@@ -287,7 +300,6 @@ def prepare_provider_documentation(
PrepareReleaseDocsNoChangesException,
PrepareReleaseDocsUserQuitException,
PrepareReleaseDocsUserSkippedException,
- make_sure_remote_apache_exists_and_fetch,
update_changelog,
update_min_airflow_version,
update_release_notes,
@@ -300,7 +312,6 @@ def prepare_provider_documentation(
if not skip_git_fetch:
run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL)
make_sure_remote_apache_exists_and_fetch(github_repository=github_repository)
- provider_packages_metadata = get_provider_packages_metadata()
no_changes_packages = []
doc_only_packages = []
error_packages = []
@@ -308,76 +319,62 @@ def prepare_provider_documentation(
success_packages = []
suspended_packages = []
removed_packages = []
- for provider_package_id in provider_packages:
- provider_metadata = provider_packages_metadata.get(provider_package_id)
- if not provider_metadata:
- get_console().print(
- f"[error]The package {provider_package_id} is not a provider package. Exiting[/]"
- )
- sys.exit(1)
- if provider_metadata.get("removed", False):
- get_console().print(
- f"[warning]The package: {provider_package_id} is scheduled for removal, but "
- f"since you asked for it, it will be built [/]\n"
- )
- elif provider_metadata.get("suspended"):
- get_console().print(
- f"[warning]The package: {provider_package_id} is suspended " f"skipping it [/]\n"
- )
- suspended_packages.append(provider_package_id)
- continue
+ for provider_id in provider_packages:
+ provider_metadata = basic_provider_checks(provider_id)
if os.environ.get("GITHUB_ACTIONS", "false") != "true":
get_console().print("-" * get_console().width)
try:
with_breaking_changes = False
maybe_with_new_features = False
- with ci_group(f"Update release notes for package '{provider_package_id}' "):
+ with ci_group(f"Update release notes for package '{provider_id}' "):
get_console().print("Updating documentation for the latest release version.")
if not only_min_version_update:
with_breaking_changes, maybe_with_new_features = update_release_notes(
- provider_package_id,
+ provider_id,
reapply_templates_only=reapply_templates_only,
base_branch=base_branch,
regenerate_missing_docs=reapply_templates_only,
non_interactive=non_interactive,
)
update_min_airflow_version(
- provider_package_id=provider_package_id,
+ provider_package_id=provider_id,
with_breaking_changes=with_breaking_changes,
maybe_with_new_features=maybe_with_new_features,
)
- with ci_group(f"Updates changelog for last release of package '{provider_package_id}'"):
+ with ci_group(f"Updates changelog for last release of package '{provider_id}'"):
update_changelog(
- package_id=provider_package_id,
+ package_id=provider_id,
base_branch=base_branch,
reapply_templates_only=reapply_templates_only,
with_breaking_changes=with_breaking_changes,
maybe_with_new_features=maybe_with_new_features,
)
except PrepareReleaseDocsNoChangesException:
- no_changes_packages.append(provider_package_id)
+ no_changes_packages.append(provider_id)
except PrepareReleaseDocsChangesOnlyException:
- doc_only_packages.append(provider_package_id)
+ doc_only_packages.append(provider_id)
except PrepareReleaseDocsErrorOccurredException:
- error_packages.append(provider_package_id)
+ error_packages.append(provider_id)
except PrepareReleaseDocsUserSkippedException:
- user_skipped_packages.append(provider_package_id)
+ user_skipped_packages.append(provider_id)
+ except PackageSuspendedException:
+ suspended_packages.append(provider_id)
except PrepareReleaseDocsUserQuitException:
break
else:
if provider_metadata.get("removed"):
- removed_packages.append(provider_package_id)
+ removed_packages.append(provider_id)
else:
- success_packages.append(provider_package_id)
+ success_packages.append(provider_id)
get_console().print()
- get_console().print("\n[info]Summary of prepared packages:\n")
- provider_documentation_summary("Success", MessageType.SUCCESS, success_packages)
- provider_documentation_summary("Scheduled for removal", MessageType.SUCCESS, removed_packages)
- provider_documentation_summary("Docs only", MessageType.SUCCESS, doc_only_packages)
- provider_documentation_summary("Skipped on no changes", MessageType.WARNING, no_changes_packages)
- provider_documentation_summary("Suspended", MessageType.WARNING, suspended_packages)
- provider_documentation_summary("Skipped by user", MessageType.SPECIAL, user_skipped_packages)
- provider_documentation_summary("Errors", MessageType.ERROR, error_packages)
+ get_console().print("\n[info]Summary of prepared documentation:\n")
+ provider_action_summary("Success", MessageType.SUCCESS, success_packages)
+ provider_action_summary("Scheduled for removal", MessageType.SUCCESS, removed_packages)
+ provider_action_summary("Docs only", MessageType.SUCCESS, doc_only_packages)
+ provider_action_summary("Skipped on no changes", MessageType.WARNING, no_changes_packages)
+ provider_action_summary("Suspended", MessageType.WARNING, suspended_packages)
+ provider_action_summary("Skipped by user", MessageType.SPECIAL, user_skipped_packages)
+ provider_action_summary("Errors", MessageType.ERROR, error_packages)
if error_packages:
get_console().print("\n[errors]There were errors when generating packages. Exiting!\n")
sys.exit(1)
@@ -386,10 +383,27 @@ def prepare_provider_documentation(
sys.exit(0)
get_console().print("\n[success]Successfully prepared documentation for packages!\n\n")
get_console().print(
- "\n[info]Please review the updated files, classify " "the changelog entries and commit the changes.\n"
+ "\n[info]Please review the updated files, classify the changelog entries and commit the changes.\n"
)
+def basic_provider_checks(provider_package_id: str) -> dict[str, Any]:
+ provider_packages_metadata = get_provider_packages_metadata()
+ provider_metadata = provider_packages_metadata.get(provider_package_id)
+ if not provider_metadata:
+ get_console().print(f"[error]The package {provider_package_id} is not a provider package. Exiting[/]")
+ sys.exit(1)
+ if provider_metadata.get("removed", False):
+ get_console().print(
+ f"[warning]The package: {provider_package_id} is scheduled for removal, but "
+ f"since you asked for it, it will be built [/]\n"
+ )
+ elif provider_metadata.get("suspended"):
+ get_console().print(f"[warning]The package: {provider_package_id} is suspended " f"skipping it [/]\n")
+ raise PackageSuspendedException()
+ return provider_metadata
+
+
@release_management.command(
name="prepare-provider-packages",
help="Prepare sdist/whl packages of Airflow Providers.",
@@ -401,48 +415,110 @@ def prepare_provider_documentation(
type=click.File("rt"),
help="Read list of packages from text file (one package per line).",
)
-@option_debug_release_management
-@argument_provider_packages
+@click.option(
+ "--skip-tag-check",
+ default=False,
+ is_flag=True,
+ help="Skip checking if the tag already exists in the remote repository",
+)
+@click.option(
+ "--skip-deleting-generated-files",
+ default=False,
+ is_flag=True,
+ help="Skip deleting files that were used to generate provider package. Useful for debugging and "
+ "developing changes to the build process.",
+)
+@click.option(
+ "--clean-dist",
+ default=False,
+ is_flag=True,
+ help="Clean dist directory before building packages. Useful when you want to build multiple packages "
+ " in a clean environment",
+)
@option_github_repository
+@argument_provider_packages
@option_verbose
@option_dry_run
def prepare_provider_packages(
package_format: str,
version_suffix_for_pypi: str,
- package_list_file: IO,
- debug: bool,
- provider_packages: tuple[str, ...],
+ package_list_file: IO | None,
+ skip_tag_check: bool,
+ skip_deleting_generated_files: bool,
+ clean_dist: bool,
github_repository: str,
+ provider_packages: tuple[str, ...],
):
perform_environment_checks()
cleanup_python_generated_files()
- packages_list = list(provider_packages)
-
- removed_provider_ids = get_removed_provider_ids()
- if package_list_file:
- packages_list.extend(
- [
- package.strip()
- for package in package_list_file.readlines()
- if package.strip() not in removed_provider_ids
- ]
- )
- shell_params = ShellParams(
- mount_sources=MOUNT_ALL,
- github_repository=github_repository,
- python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
- package_format=package_format,
- skip_environment_initialization=True,
- version_suffix_for_pypi=version_suffix_for_pypi,
- )
- rebuild_or_pull_ci_image_if_needed(command_params=shell_params)
- cmd_to_run = ["/opt/airflow/scripts/in_container/run_prepare_provider_packages.sh", *packages_list]
- result_command = run_docker_command_with_debug(
- params=shell_params,
- command=cmd_to_run,
- debug=debug,
+ packages_list = get_packages_list_to_act_on(package_list_file, provider_packages)
+ if not skip_tag_check:
+ run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL)
+ make_sure_remote_apache_exists_and_fetch(github_repository=github_repository)
+ success_packages = []
+ skipped_as_already_released_packages = []
+ suspended_packages = []
+ wrong_setup_packages = []
+ error_packages = []
+ if clean_dist:
+ get_console().print("\n[warning]Cleaning dist directory before building packages[/]\n")
+ shutil.rmtree(DIST_DIR, ignore_errors=True)
+ DIST_DIR.mkdir(parents=True, exist_ok=True)
+ for provider_id in packages_list:
+ try:
+ basic_provider_checks(provider_id)
+ if not skip_tag_check and should_skip_the_package(provider_id, version_suffix_for_pypi):
+ continue
+ get_console().print()
+ with ci_group(f"Preparing provider package [special]{provider_id}"):
+ get_console().print()
+ target_provider_root_sources_path = copy_provider_sources_to_target(provider_id)
+ generate_build_files(
+ provider_id=provider_id,
+ version_suffix=version_suffix_for_pypi,
+ target_provider_root_sources_path=target_provider_root_sources_path,
+ )
+ cleanup_build_remnants(target_provider_root_sources_path)
+ build_provider_package(
+ provider_id=provider_id,
+ version_suffix=version_suffix_for_pypi,
+ package_format=package_format,
+ target_provider_root_sources_path=target_provider_root_sources_path,
+ )
+ move_built_packages_and_cleanup(
+ target_provider_root_sources_path, DIST_DIR, skip_cleanup=skip_deleting_generated_files
+ )
+ except PrepareReleasePackageTagExistException:
+ skipped_as_already_released_packages.append(provider_id)
+ except PrepareReleasePackageWrongSetupException:
+ wrong_setup_packages.append(provider_id)
+ except PrepareReleasePackageErrorBuildingPackageException:
+ error_packages.append(provider_id)
+ except PackageSuspendedException:
+ suspended_packages.append(provider_id)
+ else:
+ get_console().print(f"\n[success]Generated package [special]{provider_id}")
+ success_packages.append(provider_id)
+ get_console().print()
+ get_console().print("\n[info]Summary of prepared packages:\n")
+ provider_action_summary("Success", MessageType.SUCCESS, success_packages)
+ provider_action_summary(
+ "Skipped as already released", MessageType.SUCCESS, skipped_as_already_released_packages
)
- sys.exit(result_command.returncode)
+ provider_action_summary("Suspended", MessageType.WARNING, suspended_packages)
+ provider_action_summary("Wrong setup generated", MessageType.ERROR, wrong_setup_packages)
+ provider_action_summary("Errors", MessageType.ERROR, error_packages)
+ if error_packages or wrong_setup_packages:
+ get_console().print("\n[errors]There were errors when generating packages. Exiting!\n")
+ sys.exit(1)
+ if not success_packages and not skipped_as_already_released_packages:
+ get_console().print("\n[warning]No packages prepared!\n")
+ sys.exit(0)
+ get_console().print("\n[success]Successfully built packages!\n\n")
+ get_console().print("\n[info]Packages available in dist:\n")
+ for file in sorted(DIST_DIR.glob("apache*")):
+ get_console().print(file.name)
+ get_console().print()
def run_generate_constraints(
diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
index ad658d228c..a5b84869b4 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py
@@ -107,8 +107,10 @@ RELEASE_MANAGEMENT_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"options": [
"--package-format",
"--version-suffix-for-pypi",
+ "--clean-dist",
+ "--skip-tag-check",
+ "--skip-deleting-generated-files",
"--package-list-file",
- "--debug",
"--github-repository",
],
}
diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
index fc35eacc06..3c0370f33e 100644
--- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_documentation.py
@@ -28,32 +28,27 @@ from copy import deepcopy
from enum import Enum
from pathlib import Path
from shutil import copyfile
-from typing import Any, Iterable, NamedTuple
+from typing import Any, NamedTuple
import jinja2
import semver
from rich.syntax import Syntax
-from airflow_breeze.global_constants import PROVIDER_DEPENDENCIES
from airflow_breeze.utils.black_utils import black_format
from airflow_breeze.utils.confirm import Answer, user_confirm
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.packages import (
+ HTTPS_REMOTE,
ProviderPackageDetails,
get_provider_details,
get_provider_jinja_context,
get_provider_packages_metadata,
- get_provider_requirements,
- get_removed_provider_ids,
get_source_package_path,
-)
-from airflow_breeze.utils.path_utils import (
- BREEZE_SOURCES_ROOT,
+ render_template,
)
from airflow_breeze.utils.run_utils import run_command
from airflow_breeze.utils.shared_options import get_verbose
-
-HTTPS_REMOTE = "apache-https-for-providers"
+from airflow_breeze.utils.versions import get_version_tag
PR_PATTERN = re.compile(r".*\(#(\d+)\)")
@@ -162,69 +157,6 @@ TYPE_OF_CHANGE_DESCRIPTION = {
}
-def make_sure_remote_apache_exists_and_fetch(github_repository: str = "apache/airflow"):
- """Make sure that apache remote exist in git.
-
- We need to take a log from the apache repository main branch - not locally because we might
- not have the latest version. Also, the local repo might be shallow, so we need to
- un-shallow it to see all the history.
-
- This will:
- * check if the remote exists and add if it does not
- * check if the local repo is shallow, mark it to un-shallow in this case
- * fetch from the remote including all tags and overriding local tags in case
- they are set differently
-
- """
- try:
- run_command(["git", "remote", "get-url", HTTPS_REMOTE], text=True, capture_output=True)
- except subprocess.CalledProcessError as ex:
- if ex.returncode == 128 or ex.returncode == 2:
- run_command(
- [
- "git",
- "remote",
- "add",
- HTTPS_REMOTE,
- f"https://github.com/{github_repository}.git",
- ],
- check=True,
- )
- else:
- get_console().print(
- f"[error]Error {ex}[/]\n" f"[error]When checking if {HTTPS_REMOTE} is set.[/]\n\n"
- )
- sys.exit(1)
- get_console().print("[info]Fetching full history and tags from remote.")
- get_console().print("[info]This might override your local tags!")
- result = run_command(
- ["git", "rev-parse", "--is-shallow-repository"],
- check=True,
- capture_output=True,
- text=True,
- )
- is_shallow_repo = result.stdout.strip() == "true"
- fetch_command = ["git", "fetch", "--tags", "--force", HTTPS_REMOTE]
- if is_shallow_repo:
- fetch_command.append("--unshallow")
- try:
- run_command(fetch_command)
- except subprocess.CalledProcessError as e:
- get_console().print(
- f"[error]Error {e}[/]\n"
- f"[error]When fetching tags from remote. Your tags might not be refreshed.[/]\n\n"
- f'[warning]Please refresh the tags manually via:[/]\n\n"'
- f'{" ".join(fetch_command)}\n\n'
- )
- sys.exit(1)
-
-
-def _get_version_tag(version: str, provider_package_id: str, version_suffix: str = ""):
- if version_suffix is None:
- version_suffix = ""
- return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
-
-
def _get_git_log_command(from_commit: str | None = None, to_commit: str | None = None) -> list[str]:
"""Get git command to run for the current repo from the current folder.
@@ -340,7 +272,7 @@ def _get_all_changes_for_package(
"""
provider_details = get_provider_details(provider_package_id)
current_version = provider_details.versions[0]
- current_tag_no_suffix = _get_version_tag(current_version, provider_package_id)
+ current_tag_no_suffix = get_version_tag(current_version, provider_package_id)
if get_verbose():
get_console().print(f"[info]Checking if tag '{current_tag_no_suffix}' exist.")
result = run_command(
@@ -420,7 +352,7 @@ def _get_all_changes_for_package(
current_version = provider_details.versions[0]
list_of_list_of_changes: list[list[Change]] = []
for version in provider_details.versions[1:]:
- version_tag = _get_version_tag(version, provider_package_id)
+ version_tag = get_version_tag(version, provider_package_id)
result = run_command(
_get_git_log_command(next_version_tag, version_tag),
cwd=provider_details.source_provider_package_path,
@@ -554,63 +486,6 @@ def _verify_changelog_exists(package: str) -> Path:
return changelog_path
-def _convert_pip_requirements_to_table(requirements: Iterable[str], markdown: bool = True) -> str:
- """
- Converts PIP requirement list to a Markdown table.
- :param requirements: requirements list
- :param markdown: if True, Markdown format is used else rst
- :return: formatted table
- """
- from tabulate import tabulate
-
- headers = ["PIP package", "Version required"]
- table_data = []
- for dependency in requirements:
- found = re.match(r"(^[^<=>~]*)([^<=>~]?.*)$", dependency)
- if found:
- package = found.group(1)
- version_required = found.group(2)
- if version_required != "":
- version_required = f"`{version_required}`" if markdown else f"``{version_required}``"
- table_data.append((f"`{package}`" if markdown else f"``{package}``", version_required))
- else:
- table_data.append((dependency, ""))
- return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
-
-
-def _convert_cross_package_dependencies_to_table(
- cross_package_dependencies: list[str],
- markdown: bool = True,
-) -> str:
- """
- Converts cross-package dependencies to a Markdown table
- :param cross_package_dependencies: list of cross-package dependencies
- :param markdown: if True, Markdown format is used else rst
- :return: formatted table
- """
- from tabulate import tabulate
-
- headers = ["Dependent package", "Extra"]
- table_data = []
- prefix = "apache-airflow-providers-"
- base_url = "https://airflow.apache.org/docs/"
- for dependency in cross_package_dependencies:
- pip_package_name = f"{prefix}{dependency.replace('.','-')}"
- url_suffix = f"{dependency.replace('.','-')}"
- if markdown:
- url = f"[{pip_package_name}]({base_url}{url_suffix})"
- else:
- url = f"`{pip_package_name} <{base_url}{prefix}{url_suffix}>`_"
- table_data.append((url, f"`{dependency}`" if markdown else f"``{dependency}``"))
- return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
-
-
-def _get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]:
- if provider_package_id in get_removed_provider_ids():
- return []
- return PROVIDER_DEPENDENCIES[provider_package_id]["cross-providers-deps"]
-
-
def _get_additional_package_info(provider_package_path: Path) -> str:
"""Returns additional info for the package.
@@ -632,38 +507,6 @@ def _get_additional_package_info(provider_package_path: Path) -> str:
return ""
-def render_template(
- template_name: str,
- context: dict[str, Any],
- extension: str,
- autoescape: bool = True,
- keep_trailing_newline: bool = False,
-) -> str:
- """
- Renders template based on its name. Reads the template from <name>_TEMPLATE.md.jinja2 in current dir.
- :param template_name: name of the template to use
- :param context: Jinja2 context
- :param extension: Target file extension
- :param autoescape: Whether to autoescape HTML
- :param keep_trailing_newline: Whether to keep the newline in rendered output
- :return: rendered template
- """
- import jinja2
-
- template_loader = jinja2.FileSystemLoader(
- searchpath=BREEZE_SOURCES_ROOT / "src" / "airflow_breeze" / "templates"
- )
- template_env = jinja2.Environment(
- loader=template_loader,
- undefined=jinja2.StrictUndefined,
- autoescape=autoescape,
- keep_trailing_newline=keep_trailing_newline,
- )
- template = template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
- content: str = template.render(context)
- return content
-
-
def replace_content(file_path: Path, old_text: str, new_text: str, provider_id: str):
if new_text != old_text:
_, temp_file_path = tempfile.mkstemp()
@@ -1022,31 +865,17 @@ def get_provider_documentation_jinja_context(
provider_id: str, with_breaking_changes: bool, maybe_with_new_features: bool
) -> dict[str, Any]:
provider_details = get_provider_details(provider_id)
- current_release_version = provider_details.versions[0]
jinja_context = get_provider_jinja_context(
provider_id=provider_id,
- current_release_version=current_release_version,
+ current_release_version=provider_details.versions[0],
version_suffix="",
- with_breaking_changes=with_breaking_changes,
- maybe_with_new_features=maybe_with_new_features,
)
+ jinja_context["WITH_BREAKING_CHANGES"] = with_breaking_changes
+ jinja_context["MAYBE_WITH_NEW_FEATURES"] = maybe_with_new_features
+
jinja_context["ADDITIONAL_INFO"] = (
_get_additional_package_info(provider_package_path=provider_details.source_provider_package_path),
)
- jinja_context["CROSS_PROVIDERS_DEPENDENCIES"] = _get_cross_provider_dependent_packages(provider_id)
- cross_providers_dependencies = _get_cross_provider_dependent_packages(provider_package_id=provider_id)
- jinja_context["CROSS_PROVIDERS_DEPENDENCIES_TABLE"] = _convert_cross_package_dependencies_to_table(
- cross_providers_dependencies
- )
- jinja_context["CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST"] = _convert_cross_package_dependencies_to_table(
- cross_providers_dependencies, markdown=False
- )
- jinja_context["PIP_REQUIREMENTS_TABLE"] = _convert_pip_requirements_to_table(
- get_provider_requirements(provider_id)
- )
- jinja_context["PIP_REQUIREMENTS_TABLE_RST"] = _convert_pip_requirements_to_table(
- get_provider_requirements(provider_id), markdown=False
- )
return jinja_context
diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
new file mode 100644
index 0000000000..5b004eb27d
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py
@@ -0,0 +1,245 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from shutil import copytree, rmtree
+from stat import S_IRGRP, S_IROTH, S_IRUSR, S_IWUSR
+from typing import IO, Any
+
+from airflow_breeze.utils.console import get_console
+from airflow_breeze.utils.packages import (
+ get_available_packages,
+ get_latest_provider_tag,
+ get_provider_details,
+ get_provider_jinja_context,
+ get_removed_provider_ids,
+ get_source_package_path,
+ get_target_root_for_copied_provider_sources,
+ render_template,
+ tag_exists_for_provider,
+)
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
+from airflow_breeze.utils.run_utils import run_command
+
+LICENCE_RST = """
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+"""
+
+
+class PrepareReleasePackageTagExistException(Exception):
+ """Tag already exist for the package."""
+
+
+class PrepareReleasePackageWrongSetupException(Exception):
+ """Wrong setup prepared for the package."""
+
+
+class PrepareReleasePackageErrorBuildingPackageException(Exception):
+ """Error when building the package."""
+
+
+def copy_provider_sources_to_target(provider_id: str) -> Path:
+ target_provider_root_path = get_target_root_for_copied_provider_sources(provider_id)
+
+ if target_provider_root_path.exists() and not target_provider_root_path.is_dir():
+ get_console().print(
+ f"[error]Target folder for {provider_id} sources is not a directory "
+ f"please delete {target_provider_root_path} and try again!"
+ )
+ rmtree(target_provider_root_path, ignore_errors=True)
+ target_provider_root_path.mkdir(parents=True)
+ source_provider_sources_path = get_source_package_path(provider_id)
+ relative_provider_path = source_provider_sources_path.relative_to(AIRFLOW_SOURCES_ROOT)
+ target_providers_sub_folder = target_provider_root_path / relative_provider_path
+ get_console().print(
+ f"[info]Copying provider sources: " f"{source_provider_sources_path} -> {target_providers_sub_folder}"
+ )
+ copytree(source_provider_sources_path, target_providers_sub_folder)
+ shutil.copy(AIRFLOW_SOURCES_ROOT / "LICENSE", target_providers_sub_folder / "LICENSE")
+ # We do not copy NOTICE from the top level source of Airflow because NOTICE only refers to
+ # Airflow sources - not to providers. If any of the providers is going to have a code that
+ # requires NOTICE, then it should be stored in the provider sources (airflow/providers/PROVIDER_ID)
+ # And it will be copied from there.
+ (target_providers_sub_folder / ".latest-doc-only-change.txt").unlink(missing_ok=True)
+ (target_providers_sub_folder / "CHANGELOG.rst").unlink(missing_ok=True)
+ (target_providers_sub_folder / "provider.yaml").unlink(missing_ok=True)
+ return target_provider_root_path
+
+
+def get_provider_package_jinja_context(provider_id: str, version_suffix: str) -> dict[str, Any]:
+ provider_details = get_provider_details(provider_id)
+ jinja_context = get_provider_jinja_context(
+ provider_id=provider_id,
+ current_release_version=provider_details.versions[0],
+ version_suffix=version_suffix,
+ )
+ return jinja_context
+
+
+def _prepare_get_provider_info_py_file(context: dict[str, Any], provider_id: str, target_path: Path):
+ from airflow_breeze.utils.black_utils import black_format
+
+ get_provider_template_name = "get_provider_info"
+ get_provider_content = render_template(
+ template_name=get_provider_template_name,
+ context=context,
+ extension=".py",
+ autoescape=False,
+ keep_trailing_newline=True,
+ )
+ target_provider_specific_path = (target_path / "airflow" / "providers").joinpath(*provider_id.split("."))
+ (target_provider_specific_path / "get_provider_info.py").write_text(black_format(get_provider_content))
+ get_console().print(f"[info]Generated get_provider_info.py in {target_provider_specific_path}[/]")
+
+
+def _prepare_pyproject_toml_file(context: dict[str, Any], target_path: Path):
+ manifest_content = render_template(
+ template_name="pyproject",
+ context=context,
+ extension=".toml",
+ autoescape=False,
+ keep_trailing_newline=True,
+ )
+ (target_path / "pyproject.toml").write_text(manifest_content)
+ get_console().print(f"[info]Generated pyproject.toml in {target_path}[/]")
+
+
+def _prepare_readme_file(context: dict[str, Any], target_path: Path):
+ readme_content = LICENCE_RST + render_template(
+ template_name="PROVIDER_README", context=context, extension=".rst"
+ )
+ (target_path / "README.rst").write_text(readme_content)
+ get_console().print(f"[info]Generated README.rst in {target_path}[/]")
+
+
+def generate_build_files(provider_id: str, version_suffix: str, target_provider_root_sources_path: Path):
+ get_console().print(f"\n[info]Generate build files for {provider_id}\n")
+ jinja_context = get_provider_package_jinja_context(provider_id=provider_id, version_suffix=version_suffix)
+ _prepare_get_provider_info_py_file(jinja_context, provider_id, target_provider_root_sources_path)
+ _prepare_pyproject_toml_file(jinja_context, target_provider_root_sources_path)
+ _prepare_readme_file(jinja_context, target_provider_root_sources_path)
+ get_console().print(f"\n[info]Generated package build files for {provider_id}[/]\n")
+
+
+def should_skip_the_package(provider_id: str, version_suffix: str) -> bool:
+ """Return True if the package should be skipped.
+
+ For RC and official releases we check if the "officially released" version exists
+ and skip the released if it was. This allows to skip packages that have not been
+ marked for release in this wave. For "dev" suffixes, we always build all packages.
+ """
+ if version_suffix.startswith("rc") or version_suffix == "":
+ current_tag = get_latest_provider_tag(provider_id, version_suffix)
+ if tag_exists_for_provider(provider_id, current_tag):
+ get_console().print(f"[warning]The tag {current_tag} exists. Skipping the package.[/]")
+ return True
+ return False
+
+
+def cleanup_build_remnants(target_provider_root_sources_path: Path):
+ get_console().print(f"\n[info]Cleaning remnants in {target_provider_root_sources_path}")
+ for file in target_provider_root_sources_path.glob("*.egg-info"):
+ shutil.rmtree(file, ignore_errors=True)
+ shutil.rmtree(target_provider_root_sources_path / "build", ignore_errors=True)
+ shutil.rmtree(target_provider_root_sources_path / "dist", ignore_errors=True)
+ get_console().print(f"[info]Cleaned remnants in {target_provider_root_sources_path}\n")
+
+
+def build_provider_package(
+ provider_id: str, version_suffix: str, target_provider_root_sources_path: Path, package_format: str
+):
+ get_console().print(
+ f"\n[info]Building provider package: {provider_id} in format {package_format} in "
+ f"{target_provider_root_sources_path}\n"
+ )
+ command: list[str] = [sys.executable, "-m", "flit", "build", "--no-setup-py", "--no-use-vcs"]
+ if package_format != "both":
+ command.extend(["--format", package_format])
+ try:
+ run_command(command, check=True, cwd=target_provider_root_sources_path)
+ except subprocess.CalledProcessError as ex:
+ get_console().print("[error]The command returned an error %s", ex)
+ raise PrepareReleasePackageErrorBuildingPackageException()
+ get_console().print(
+ f"\n[info]Prepared provider package {provider_id} in " f"format {package_format}[/]\n"
+ )
+
+
+def move_built_packages_and_cleanup(
+ target_provider_root_sources_path: Path, dist_folder: Path, skip_cleanup: bool
+):
+ for file in (target_provider_root_sources_path / "dist").glob("apache*"):
+ get_console().print(f"[info]Moving {file} to {dist_folder}")
+ # Shutil can move packages also between filesystems
+ target_file = dist_folder / file.name
+ target_file.unlink(missing_ok=True)
+ # Change ownership to group/other to read the file
+ file.chmod(S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
+ shutil.move(file.as_posix(), dist_folder.as_posix())
+
+ if skip_cleanup:
+ get_console().print(
+ f"[warning]NOT Cleaning up the {target_provider_root_sources_path} because "
+ f"it was requested by the user[/]\n"
+ f"\nYou can use the generated packages to work on the build"
+ f"process and bring the changes back to the templates in Breeze "
+ f"src/airflow_breeze/templates"
+ )
+ else:
+ get_console().print(f"[info]Cleaning up {target_provider_root_sources_path}")
+ shutil.rmtree(target_provider_root_sources_path, ignore_errors=True)
+ get_console().print(f"[info]Cleaned up {target_provider_root_sources_path}")
+
+
+def get_packages_list_to_act_on(
+ package_list_file: IO | None, provider_packages: tuple[str, ...]
+) -> list[str]:
+ if package_list_file and provider_packages:
+ get_console().print(
+ "[error]You cannot specify individual provider packages when you specify package list file."
+ )
+ sys.exit(1)
+ if package_list_file:
+ removed_provider_ids = get_removed_provider_ids()
+ return [
+ package.strip()
+ for package in package_list_file.readlines()
+ if package.strip() not in removed_provider_ids
+ ]
+ elif provider_packages:
+ return list(provider_packages)
+ return get_available_packages()
diff --git a/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2
index 40699b0208..c0ba1da955 100644
--- a/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/CHANGELOG_TEMPLATE.rst.jinja2
@@ -16,13 +16,30 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
+#}
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+ OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ `CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
-#}
{{ version }}
{{ version_header }}
diff --git a/dev/breeze/src/airflow_breeze/templates/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2
index 43e4aec48b..38e1543bcf 100644
--- a/dev/breeze/src/airflow_breeze/templates/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2
@@ -16,11 +16,6 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
#}
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -39,4 +34,10 @@
specific language governing permissions and limitations
under the License.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+ OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ `PROVIDER_CHANGELOG_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
.. include:: {{ CHANGELOG_RELATIVE_PATH }}/CHANGELOG.rst
diff --git a/dev/breeze/src/airflow_breeze/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2
index fd5e69ee97..6adb897305 100644
--- a/dev/breeze/src/airflow_breeze/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/PROVIDER_COMMITS_TEMPLATE.rst.jinja2
@@ -16,12 +16,6 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
-
#}
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -40,7 +34,14 @@
specific language governing permissions and limitations
under the License.
-.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+ OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+ .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
+
Package {{ PACKAGE_PIP_NAME }}
------------------------------------------------------
diff --git a/dev/breeze/src/airflow_breeze/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2
index cb2bf14ad0..08d1084742 100644
--- a/dev/breeze/src/airflow_breeze/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/PROVIDER_INDEX_TEMPLATE.rst.jinja2
@@ -16,13 +16,31 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
-
-#}
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+ OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+ .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
.. toctree::
:hidden:
diff --git a/dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE.rst.jinja2
index 3c77f8017e..9bcff72fe8 100644
--- a/dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE.rst.jinja2
@@ -16,12 +16,6 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
-
#}
.. Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -40,6 +34,12 @@
specific language governing permissions and limitations
under the License.
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+ OVERWRITTEN WHEN PREPARING PACKAGES.
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+ `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
Package ``{{ PACKAGE_PIP_NAME }}``
diff --git a/dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE.py.jinja2 b/dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE.py.jinja2
index fa40820798..9acce6226f 100644
--- a/dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE.py.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE.py.jinja2
@@ -16,8 +16,7 @@
specific language governing permissions and limitations
under the License.
--#}
-#
+#}
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -38,8 +37,8 @@
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
#
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
#
from __future__ import annotations
diff --git a/dev/breeze/src/airflow_breeze/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2 b/dev/breeze/src/airflow_breeze/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2
index 421a96a3eb..f38c103ae7 100644
--- a/dev/breeze/src/airflow_breeze/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/UPDATE_CHANGELOG_TEMPLATE.rst.jinja2
@@ -16,12 +16,6 @@
specific language governing permissions and limitations
under the License.
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
-
#}
.. Review and move the new changes to one of the sections above:
{%- for change in new_changes %}
diff --git a/dev/provider_packages/get_provider_info_TEMPLATE.py.jinja2 b/dev/breeze/src/airflow_breeze/templates/get_provider_info_TEMPLATE.py.jinja2
similarity index 50%
rename from dev/provider_packages/get_provider_info_TEMPLATE.py.jinja2
rename to dev/breeze/src/airflow_breeze/templates/get_provider_info_TEMPLATE.py.jinja2
index b6e50ceeed..5340dc9b76 100644
--- a/dev/provider_packages/get_provider_info_TEMPLATE.py.jinja2
+++ b/dev/breeze/src/airflow_breeze/templates/get_provider_info_TEMPLATE.py.jinja2
@@ -1,3 +1,22 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+#}
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -18,8 +37,8 @@
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
# OVERWRITTEN WHEN PREPARING PACKAGES.
#
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `get_provider_info_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
def get_provider_info():
diff --git a/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2 b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
new file mode 100644
index 0000000000..2965235c23
--- /dev/null
+++ b/dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
@@ -0,0 +1,109 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+#}
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
+# OVERWRITTEN WHEN PREPARING PACKAGES.
+
+# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+#
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "{{ PACKAGE_PIP_NAME }}"
+version = "{{RELEASE}}{{ VERSION_SUFFIX }}"
+description = "Provider package {{ PACKAGE_PIP_NAME }} for Apache Airflow"
+readme = "README.rst"
+authors = [
+ {name="Apache Software Foundation", email="dev@airflow.apache.org"},
+]
+maintainers = [
+ {name="Apache Software Foundation", email="dev@airflow.apache.org"},
+]
+keywords = [ "airflow-provider", "{{ PROVIDER_ID }}", "airflow", "integration" ]
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Console",
+ "Environment :: Web Environment",
+ "Intended Audience :: Developers",
+ "Intended Audience :: System Administrators",
+ "Framework :: Apache Airflow",
+ "Framework :: Apache Airflow :: Provider",
+ "License :: OSI Approved :: Apache Software License",
+ {%- for python_version in SUPPORTED_PYTHON_VERSIONS %}
+ "Programming Language :: Python :: {{ python_version }}",
+ {%- endfor %}
+ "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.8"
+dependencies = [
+{{- INSTALL_REQUIREMENTS }}
+]
+
+[project.urls]
+"Documentation" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}"
+"Changelog" = "https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}/changelog.html"
+"Bug Tracker" = "https://github.com/apache/airflow/issues"
+"Source Code" = "https://github.com/apache/airflow"
+"Slack Chat" = "https://s.apache.org/airflow-slack"
+"Twitter" = "https://twitter.com/ApacheAirflow"
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = "airflow.providers.{{ PROVIDER_ID }}.get_provider_info:get_provider_info"
+
+{%- if PLUGINS %}
+[project.entry-points."airflow.plugins"]
+{%- for plugin in PLUGINS %}
+{{ plugin.name }} = "{{ plugin.package_name }}:{{ plugin.class_name }}"
+{%- endfor %}
+{%- endif %}
+
+{%- if EXTRAS_REQUIREMENTS %}
+[project.optional-dependencies]
+{%- for extra_name, dependencies_list in EXTRAS_REQUIREMENTS.items() %}
+"{{ extra_name }}" = [
+{%- for dependency in dependencies_list %}
+ "{{ dependency }}",
+{%- endfor %}
+]
+{%- endfor %}
+{%- endif %}
+
+[tool.flit.module]
+name = "airflow.providers.{{ PROVIDER_ID }}"
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py
index 3f6d6dba06..836d4054f0 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -20,6 +20,9 @@ from __future__ import annotations
import fnmatch
import json
import os
+import re
+import subprocess
+import sys
from enum import Enum
from functools import lru_cache
from pathlib import Path
@@ -34,18 +37,20 @@ from airflow_breeze.global_constants import (
from airflow_breeze.utils.console import get_console
from airflow_breeze.utils.path_utils import (
AIRFLOW_PROVIDERS_ROOT,
+ BREEZE_SOURCES_ROOT,
+ DIST_DIR,
DOCS_ROOT,
PROVIDER_DEPENDENCIES_JSON_FILE_PATH,
)
from airflow_breeze.utils.publish_docs_helpers import (
- _filepath_to_module,
- _filepath_to_system_tests,
_load_schema,
get_provider_yaml_paths,
)
-from airflow_breeze.utils.versions import strip_leading_zeros_from_version
+from airflow_breeze.utils.run_utils import run_command
+from airflow_breeze.utils.versions import get_version_tag, strip_leading_zeros_from_version
MIN_AIRFLOW_VERSION = "2.5.0"
+HTTPS_REMOTE = "apache-https-for-providers"
LONG_PROVIDERS_PREFIX = "apache-airflow-providers-"
@@ -93,6 +98,10 @@ class ProviderPackageDetails(NamedTuple):
removed: bool
+class PackageSuspendedException(Exception):
+ """Exception raised when package is suspended."""
+
+
@lru_cache
def get_provider_packages_metadata() -> dict[str, dict[str, Any]]:
"""
@@ -112,10 +121,6 @@ def get_provider_packages_metadata() -> dict[str, dict[str, Any]]:
jsonschema.validate(provider, schema=schema)
except jsonschema.ValidationError:
raise Exception(f"Unable to parse: {provider_yaml_path}.")
- provider_yaml_dir = os.path.dirname(provider_yaml_path)
- provider["python-module"] = _filepath_to_module(provider_yaml_dir)
- provider["package-dir"] = provider_yaml_dir
- provider["system-tests-dir"] = _filepath_to_system_tests(provider_yaml_dir)
result[get_short_package_name(provider["package-name"])] = provider
return result
@@ -189,6 +194,7 @@ def get_available_packages(
"""
Return provider ids for all packages that are available currently (not suspended).
+ :rtype: object
:param include_non_provider_doc_packages: whether the non-provider doc packages should be included
(packages like apache-airflow, helm-chart, docker-stack)
:param include_all_providers: whether "all-providers" should be included ni the list.
@@ -300,6 +306,10 @@ def get_documentation_package_path(provider_id: str) -> Path:
return DOCS_ROOT / f"apache-airflow-providers-{provider_id.replace('.', '-')}"
+def get_target_root_for_copied_provider_sources(provider_id: str) -> Path:
+ return (DIST_DIR / "provider_packages").joinpath(*provider_id.split("."))
+
+
def get_pip_package_name(provider_id: str) -> str:
"""
Returns PIP package name for the package id.
@@ -346,7 +356,7 @@ def get_install_requirements(provider_id: str, version_suffix: str) -> str:
else:
dependencies = PROVIDER_DEPENDENCIES.get(provider_id)["deps"]
install_requires = [apply_version_suffix(clause) for clause in dependencies]
- return "".join(f"\n {ir}" for ir in install_requires)
+ return "".join(f'\n "{ir}",' for ir in install_requires)
def get_package_extras(provider_id: str) -> dict[str, list[str]]:
@@ -436,12 +446,43 @@ def get_python_requires(provider_id: str) -> str:
return python_requires
+def convert_cross_package_dependencies_to_table(
+ cross_package_dependencies: list[str],
+ markdown: bool = True,
+) -> str:
+ """
+ Converts cross-package dependencies to a Markdown table
+ :param cross_package_dependencies: list of cross-package dependencies
+ :param markdown: if True, Markdown format is used else rst
+ :return: formatted table
+ """
+ from tabulate import tabulate
+
+ headers = ["Dependent package", "Extra"]
+ table_data = []
+ prefix = "apache-airflow-providers-"
+ base_url = "https://airflow.apache.org/docs/"
+ for dependency in cross_package_dependencies:
+ pip_package_name = f"{prefix}{dependency.replace('.','-')}"
+ url_suffix = f"{dependency.replace('.','-')}"
+ if markdown:
+ url = f"[{pip_package_name}]({base_url}{url_suffix})"
+ else:
+ url = f"`{pip_package_name} <{base_url}{prefix}{url_suffix}>`_"
+ table_data.append((url, f"`{dependency}`" if markdown else f"``{dependency}``"))
+ return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
+
+
+def get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]:
+ if provider_package_id in get_removed_provider_ids():
+ return []
+ return PROVIDER_DEPENDENCIES[provider_package_id]["cross-providers-deps"]
+
+
def get_provider_jinja_context(
provider_id: str,
current_release_version: str,
version_suffix: str,
- with_breaking_changes: bool,
- maybe_with_new_features: bool,
):
provider_details = get_provider_details(provider_id=provider_id)
release_version_no_leading_zeros = strip_leading_zeros_from_version(current_release_version)
@@ -449,30 +490,20 @@ def get_provider_jinja_context(
supported_python_versions = [
p for p in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS if p not in provider_details.excluded_python_versions
]
+ cross_providers_dependencies = get_cross_provider_dependent_packages(provider_package_id=provider_id)
context: dict[str, Any] = {
- "WITH_BREAKING_CHANGES": with_breaking_changes,
- "MAYBE_WITH_NEW_FEATURES": maybe_with_new_features,
- "ENTITY_TYPES": list(EntityType),
- "README_FILE": "README.rst",
"PROVIDER_ID": provider_details.provider_id,
"PACKAGE_PIP_NAME": get_pip_package_name(provider_details.provider_id),
"PACKAGE_WHEEL_NAME": get_wheel_package_name(provider_details.provider_id),
"FULL_PACKAGE_NAME": provider_details.full_package_name,
- "PROVIDER_PATH": provider_details.full_package_name.replace(".", "/"),
"RELEASE": current_release_version,
"RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros,
- "VERSION_SUFFIX": version_suffix or "",
+ "VERSION_SUFFIX": f".{version_suffix}" if version_suffix else "",
"PIP_REQUIREMENTS": get_provider_requirements(provider_details.provider_id),
- "PROVIDER_TYPE": "Provider",
- "PROVIDERS_FOLDER": "providers",
"PROVIDER_DESCRIPTION": provider_details.provider_description,
"INSTALL_REQUIREMENTS": get_install_requirements(
provider_id=provider_details.provider_id, version_suffix=version_suffix
),
- "SETUP_REQUIREMENTS": """
- setuptools
- wheel
-""",
"EXTRAS_REQUIREMENTS": get_package_extras(provider_id=provider_details.provider_id),
"CHANGELOG_RELATIVE_PATH": os.path.relpath(
provider_details.source_provider_package_path,
@@ -480,11 +511,149 @@ def get_provider_jinja_context(
),
"CHANGELOG": changelog,
"SUPPORTED_PYTHON_VERSIONS": supported_python_versions,
- "PYTHON_REQUIRES": get_python_requires(provider_id),
"PLUGINS": provider_details.plugins,
"MIN_AIRFLOW_VERSION": get_min_airflow_version(provider_id),
- "PREINSTALLED_PROVIDER": provider_details.provider_id in PREINSTALLED_PROVIDERS,
"PROVIDER_REMOVED": provider_details.removed,
"PROVIDER_INFO": get_provider_info_dict(provider_id),
+ "CROSS_PROVIDERS_DEPENDENCIES": get_cross_provider_dependent_packages(provider_id),
+ "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST": convert_cross_package_dependencies_to_table(
+ cross_providers_dependencies, markdown=False
+ ),
+ "PIP_REQUIREMENTS_TABLE_RST": convert_pip_requirements_to_table(
+ get_provider_requirements(provider_id), markdown=False
+ ),
}
return context
+
+
+def render_template(
+ template_name: str,
+ context: dict[str, Any],
+ extension: str,
+ autoescape: bool = True,
+ keep_trailing_newline: bool = False,
+) -> str:
+ """
+ Renders template based on its name. Reads the template from <name>_TEMPLATE.md.jinja2 in current dir.
+ :param template_name: name of the template to use
+ :param context: Jinja2 context
+ :param extension: Target file extension
+ :param autoescape: Whether to autoescape HTML
+ :param keep_trailing_newline: Whether to keep the newline in rendered output
+ :return: rendered template
+ """
+ import jinja2
+
+ template_loader = jinja2.FileSystemLoader(
+ searchpath=BREEZE_SOURCES_ROOT / "src" / "airflow_breeze" / "templates"
+ )
+ template_env = jinja2.Environment(
+ loader=template_loader,
+ undefined=jinja2.StrictUndefined,
+ autoescape=autoescape,
+ keep_trailing_newline=keep_trailing_newline,
+ )
+ template = template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
+ content: str = template.render(context)
+ return content
+
+
+def make_sure_remote_apache_exists_and_fetch(github_repository: str = "apache/airflow"):
+ """Make sure that apache remote exist in git.
+
+ We need to take a log from the apache repository main branch - not locally because we might
+ not have the latest version. Also, the local repo might be shallow, so we need to
+ un-shallow it to see all the history.
+
+ This will:
+ * check if the remote exists and add if it does not
+ * check if the local repo is shallow, mark it to un-shallow in this case
+ * fetch from the remote including all tags and overriding local tags in case
+ they are set differently
+
+ """
+ try:
+ run_command(["git", "remote", "get-url", HTTPS_REMOTE], text=True, capture_output=True)
+ except subprocess.CalledProcessError as ex:
+ if ex.returncode == 128 or ex.returncode == 2:
+ run_command(
+ [
+ "git",
+ "remote",
+ "add",
+ HTTPS_REMOTE,
+ f"https://github.com/{github_repository}.git",
+ ],
+ check=True,
+ )
+ else:
+ get_console().print(
+ f"[error]Error {ex}[/]\n" f"[error]When checking if {HTTPS_REMOTE} is set.[/]\n\n"
+ )
+ sys.exit(1)
+ get_console().print("[info]Fetching full history and tags from remote.")
+ get_console().print("[info]This might override your local tags!")
+ result = run_command(
+ ["git", "rev-parse", "--is-shallow-repository"],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ is_shallow_repo = result.stdout.strip() == "true"
+ fetch_command = ["git", "fetch", "--tags", "--force", HTTPS_REMOTE]
+ if is_shallow_repo:
+ fetch_command.append("--unshallow")
+ try:
+ run_command(fetch_command)
+ except subprocess.CalledProcessError as e:
+ get_console().print(
+ f"[error]Error {e}[/]\n"
+ f"[error]When fetching tags from remote. Your tags might not be refreshed.[/]\n\n"
+ f'[warning]Please refresh the tags manually via:[/]\n\n"'
+ f'{" ".join(fetch_command)}\n\n'
+ )
+ sys.exit(1)
+
+
+def convert_pip_requirements_to_table(requirements: Iterable[str], markdown: bool = True) -> str:
+ """
+ Converts PIP requirement list to a Markdown table.
+ :param requirements: requirements list
+ :param markdown: if True, Markdown format is used else rst
+ :return: formatted table
+ """
+ from tabulate import tabulate
+
+ headers = ["PIP package", "Version required"]
+ table_data = []
+ for dependency in requirements:
+ found = re.match(r"(^[^<=>~]*)([^<=>~]?.*)$", dependency)
+ if found:
+ package = found.group(1)
+ version_required = found.group(2)
+ if version_required != "":
+ version_required = f"`{version_required}`" if markdown else f"``{version_required}``"
+ table_data.append((f"`{package}`" if markdown else f"``{package}``", version_required))
+ else:
+ table_data.append((dependency, ""))
+ return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
+
+
+def tag_exists_for_provider(provider_id: str, current_tag: str) -> bool:
+ """Return true if the tag exists in the provider repository."""
+ provider_details = get_provider_details(provider_id)
+ result = run_command(
+ ["git", "rev-parse", current_tag],
+ cwd=provider_details.source_provider_package_path,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ check=False,
+ )
+ return result.returncode == 0
+
+
+def get_latest_provider_tag(provider_id: str, suffix: str) -> str:
+ """Returns latest tag for the provider."""
+ provider_details = get_provider_details(provider_id)
+ current_version = provider_details.versions[0]
+ return get_version_tag(current_version, provider_id, suffix)
diff --git a/dev/breeze/src/airflow_breeze/utils/versions.py b/dev/breeze/src/airflow_breeze/utils/versions.py
index abb8d601f2..88c5986f7d 100644
--- a/dev/breeze/src/airflow_breeze/utils/versions.py
+++ b/dev/breeze/src/airflow_breeze/utils/versions.py
@@ -28,3 +28,9 @@ def strip_leading_zeros_from_version(version: str) -> str:
:return: string with leading 0s after dot replaced.
"""
return ".".join(str(int(i)) for i in version.split("."))
+
+
+def get_version_tag(version: str, provider_package_id: str, version_suffix: str = ""):
+ if version_suffix is None:
+ version_suffix = ""
+ return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py
index d34892fde4..24b9b571a9 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -16,10 +16,13 @@
# under the License.
from __future__ import annotations
+from typing import Iterable
+
import pytest
from airflow_breeze.global_constants import REGULAR_DOC_PACKAGES
from airflow_breeze.utils.packages import (
+ convert_pip_requirements_to_table,
expand_all_provider_packages,
find_matching_long_package_names,
get_available_packages,
@@ -27,6 +30,7 @@ from airflow_breeze.utils.packages import (
get_install_requirements,
get_long_package_name,
get_package_extras,
+ get_pip_package_name,
get_provider_details,
get_provider_requirements,
get_removed_provider_ids,
@@ -34,6 +38,7 @@ from airflow_breeze.utils.packages import (
get_source_package_path,
get_suspended_provider_folders,
get_suspended_provider_ids,
+ get_wheel_package_name,
)
from airflow_breeze.utils.path_utils import AIRFLOW_PROVIDERS_ROOT, AIRFLOW_SOURCES_ROOT, DOCS_ROOT
@@ -151,8 +156,8 @@ def test_get_install_requirements():
assert (
get_install_requirements("asana", "").strip()
== """
- apache-airflow>=2.5.0
- asana>=0.10,<4.0.0
+ "apache-airflow>=2.5.0",
+ "asana>=0.10,<4.0.0",
""".strip()
)
@@ -198,3 +203,55 @@ def test_get_provider_details():
assert provider_details.plugins == []
assert provider_details.changelog_path == provider_details.source_provider_package_path / "CHANGELOG.rst"
assert not provider_details.removed
+
+
+@pytest.mark.parametrize(
+ "provider_id, pip_package_name",
+ [
+ ("asana", "apache-airflow-providers-asana"),
+ ("apache.hdfs", "apache-airflow-providers-apache-hdfs"),
+ ],
+)
+def test_get_pip_package_name(provider_id: str, pip_package_name: str):
+ assert get_pip_package_name(provider_id) == pip_package_name
+
+
+@pytest.mark.parametrize(
+ "provider_id, wheel_package_name",
+ [
+ ("asana", "apache_airflow_providers_asana"),
+ ("apache.hdfs", "apache_airflow_providers_apache_hdfs"),
+ ],
+)
+def test_get_wheel_package_name(provider_id: str, wheel_package_name: str):
+ assert get_wheel_package_name(provider_id) == wheel_package_name
+
+
+@pytest.mark.parametrize(
+ "requirements, markdown, table",
+ [
+ (
+ ["apache-airflow>2.5.0"],
+ False,
+ """
+================== ==================
+PIP package Version required
+================== ==================
+``apache-airflow`` ``>2.5.0``
+================== ==================
+""",
+ ),
+ (
+ ["apache-airflow>2.5.0"],
+ True,
+ """
+| PIP package | Version required |
+|:-----------------|:-------------------|
+| `apache-airflow` | `>2.5.0` |
+""",
+ ),
+ ],
+)
+def test_convert_pip_requirements_to_table(requirements: Iterable[str], markdown: bool, table: str):
+ print(convert_pip_requirements_to_table(requirements, markdown))
+ assert convert_pip_requirements_to_table(requirements, markdown).strip() == table.strip()
diff --git a/dev/breeze/tests/test_provider_documentation.py b/dev/breeze/tests/test_provider_documentation.py
index cf1a1e3024..c11e73349b 100644
--- a/dev/breeze/tests/test_provider_documentation.py
+++ b/dev/breeze/tests/test_provider_documentation.py
@@ -16,22 +16,18 @@
# under the License.
from __future__ import annotations
-from typing import Iterable
-
import pytest
from airflow_breeze.prepare_providers.provider_documentation import (
Change,
_convert_git_changes_to_table,
- _convert_pip_requirements_to_table,
_find_insertion_index_for_version,
_get_change_from_line,
_get_changes_classified,
_get_git_log_command,
- _get_version_tag,
_verify_changelog_exists,
+ get_version_tag,
)
-from airflow_breeze.utils.packages import get_pip_package_name, get_wheel_package_name
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
CHANGELOG_CONTENT = """
@@ -93,7 +89,7 @@ def test_find_insertion_index_insert_new_changelog():
],
)
def test_get_version_tag(version: str, provider_id: str, suffix: str, tag: str):
- assert _get_version_tag(version, provider_id, suffix) == tag
+ assert get_version_tag(version, provider_id, suffix) == tag
@pytest.mark.parametrize(
@@ -121,28 +117,6 @@ def test_get_git_log_command_wrong():
_get_git_log_command(None, "to_commit")
-@pytest.mark.parametrize(
- "provider_id, pip_package_name",
- [
- ("asana", "apache-airflow-providers-asana"),
- ("apache.hdfs", "apache-airflow-providers-apache-hdfs"),
- ],
-)
-def test_get_pip_package_name(provider_id: str, pip_package_name: str):
- assert get_pip_package_name(provider_id) == pip_package_name
-
-
-@pytest.mark.parametrize(
- "provider_id, wheel_package_name",
- [
- ("asana", "apache_airflow_providers_asana"),
- ("apache.hdfs", "apache_airflow_providers_apache_hdfs"),
- ],
-)
-def test_get_wheel_package_name(provider_id: str, wheel_package_name: str):
- assert get_wheel_package_name(provider_id) == wheel_package_name
-
-
@pytest.mark.parametrize(
"line, version, change",
[
@@ -236,36 +210,6 @@ def test_convert_git_changes_to_table(input: str, output: str, markdown: bool, c
assert list_of_changes[2].pr == "12346"
-@pytest.mark.parametrize(
- "requirements, markdown, table",
- [
- (
- ["apache-airflow>2.5.0"],
- False,
- """
-================== ==================
-PIP package Version required
-================== ==================
-``apache-airflow`` ``>2.5.0``
-================== ==================
-""",
- ),
- (
- ["apache-airflow>2.5.0"],
- True,
- """
-| PIP package | Version required |
-|:-----------------|:-------------------|
-| `apache-airflow` | `>2.5.0` |
-""",
- ),
- ],
-)
-def test_convert_pip_requirements_to_table(requirements: Iterable[str], markdown: bool, table: str):
- print(_convert_pip_requirements_to_table(requirements, markdown))
- assert _convert_pip_requirements_to_table(requirements, markdown).strip() == table.strip()
-
-
def test_verify_changelog_exists():
assert (
_verify_changelog_exists("asana")
diff --git a/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2 b/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2
deleted file mode 100644
index 83013eefb4..0000000000
--- a/dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
-
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `MANIFEST_TEMPLATE.py.jinja2` IN the `provider_packages` DIRECTORY
-
-
-{% if PROVIDER_PACKAGE_ID == 'amazon' %}
-include airflow/providers/amazon/aws/hooks/batch_waiters.json
-include airflow/providers/amazon/aws/waiters/*.json
-{% elif PROVIDER_PACKAGE_ID == 'cncf.kubernetes' %}
-include airflow/providers/cncf/kubernetes/*.jinja2
-{% endif %}
-
-include NOTICE
-include LICENSE
-include CHANGELOG.rst
-global-exclude __pycache__ *.pyc
diff --git a/dev/provider_packages/PROVIDER_README_TEMPLATE.rst.jinja2 b/dev/provider_packages/PROVIDER_README_TEMPLATE.rst.jinja2
deleted file mode 100644
index 13e088aa94..0000000000
--- a/dev/provider_packages/PROVIDER_README_TEMPLATE.rst.jinja2
+++ /dev/null
@@ -1,109 +0,0 @@
-{#
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
- NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
- OVERWRITTEN WHEN PREPARING PACKAGES.
-
- IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
- `PROVIDER_INDEX_TEMPLATE.rst.jinja2` IN the `dev/provider_packages` DIRECTORY
-
--#}
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-
-Package ``{{ PACKAGE_PIP_NAME }}``
-
-Release: ``{{ RELEASE }}{{ VERSION_SUFFIX }}``
-
-
-{{ PROVIDER_DESCRIPTION | safe }}
-
-Provider package
-----------------
-
-This is a provider package for ``{{PROVIDER_PACKAGE_ID}}`` provider. All classes for this provider package
-are in ``{{FULL_PACKAGE_NAME}}`` python package.
-
-You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}/>`_.
-
-{%- if PROVIDER_REMOVED %}
-
- .. warning::
-
- This provider is not maintained anymore by the community. It has been removed and is not going to be
- updated anymore. The removal was done according to the process described in
- `Removing community providers <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#removing-community-providers>`_
-
- Feel free to contact Airflow Development Mailing List if you have any questions.
-
-{%- endif %}
-
-Installation
-------------
-
-You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
-for the minimum Airflow version supported) via
-``pip install {{PACKAGE_PIP_NAME}}``
-
-The package supports the following python versions: {{ ",".join(SUPPORTED_PYTHON_VERSIONS) }}
-
-{%- if PIP_REQUIREMENTS %}
-
-Requirements
-------------
-
-{{ PIP_REQUIREMENTS_TABLE_RST | safe }}
-
-{%- endif %}
-{%- if CROSS_PROVIDERS_DEPENDENCIES %}
-
-Cross provider package dependencies
------------------------------------
-
-Those are dependencies that might be needed in order to use all the features of the package.
-You need to install the specified provider packages in order to use them.
-
-You can install such cross-provider dependencies when installing from PyPI. For example:
-
-.. code-block:: bash
-
- pip install {{ PACKAGE_PIP_NAME }}[{{ CROSS_PROVIDERS_DEPENDENCIES[0] }}]
-
-
-{{ CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST | safe }}
-
-{%- endif %}
-
-The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}/changelog.html>`_.
diff --git a/dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2 b/dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2
deleted file mode 100644
index c3fdba076b..0000000000
--- a/dev/provider_packages/SETUP_TEMPLATE.cfg.jinja2
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
-#
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `SETUP_TEMPLATE.cfg.jinja2` IN the `dev/provider_packages` DIRECTORY
-
-
-[metadata]
-name = {{ PACKAGE_PIP_NAME }}
-summary = {{ PROVIDER_TYPE }} for Apache Airflow. Implements {{ PACKAGE_PIP_NAME }} package
-description = {{ PROVIDER_TYPE }} package {{ PACKAGE_PIP_NAME }} for Apache Airflow
-long_description = file: README.rst
-long_description_content_type = text/x-rst
-author = Apache Software Foundation
-author_email = dev@airflow.apache.org
-url = https://airflow.apache.org/
-download_url = https://archive.apache.org/dist/airflow/{{ PROVIDERS_FOLDER }}
-license = Apache License 2.0
-license_files =
- LICENSE
- NOTICE
-classifiers =
- Development Status :: 5 - Production/Stable
- Environment :: Console
- Environment :: Web Environment
- Intended Audience :: Developers
- Intended Audience :: System Administrators
- Framework :: Apache Airflow
- Framework :: Apache Airflow :: Provider
- License :: OSI Approved :: Apache Software License
-{%- for python_version in SUPPORTED_PYTHON_VERSIONS %}
- Programming Language :: Python :: {{ python_version }}
-{%- endfor %}
- Topic :: System :: Monitoring
-project_urls=
- Documentation=https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}/
- Changelog=https://airflow.apache.org/docs/{{ PACKAGE_PIP_NAME }}/{{RELEASE}}/changelog.html
- Bug Tracker=https://github.com/apache/airflow/issues
- Source Code=https://github.com/apache/airflow
- Slack Chat=https://s.apache.org/airflow-slack
- Twitter=https://twitter.com/ApacheAirflow
- YouTube=https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
-
-[bdist_wheel]
-python_tag=py3
-
-[options]
-zip_safe = False
-include_package_data = True
-python_requires = ~=3.8
-packages = find:
-setup_requires = {{ SETUP_REQUIREMENTS }}
-install_requires = {{ INSTALL_REQUIREMENTS }}
-
-[options.entry_points]
-apache_airflow_provider=
- provider_info=airflow.providers.{{ PROVIDER_PACKAGE_ID }}.get_provider_info:get_provider_info
-{%- if PLUGINS %}
-airflow.plugins=
-{%- for plugin in PLUGINS %}
- {{ plugin.name }}={{ plugin.package_name }}:{{ plugin.class_name }}
-{%- endfor %}
-{%- endif %}
-
-
-[files]
-packages = airflow.providers.{{ PROVIDER_PACKAGE_ID }}
diff --git a/dev/provider_packages/SETUP_TEMPLATE.py.jinja2 b/dev/provider_packages/SETUP_TEMPLATE.py.jinja2
deleted file mode 100644
index 4f47265321..0000000000
--- a/dev/provider_packages/SETUP_TEMPLATE.py.jinja2
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
-# OVERWRITTEN WHEN PREPARING PACKAGES.
-#
-# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE
-# `SETUP_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY
-
-"""Setup.py for the {{ PACKAGE_PIP_NAME }} package."""
-
-from setuptools import find_namespace_packages, setup
-
-version = '{{ RELEASE_NO_LEADING_ZEROS }}'
-
-
-def do_setup():
- """Perform the package {{ PACKAGE_PIP_NAME }} setup."""
- setup(
- version=version,
- extras_require={{ EXTRAS_REQUIREMENTS }},
- packages=find_namespace_packages(
- include=['airflow.providers.{{ PROVIDER_PACKAGE_ID }}',
- 'airflow.providers.{{ PROVIDER_PACKAGE_ID }}.*',
- 'airflow.providers.{{ PROVIDER_PACKAGE_ID }}_vendor',
- 'airflow.providers.{{ PROVIDER_PACKAGE_ID }}_vendor.*'],
- ),
- )
-
-
-if __name__ == "__main__":
- do_setup()
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
deleted file mode 100755
index f018c30c21..0000000000
--- a/dev/provider_packages/prepare_provider_packages.py
+++ /dev/null
@@ -1,1277 +0,0 @@
-#!/usr/bin/env python3
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Setup.py for the Provider packages of Airflow project."""
-from __future__ import annotations
-
-import glob
-import json
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
-import textwrap
-from collections import namedtuple
-from contextlib import contextmanager
-from datetime import datetime, timedelta
-from enum import Enum
-from functools import lru_cache
-from pathlib import Path
-from shutil import copyfile
-from typing import Any, Generator, Iterable, NamedTuple
-
-import jinja2
-import jsonschema
-import rich_click as click
-import semver as semver
-from black import Mode, TargetVersion, format_str, parse_pyproject_toml
-from packaging.version import Version
-from rich.console import Console
-from rich.syntax import Syntax
-from yaml import safe_load
-
-ALL_PYTHON_VERSIONS = ["3.8", "3.9", "3.10", "3.11"]
-
-MIN_AIRFLOW_VERSION = "2.5.0"
-
-INITIAL_CHANGELOG_CONTENT = """
- .. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- .. http://www.apache.org/licenses/LICENSE-2.0
-
- .. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
-.. NOTE TO CONTRIBUTORS:
- Please, only add notes to the Changelog just below the "Changelog" header when there
- are some breaking changes and you want to add an explanation to the users on how they are supposed
- to deal with them. The changelog is updated and maintained semi-automatically by release manager.
-
-``{{ package_name }}``
-
-Changelog
----------
-
-1.0.0
-.....
-
-Initial version of the provider.
-"""
-
-HTTPS_REMOTE = "apache-https-for-providers"
-HEAD_OF_HTTPS_REMOTE = f"{HTTPS_REMOTE}"
-
-MY_DIR_PATH = Path(__file__).parent
-AIRFLOW_SOURCES_ROOT_PATH = MY_DIR_PATH.parents[1]
-AIRFLOW_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow"
-DIST_PATH = AIRFLOW_SOURCES_ROOT_PATH / "dist"
-PROVIDERS_PATH = AIRFLOW_PATH / "providers"
-DOCUMENTATION_PATH = AIRFLOW_SOURCES_ROOT_PATH / "docs"
-
-DEPENDENCIES_JSON_FILE_PATH = AIRFLOW_SOURCES_ROOT_PATH / "generated" / "provider_dependencies.json"
-
-TARGET_PROVIDER_PACKAGES_PATH = AIRFLOW_SOURCES_ROOT_PATH / "provider_packages"
-GENERATED_AIRFLOW_PATH = TARGET_PROVIDER_PACKAGES_PATH / "airflow"
-GENERATED_PROVIDERS_PATH = GENERATED_AIRFLOW_PATH / "providers"
-
-PROVIDER_RUNTIME_DATA_SCHEMA_PATH = AIRFLOW_SOURCES_ROOT_PATH / "airflow" / "provider_info.schema.json"
-
-CROSS_PROVIDERS_DEPS = "cross-providers-deps"
-DEPS = "deps"
-
-sys.path.insert(0, str(AIRFLOW_SOURCES_ROOT_PATH))
-
-
-ALL_DEPENDENCIES = json.loads(DEPENDENCIES_JSON_FILE_PATH.read_text())
-
-# those imports need to come after the above sys.path.insert to make sure that Airflow
-# sources are importable without having to add the airflow sources to the PYTHONPATH before
-# running the script
-from setup import PREINSTALLED_PROVIDERS, ALL_PROVIDERS # type: ignore[attr-defined] # isort:skip # noqa
-
-# Note - we do not test protocols as they are not really part of the official API of
-# Apache Airflow
-
-logger = logging.getLogger(__name__)
-
-PY3 = sys.version_info[0] == 3
-
-console = Console(width=400, color_system="standard")
-
-
-class PluginInfo(NamedTuple):
- name: str
- package_name: str
- class_name: str
-
-
-class ProviderPackageDetails(NamedTuple):
- provider_package_id: str
- full_package_name: str
- pypi_package_name: str
- source_provider_package_path: str
- documentation_provider_package_path: Path
- provider_description: str
- versions: list[str]
- excluded_python_versions: list[str]
- plugins: list[PluginInfo]
- removed: bool
-
-
-class EntityType(Enum):
- Operators = "Operators"
- Transfers = "Transfers"
- Sensors = "Sensors"
- Hooks = "Hooks"
- Secrets = "Secrets"
-
-
-@click.group(context_settings={"help_option_names": ["-h", "--help"], "max_content_width": 500})
-def cli():
- ...
-
-
-option_skip_tag_check = click.option(
- "--skip-tag-check/--no-skip-tag-check",
- default=False,
- is_flag=True,
- help="Skip checking if the tag already exists in the remote repository",
-)
-
-option_git_update = click.option(
- "--git-update/--no-git-update",
- default=True,
- is_flag=True,
- help=f"If the git remote {HTTPS_REMOTE} already exists, don't try to update it",
-)
-
-option_package_format = click.option(
- "--package-format",
- type=click.Choice(["wheel", "sdist", "both"]),
- help="Format of packages.",
- default="wheel",
- show_default=True,
- envvar="PACKAGE_FORMAT",
-)
-
-option_version_suffix = click.option(
- "--version-suffix",
- metavar="suffix",
- help=textwrap.dedent(
- """
- adds version suffix to version of the packages.
- only useful when generating rc candidates for pypi."""
- ),
-)
-option_verbose = click.option(
- "--verbose",
- is_flag=True,
- help="Print verbose information about performed steps",
-)
-argument_package_id = click.argument("package_id")
-
-
-@contextmanager
-def with_group(title: str) -> Generator[None, None, None]:
- """
- If used in GitHub Action, creates an expandable group in the GitHub Action log.
- Otherwise, display simple text groups.
-
- For more information, see:
- https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#grouping-log-lines
- """
- if os.environ.get("GITHUB_ACTIONS", "false") != "true":
- console.print("#" * 10 + " [bright_blue]" + title + "[/] " + "#" * 10)
- yield
- return
- console.print(f"::group::[bright_blue]{title}[/]")
- yield
- console.print("::endgroup::")
-
-
-def get_source_airflow_folder() -> str:
- """
- Returns source directory for whole airflow (from the main airflow project).
-
- :return: the folder path
- """
- return os.path.abspath(AIRFLOW_SOURCES_ROOT_PATH)
-
-
-def get_source_providers_folder() -> str:
- """
- Returns source directory for providers (from the main airflow project).
-
- :return: the folder path
- """
- return os.path.join(get_source_airflow_folder(), "airflow", "providers")
-
-
-def get_target_folder() -> str:
- """
- Returns target directory for providers (in the provider_packages folder)
-
- :return: the folder path
- """
- return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "provider_packages"))
-
-
-def get_target_providers_folder() -> str:
- """
- Returns target directory for providers (in the provider_packages folder)
-
- :return: the folder path
- """
- return os.path.abspath(os.path.join(get_target_folder(), "airflow", "providers"))
-
-
-def get_target_providers_package_folder(provider_package_id: str) -> str:
- """
- Returns target package folder based on package_id
-
- :return: the folder path
- """
- return os.path.join(get_target_providers_folder(), *provider_package_id.split("."))
-
-
-def get_pip_package_name(provider_package_id: str) -> str:
- """
- Returns PIP package name for the package id.
-
- :param provider_package_id: id of the package
- :return: the name of pip package
- """
- return "apache-airflow-providers-" + provider_package_id.replace(".", "-")
-
-
-def get_wheel_package_name(provider_package_id: str) -> str:
- """
- Returns Wheel package name for the package id.
-
- :param provider_package_id: id of the package
- :return: the name of pip package
- """
- return "apache_airflow_providers_" + provider_package_id.replace(".", "_")
-
-
-def get_install_requirements(provider_package_id: str, version_suffix: str) -> str:
- """
- Returns install requirements for the package.
-
- :param provider_package_id: id of the provider package
- :param version_suffix: optional version suffix for packages
-
- :return: install requirements of the package
- """
-
- def apply_version_suffix(install_clause: str) -> str:
- if install_clause.startswith("apache-airflow") and ">=" in install_clause and version_suffix != "":
- # This is workaround for `pip` way of handling `--pre` installation switch. It apparently does
- # not modify the meaning of `install_requires` to include also pre-releases, so we need to
- # modify our internal provider and airflow package version references to include all pre-releases
- # including all development releases. When you specify dependency as >= X.Y.Z, and you
- # have packages X.Y.Zdev0 or X.Y.Zrc1 in a local file, such package is not considered
- # as fulfilling the requirement even if `--pre` switch is used.
- return install_clause + ".dev0"
- return install_clause
-
- if provider_package_id in get_removed_provider_ids():
- provider_info = get_provider_info_from_provider_yaml(provider_package_id)
- dependencies = provider_info["dependencies"]
- else:
- dependencies = ALL_DEPENDENCIES[provider_package_id][DEPS]
- install_requires = [apply_version_suffix(clause) for clause in dependencies]
- return "".join(f"\n {ir}" for ir in install_requires)
-
-
-def get_setup_requirements() -> str:
- """
- Returns setup requirements (common for all package for now).
- :return: setup requirements
- """
- return """
- setuptools
- wheel
-"""
-
-
-def get_package_extras(provider_package_id: str) -> dict[str, list[str]]:
- """
- Finds extras for the package specified.
-
- :param provider_package_id: id of the package
- """
- if provider_package_id == "providers":
- return {}
- if provider_package_id in get_removed_provider_ids():
- return {}
- extras_dict: dict[str, list[str]] = {
- module: [get_pip_package_name(module)]
- for module in ALL_DEPENDENCIES[provider_package_id][CROSS_PROVIDERS_DEPS]
- }
- provider_yaml_dict = get_provider_yaml(provider_package_id)
- additional_extras = provider_yaml_dict.get("additional-extras")
- if additional_extras:
- for entry in additional_extras:
- name = entry["name"]
- dependencies = entry["dependencies"]
- if name in extras_dict:
- # remove non-versioned dependencies if versioned ones are coming
- existing_dependencies = set(extras_dict[name])
- for new_dependency in dependencies:
- for dependency in existing_dependencies:
- # remove extra if exists as non-versioned one
- if new_dependency.startswith(dependency):
- extras_dict[name].remove(dependency)
- break
- extras_dict[name].append(new_dependency)
- else:
- extras_dict[name] = dependencies
- return extras_dict
-
-
-def render_template(
- template_name: str,
- context: dict[str, Any],
- extension: str,
- autoescape: bool = True,
- keep_trailing_newline: bool = False,
-) -> str:
- """
- Renders template based on its name. Reads the template from <name>_TEMPLATE.md.jinja2 in current dir.
- :param template_name: name of the template to use
- :param context: Jinja2 context
- :param extension: Target file extension
- :param autoescape: Whether to autoescape HTML
- :param keep_trailing_newline: Whether to keep the newline in rendered output
- :return: rendered template
- """
- import jinja2
-
- template_loader = jinja2.FileSystemLoader(searchpath=MY_DIR_PATH)
- template_env = jinja2.Environment(
- loader=template_loader,
- undefined=jinja2.StrictUndefined,
- autoescape=autoescape,
- keep_trailing_newline=keep_trailing_newline,
- )
- template = template_env.get_template(f"{template_name}_TEMPLATE{extension}.jinja2")
- content: str = template.render(context)
- return content
-
-
-PR_PATTERN = re.compile(r".*\(#(\d+)\)")
-
-
-class Change(NamedTuple):
- """Stores details about commits"""
-
- full_hash: str
- short_hash: str
- date: str
- version: str
- message: str
- message_without_backticks: str
- pr: str | None
-
-
-def get_change_from_line(line: str, version: str):
- split_line = line.split(" ", maxsplit=3)
- message = split_line[3]
- pr = None
- pr_match = PR_PATTERN.match(message)
- if pr_match:
- pr = pr_match.group(1)
- return Change(
- full_hash=split_line[0],
- short_hash=split_line[1],
- date=split_line[2],
- version=version,
- message=message,
- message_without_backticks=message.replace("`", "'").replace("&39;", "'"),
- pr=pr,
- )
-
-
-def convert_pip_requirements_to_table(requirements: Iterable[str], markdown: bool = True) -> str:
- """
- Converts PIP requirement list to a Markdown table.
- :param requirements: requirements list
- :param markdown: if True, Markdown format is used else rst
- :return: formatted table
- """
- from tabulate import tabulate
-
- headers = ["PIP package", "Version required"]
- table_data = []
- for dependency in requirements:
- found = re.match(r"(^[^<=>~]*)([^<=>~]?.*)$", dependency)
- if found:
- package = found.group(1)
- version_required = found.group(2)
- if version_required != "":
- version_required = f"`{version_required}`" if markdown else f"``{version_required}``"
- table_data.append((f"`{package}`" if markdown else f"``{package}``", version_required))
- else:
- table_data.append((dependency, ""))
- return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
-
-
-def convert_cross_package_dependencies_to_table(
- cross_package_dependencies: list[str],
- markdown: bool = True,
-) -> str:
- """
- Converts cross-package dependencies to a Markdown table
- :param cross_package_dependencies: list of cross-package dependencies
- :param markdown: if True, Markdown format is used else rst
- :return: formatted table
- """
- from tabulate import tabulate
-
- headers = ["Dependent package", "Extra"]
- table_data = []
- prefix = "apache-airflow-providers-"
- base_url = "https://airflow.apache.org/docs/"
- for dependency in cross_package_dependencies:
- pip_package_name = f"{prefix}{dependency.replace('.','-')}"
- url_suffix = f"{dependency.replace('.','-')}"
- if markdown:
- url = f"[{pip_package_name}]({base_url}{url_suffix})"
- else:
- url = f"`{pip_package_name} <{base_url}{prefix}{url_suffix}>`_"
- table_data.append((url, f"`{dependency}`" if markdown else f"``{dependency}``"))
- return tabulate(table_data, headers=headers, tablefmt="pipe" if markdown else "rst")
-
-
-LICENCE = """<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
- -->
-"""
-
-LICENCE_RST = """
-.. Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
-.. http://www.apache.org/licenses/LICENSE-2.0
-
-.. Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-"""
-
-"""
-Keeps information about historical releases.
-"""
-ReleaseInfo = namedtuple(
- "ReleaseInfo", "release_version release_version_no_leading_zeros last_commit_hash content file_name"
-)
-
-
-def strip_leading_zeros(version: str) -> str:
- """
- Strips leading zeros from version number.
-
- This converts 1974.04.03 to 1974.4.3 as the format with leading month and day zeros is not accepted
- by PIP versioning.
-
- :param version: version number in CALVER format (potentially with leading 0s in date and month)
- :return: string with leading 0s after dot replaced.
- """
- return ".".join(str(int(i)) for i in version.split("."))
-
-
-def get_previous_release_info(
- previous_release_version: str | None, past_releases: list[ReleaseInfo], current_release_version: str
-) -> str | None:
- """Find previous release.
-
- In case we are re-running current release, we assume that last release was
- the previous one. This is needed so that we can generate list of changes
- since the previous release.
-
- :param previous_release_version: known last release version
- :param past_releases: list of past releases
- :param current_release_version: release that we are working on currently
- """
- previous_release = None
- if previous_release_version == current_release_version:
- # Re-running for current release - use previous release as base for git log
- if len(past_releases) > 1:
- previous_release = past_releases[1].last_commit_hash
- else:
- previous_release = past_releases[0].last_commit_hash if past_releases else None
- return previous_release
-
-
-def check_if_release_version_ok(
- past_releases: list[ReleaseInfo],
- current_release_version: str,
-) -> tuple[str, str | None]:
- """Check if the release version passed is not later than the last release version.
-
- :param past_releases: all past releases (if there are any)
- :param current_release_version: release version to check
- :return: Tuple of current/previous_release (previous might be None if there are no releases)
- """
- previous_release_version = past_releases[0].release_version if past_releases else None
- if current_release_version == "":
- if previous_release_version:
- current_release_version = previous_release_version
- else:
- current_release_version = (datetime.today() + timedelta(days=5)).strftime("%Y.%m.%d")
- if previous_release_version:
- if Version(current_release_version) < Version(previous_release_version):
- console.print(
- f"[red]The release {current_release_version} must be not less than "
- f"{previous_release_version} - last release for the package[/]"
- )
- raise Exception("Bad release version")
- return current_release_version, previous_release_version
-
-
-def get_cross_provider_dependent_packages(provider_package_id: str) -> list[str]:
- """Returns cross-provider dependencies for the package.
-
- :param provider_package_id: package id
- :return: list of cross-provider dependencies
- """
- if provider_package_id in get_removed_provider_ids():
- return []
- return ALL_DEPENDENCIES[provider_package_id][CROSS_PROVIDERS_DEPS]
-
-
-def make_current_directory_safe(verbose: bool):
- """Makes current directory safe for Git.
-
- New git checks if git ownership for the folder is not manipulated with. We
- are running this command only inside the container where the directory is
- mounted from "regular" user to "root" user which is used inside the
- container, so this is quite ok to assume the directory it is used is safe.
-
- It's also ok to leave it as safe - it is a global option inside the
- container so it will disappear when we exit.
-
- :param verbose: whether to print commands being executed
- """
- safe_dir_remove_command = ["git", "config", "--global", "--unset-all", "safe.directory"]
- if verbose:
- console.print(f"Running command: '{' '.join(safe_dir_remove_command)}'")
- # we ignore result of this call
- subprocess.call(safe_dir_remove_command)
- safe_dir_add_command = ["git", "config", "--global", "--add", "safe.directory", "/opt/airflow"]
- if verbose:
- console.print(f"Running command: '{' '.join(safe_dir_add_command)}'")
- subprocess.check_call(safe_dir_add_command)
-
-
-def get_git_tag_check_command(tag: str) -> list[str]:
- """Get git command to check if tag exits.
-
- :param tag: Tag to check
- :return: git command to run
- """
- return [
- "git",
- "rev-parse",
- tag,
- ]
-
-
-def get_source_package_path(provider_package_id: str) -> str:
- """Retrieves source package path from package id.
-
- :param provider_package_id: id of the package
- :return: path of the providers folder
- """
- return os.path.join(PROVIDERS_PATH, *provider_package_id.split("."))
-
-
-def get_documentation_package_path(provider_package_id: str) -> Path:
- """Retrieves documentation package path from package id.
-
- :param provider_package_id: id of the package
- :return: path of the documentation folder
- """
- return DOCUMENTATION_PATH / f"apache-airflow-providers-{provider_package_id.replace('.','-')}"
-
-
-def get_generated_package_path(provider_package_id: str) -> str:
- """Retrieves generated package path from package id.
-
- :param provider_package_id: id of the package
- :return: path of the providers folder
- """
- provider_package_path = os.path.join(GENERATED_PROVIDERS_PATH, *provider_package_id.split("."))
- return provider_package_path
-
-
-def validate_provider_info_with_runtime_schema(provider_info: dict[str, Any]) -> None:
- """Validates provider info against the runtime schema.
-
- This way we check if the provider info in the packages is future-compatible.
- The Runtime Schema should only change when there is a major version change.
-
- :param provider_info: provider info to validate
- """
-
- with open(PROVIDER_RUNTIME_DATA_SCHEMA_PATH) as schema_file:
- schema = json.load(schema_file)
- try:
- jsonschema.validate(provider_info, schema=schema)
- except jsonschema.ValidationError as ex:
- console.print("[red]Provider info not validated against runtime schema[/]")
- raise Exception(
- "Error when validating schema. The schema must be compatible with "
- "airflow/provider_info.schema.json.",
- ex,
- )
-
-
-def get_provider_yaml(provider_package_id: str) -> dict[str, Any]:
- """Retrieves provider info from the provider YAML file.
-
- The provider yaml file contains more information than provider_info that is
- used at runtime. This method converts the full provider yaml file into
- stripped-down provider info and validates it against deprecated 2.0.0 schema
- and runtime schema.
-
- :param provider_package_id: package id to retrieve provider.yaml from
- :return: provider_info dictionary
- """
- provider_yaml_file_name = os.path.join(get_source_package_path(provider_package_id), "provider.yaml")
- if not os.path.exists(provider_yaml_file_name):
- raise Exception(f"The provider.yaml file is missing: {provider_yaml_file_name}")
- with open(provider_yaml_file_name) as provider_file:
- provider_yaml_dict = safe_load(provider_file)
- return provider_yaml_dict
-
-
-def get_provider_info_from_provider_yaml(provider_package_id: str) -> dict[str, Any]:
- """Retrieves provider info from the provider yaml file.
-
- :param provider_package_id: package id to retrieve provider.yaml from
- :return: provider_info dictionary
- """
- provider_yaml_dict = get_provider_yaml(provider_package_id=provider_package_id)
- validate_provider_info_with_runtime_schema(provider_yaml_dict)
- return provider_yaml_dict
-
-
-def get_version_tag(version: str, provider_package_id: str, version_suffix: str = ""):
- if version_suffix is None:
- version_suffix = ""
- return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
-
-
-def get_provider_details(provider_package_id: str) -> ProviderPackageDetails:
- provider_info = get_provider_info_from_provider_yaml(provider_package_id)
- plugins: list[PluginInfo] = []
- if "plugins" in provider_info:
- for plugin in provider_info["plugins"]:
- package_name, class_name = plugin["plugin-class"].rsplit(".", maxsplit=1)
- plugins.append(
- PluginInfo(
- name=plugin["name"],
- package_name=package_name,
- class_name=class_name,
- )
- )
- return ProviderPackageDetails(
- provider_package_id=provider_package_id,
- full_package_name=f"airflow.providers.{provider_package_id}",
- pypi_package_name=f"apache-airflow-providers-{provider_package_id.replace('.', '-')}",
- source_provider_package_path=get_source_package_path(provider_package_id),
- documentation_provider_package_path=get_documentation_package_path(provider_package_id),
- provider_description=provider_info["description"],
- versions=provider_info["versions"],
- excluded_python_versions=provider_info.get("excluded-python-versions") or [],
- plugins=plugins,
- removed=provider_info.get("removed", False),
- )
-
-
-def get_provider_requirements(provider_package_id: str) -> list[str]:
- provider_yaml = get_provider_yaml(provider_package_id)
- return provider_yaml["dependencies"]
-
-
-def get_provider_jinja_context(
- provider_info: dict[str, Any],
- provider_details: ProviderPackageDetails,
- current_release_version: str,
- version_suffix: str,
-):
- verify_provider_package(provider_details.provider_package_id)
- changelog_path = verify_changelog_exists(provider_details.provider_package_id)
- cross_providers_dependencies = get_cross_provider_dependent_packages(
- provider_package_id=provider_details.provider_package_id
- )
- release_version_no_leading_zeros = strip_leading_zeros(current_release_version)
- pip_requirements_table = convert_pip_requirements_to_table(
- get_provider_requirements(provider_details.provider_package_id)
- )
- pip_requirements_table_rst = convert_pip_requirements_to_table(
- get_provider_requirements(provider_details.provider_package_id), markdown=False
- )
- cross_providers_dependencies_table_rst = convert_cross_package_dependencies_to_table(
- cross_providers_dependencies, markdown=False
- )
- with open(changelog_path) as changelog_file:
- changelog = changelog_file.read()
- supported_python_versions = [
- p for p in ALL_PYTHON_VERSIONS if p not in provider_details.excluded_python_versions
- ]
- python_requires = "~=3.8"
- for p in provider_details.excluded_python_versions:
- python_requires += f", !={p}"
- min_airflow_version = MIN_AIRFLOW_VERSION
- for dependency in provider_info["dependencies"]:
- if dependency.startswith("apache-airflow>="):
- current_min_airflow_version = dependency.split(">=")[1]
- if Version(current_min_airflow_version) > Version(min_airflow_version):
- min_airflow_version = current_min_airflow_version
- context: dict[str, Any] = {
- "ENTITY_TYPES": list(EntityType),
- "README_FILE": "README.rst",
- "PROVIDER_PACKAGE_ID": provider_details.provider_package_id,
- "PACKAGE_PIP_NAME": get_pip_package_name(provider_details.provider_package_id),
- "PACKAGE_WHEEL_NAME": get_wheel_package_name(provider_details.provider_package_id),
- "FULL_PACKAGE_NAME": provider_details.full_package_name,
- "PROVIDER_PATH": provider_details.full_package_name.replace(".", "/"),
- "RELEASE": current_release_version,
- "RELEASE_NO_LEADING_ZEROS": release_version_no_leading_zeros,
- "VERSION_SUFFIX": version_suffix or "",
- "CROSS_PROVIDERS_DEPENDENCIES": cross_providers_dependencies,
- "PIP_REQUIREMENTS": get_provider_requirements(provider_details.provider_package_id),
- "PROVIDER_TYPE": "Provider",
- "PROVIDERS_FOLDER": "providers",
- "PROVIDER_DESCRIPTION": provider_details.provider_description,
- "INSTALL_REQUIREMENTS": get_install_requirements(
- provider_package_id=provider_details.provider_package_id, version_suffix=version_suffix
- ),
- "SETUP_REQUIREMENTS": get_setup_requirements(),
- "EXTRAS_REQUIREMENTS": get_package_extras(provider_package_id=provider_details.provider_package_id),
- "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST": cross_providers_dependencies_table_rst,
- "PIP_REQUIREMENTS_TABLE": pip_requirements_table,
- "PIP_REQUIREMENTS_TABLE_RST": pip_requirements_table_rst,
- "PROVIDER_INFO": provider_info,
- "CHANGELOG_RELATIVE_PATH": os.path.relpath(
- provider_details.source_provider_package_path,
- provider_details.documentation_provider_package_path,
- ),
- "CHANGELOG": changelog,
- "SUPPORTED_PYTHON_VERSIONS": supported_python_versions,
- "PYTHON_REQUIRES": python_requires,
- "PLUGINS": provider_details.plugins,
- "MIN_AIRFLOW_VERSION": min_airflow_version,
- "PREINSTALLED_PROVIDER": provider_details.provider_package_id in PREINSTALLED_PROVIDERS,
- "PROVIDER_REMOVED": provider_details.removed,
- }
- return context
-
-
-def prepare_readme_file(context):
- readme_content = LICENCE_RST + render_template(
- template_name="PROVIDER_README", context=context, extension=".rst"
- )
- readme_file_path = os.path.join(TARGET_PROVIDER_PACKAGES_PATH, "README.rst")
- with open(readme_file_path, "w") as readme_file:
- readme_file.write(readme_content)
-
-
-def update_setup_files(
- provider_package_id: str,
- version_suffix: str,
-):
- """Updates generated setup.cfg/setup.py/manifest.in/provider_info for packages.
-
- :param provider_package_id: id of the package
- :param version_suffix: version suffix corresponding to the version in the code
- :returns False if the package should be skipped, True if everything generated properly
- """
- verify_provider_package(provider_package_id)
- provider_details = get_provider_details(provider_package_id)
- provider_info = get_provider_info_from_provider_yaml(provider_package_id)
- current_release_version = provider_details.versions[0]
- jinja_context = get_provider_jinja_context(
- provider_info=provider_info,
- provider_details=provider_details,
- current_release_version=current_release_version,
- version_suffix=version_suffix,
- )
- console.print()
- console.print(f"Generating setup files for {provider_package_id}")
- console.print()
- prepare_setup_py_file(jinja_context)
- prepare_setup_cfg_file(jinja_context)
- prepare_get_provider_info_py_file(jinja_context, provider_package_id)
- prepare_manifest_in_file(jinja_context)
- prepare_readme_file(jinja_context)
- return True
-
-
-def replace_content(file_path, old_text, new_text, provider_package_id):
- if new_text != old_text:
- _, temp_file_path = tempfile.mkstemp()
- try:
- if os.path.isfile(file_path):
- copyfile(file_path, temp_file_path)
- with open(file_path, "w") as readme_file:
- readme_file.write(new_text)
- console.print()
- console.print(f"Generated {file_path} file for the {provider_package_id} provider")
- console.print()
- if old_text != "":
- subprocess.call(["diff", "--color=always", temp_file_path, file_path])
- finally:
- os.remove(temp_file_path)
-
-
-AUTOMATICALLY_GENERATED_MARKER = "AUTOMATICALLY GENERATED"
-AUTOMATICALLY_GENERATED_CONTENT = (
- f".. THE REMAINDER OF THE FILE IS {AUTOMATICALLY_GENERATED_MARKER}. "
- f"IT WILL BE OVERWRITTEN AT RELEASE TIME!"
-)
-
-
-# Taken from pygrep hooks we are using in pre-commit
-# https://github.com/pre-commit/pygrep-hooks/blob/main/.pre-commit-hooks.yaml
-BACKTICKS_CHECK = re.compile(r"^(?! ).*(^| )`[^`]+`([^_]|$)", re.MULTILINE)
-
-
-def _update_file(
- context: dict[str, Any],
- template_name: str,
- extension: str,
- file_name: str,
- provider_package_id: str,
- target_path: Path,
- regenerate_missing_docs: bool,
-) -> bool:
- file_path = target_path / file_name
- if regenerate_missing_docs and file_path.exists():
- return True
- new_text = render_template(
- template_name=template_name, context=context, extension=extension, keep_trailing_newline=True
- )
- file_path = target_path / file_name
- old_text = ""
- if os.path.isfile(file_path):
- with open(file_path) as readme_file_read:
- old_text = readme_file_read.read()
- replace_content(file_path, old_text, new_text, provider_package_id)
- index_path = target_path / "index.rst"
- if not index_path.exists():
- console.print(f"[red]ERROR! The index must exist for the provider docs: {index_path}")
- sys.exit(1)
-
- expected_link_in_index = f"<{file_name.split('.')[0]}>"
- if expected_link_in_index not in index_path.read_text():
- console.print(
- f"\n[red]ERROR! The {index_path} must contain "
- f"link to the generated documentation:[/]\n\n"
- f"[yellow]{expected_link_in_index}[/]\n\n"
- f"[bright_blue]Please make sure to add it to {index_path}.\n"
- )
-
- console.print(f"Checking for backticks correctly generated in: {file_path}")
- match = BACKTICKS_CHECK.search(file_path.read_text())
- if match:
- console.print(
- f"\n[red]ERROR: Single backticks (`) found in {file_path}:[/]\n\n"
- f"[yellow]{match.group(0)}[/]\n\n"
- f"[bright_blue]Please fix them by replacing with double backticks (``).[/]\n"
- )
- return False
-
- # TODO: uncomment me. Linting revealed that our already generated provider docs have duplicate links
- # in the generated files, we should fix those and uncomment linting as separate step - so that
- # we do not hold current release for fixing the docs.
- # console.print(f"Linting: {file_path}")
- # errors = restructuredtext_lint.lint_file(file_path)
- # real_errors = False
- # if errors:
- # for error in errors:
- # # Skip known issue: linter with doc role similar to https://github.com/OCA/pylint-odoo/issues/38
- # if (
- # 'No role entry for "doc"' in error.message
- # or 'Unknown interpreted text role "doc"' in error.message
- # ):
- # continue
- # real_errors = True
- # console.print(f"* [red] {error.message}")
- # if real_errors:
- # console.print(f"\n[red] Errors found in {file_path}")
- # return False
-
- console.print(f"[green]Generated {file_path} for {provider_package_id} is OK[/]")
-
- return True
-
-
-@lru_cache(maxsize=None)
-def black_mode() -> Mode:
- config = parse_pyproject_toml(os.path.join(AIRFLOW_SOURCES_ROOT_PATH, "pyproject.toml"))
- target_versions = {TargetVersion[val.upper()] for val in config.get("target_version", ())}
- return Mode(
- target_versions=target_versions,
- line_length=config.get("line_length", Mode.line_length),
- )
-
-
-def black_format(content) -> str:
- return format_str(content, mode=black_mode())
-
-
-def prepare_setup_py_file(context):
- setup_py_template_name = "SETUP"
- setup_py_file_path = os.path.abspath(os.path.join(get_target_folder(), "setup.py"))
- setup_py_content = render_template(
- template_name=setup_py_template_name, context=context, extension=".py", autoescape=False
- )
- with open(setup_py_file_path, "w") as setup_py_file:
- setup_py_file.write(black_format(setup_py_content))
-
-
-def prepare_setup_cfg_file(context):
- setup_cfg_template_name = "SETUP"
- setup_cfg_file_path = os.path.abspath(os.path.join(get_target_folder(), "setup.cfg"))
- setup_cfg_content = render_template(
- template_name=setup_cfg_template_name,
- context=context,
- extension=".cfg",
- autoescape=False,
- keep_trailing_newline=True,
- )
- with open(setup_cfg_file_path, "w") as setup_cfg_file:
- setup_cfg_file.write(setup_cfg_content)
-
-
-def prepare_get_provider_info_py_file(context, provider_package_id: str):
- get_provider_template_name = "get_provider_info"
- get_provider_file_path = os.path.abspath(
- os.path.join(
- get_target_providers_package_folder(provider_package_id),
- "get_provider_info.py",
- )
- )
- get_provider_content = render_template(
- template_name=get_provider_template_name,
- context=context,
- extension=".py",
- autoescape=False,
- keep_trailing_newline=True,
- )
- with open(get_provider_file_path, "w") as get_provider_file:
- get_provider_file.write(black_format(get_provider_content))
-
-
-def prepare_manifest_in_file(context):
- target = os.path.abspath(os.path.join(get_target_folder(), "MANIFEST.in"))
- content = render_template(
- template_name="MANIFEST",
- context=context,
- extension=".in",
- autoescape=False,
- keep_trailing_newline=True,
- )
- with open(target, "w") as fh:
- fh.write(content)
-
-
-def get_all_providers() -> list[str]:
- """Returns all providers for regular packages.
-
- :return: list of providers that are considered for provider packages
- """
- return list(ALL_PROVIDERS)
-
-
-def get_removed_provider_ids() -> list[str]:
- """
- Yields the ids of suspended providers.
- """
- import yaml
-
- removed_provider_ids = []
- for provider_path in PROVIDERS_PATH.rglob("provider.yaml"):
- provider_yaml = yaml.safe_load(provider_path.read_text())
- package_name = provider_yaml.get("package-name")
- if provider_yaml.get("removed", False):
- if not provider_yaml.get("suspended"):
- console.print(
- f"[error]The provider {package_name} is marked for removal in provider.yaml, but "
- f"not suspended. Please suspend the provider first before removing it.\n"
- )
- sys.exit(1)
- removed_provider_ids.append(package_name[len("apache-airflow-providers-") :].replace("-", "."))
- return removed_provider_ids
-
-
-def verify_provider_package(provider_package_id: str) -> None:
- """Verifies if the provider package is good.
-
- :param provider_package_id: package id to verify
- """
- if provider_package_id not in get_all_providers():
- if provider_package_id in get_removed_provider_ids():
- console.print()
- console.print(
- f"[yellow]The package: {provider_package_id} is suspended, but "
- f"since you asked for it, it will be built [/]"
- )
- console.print()
- else:
- console.print(f"[red]Wrong package name: {provider_package_id}[/]")
- console.print("Use one of:")
- console.print(get_all_providers())
- console.print(f"[red]The package {provider_package_id} is not a provider package.")
- sys.exit(1)
-
-
-def verify_changelog_exists(package: str) -> str:
- provider_details = get_provider_details(package)
- changelog_path = os.path.join(provider_details.source_provider_package_path, "CHANGELOG.rst")
- if not os.path.isfile(changelog_path):
- console.print(f"\n[red]ERROR: Missing {changelog_path}[/]\n")
- console.print("[info]Please add the file with initial content:")
- console.print("----- START COPYING AFTER THIS LINE ------- ")
- processed_changelog = jinja2.Template(INITIAL_CHANGELOG_CONTENT, autoescape=True).render(
- package_name=provider_details.pypi_package_name,
- )
- syntax = Syntax(
- processed_changelog,
- "rst",
- theme="ansi_dark",
- )
- console.print(syntax)
- console.print("----- END COPYING BEFORE THIS LINE ------- ")
- sys.exit(1)
- return changelog_path
-
-
-@cli.command()
-def list_providers_packages():
- """List all provider packages."""
- providers = get_all_providers()
- # if provider needs to be not considered in release add it here
- # this is useful for cases where provider is WIP for a long period thus we don't want to release it yet.
- providers_to_remove_from_release = []
- for provider in providers:
- if provider not in providers_to_remove_from_release:
- console.print(provider)
-
-
-def tag_exists_for_version(provider_package_id: str, current_tag: str, verbose: bool):
- provider_details = get_provider_details(provider_package_id)
- if verbose:
- console.print(f"Checking if tag `{current_tag}` exists.")
- if not subprocess.call(
- get_git_tag_check_command(current_tag),
- cwd=provider_details.source_provider_package_path,
- stderr=subprocess.DEVNULL,
- stdout=subprocess.DEVNULL,
- ):
- if verbose:
- console.print(f"Tag `{current_tag}` exists.")
- return True
- if verbose:
- console.print(f"Tag `{current_tag}` does not exist.")
- return False
-
-
-@cli.command()
-@option_version_suffix
-@option_git_update
-@argument_package_id
-@option_verbose
-@option_skip_tag_check
-def generate_setup_files(
- version_suffix: str, git_update: bool, package_id: str, verbose: bool, skip_tag_check: bool
-):
- """Generates setup files for the package.
-
- See `list-providers-packages` subcommand for the possible PACKAGE_ID values.
- """
- provider_package_id = package_id
- with with_group(f"Generate setup files for '{provider_package_id}'"):
- if not skip_tag_check:
- current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
- if tag_exists_for_version(provider_package_id, current_tag, verbose):
- console.print(f"[yellow]The tag {current_tag} exists. Not preparing the package.[/]")
- sys.exit(64)
- if update_setup_files(provider_package_id, version_suffix):
- console.print(f"[green]Generated regular package setup files for {provider_package_id}[/]")
- else:
- sys.exit(64)
-
-
-def get_current_tag(provider_package_id: str, suffix: str, git_update: bool, verbose: bool):
- verify_provider_package(provider_package_id)
- provider_info = get_provider_info_from_provider_yaml(provider_package_id)
- versions: list[str] = provider_info["versions"]
- current_version = versions[0]
- current_tag = get_version_tag(current_version, provider_package_id, suffix)
- return current_tag
-
-
-def cleanup_remnants(verbose: bool):
- if verbose:
- console.print("Cleaning remnants")
- files = glob.glob("*.egg-info")
- for file in files:
- shutil.rmtree(file, ignore_errors=True)
- files = glob.glob("build")
- for file in files:
- shutil.rmtree(file, ignore_errors=True)
-
-
-def verify_setup_cfg_prepared(provider_package):
- with open("setup.cfg") as f:
- setup_content = f.read()
- search_for = f"providers-{provider_package.replace('.','-')} for Apache Airflow"
- if search_for not in setup_content:
- console.print(
- f"[red]The setup.py is probably prepared for another package. "
- f"It does not contain [bold]{search_for}[/bold]![/]"
- )
- console.print(
- f"\nRun:\n\n[bold]./dev/provider_packages/prepare_provider_packages.py "
- f"generate-setup-files {provider_package}[/bold]\n"
- )
- raise Exception("Wrong setup!")
-
-
-@cli.command()
-@option_package_format
-@option_git_update
-@option_version_suffix
-@argument_package_id
-@option_verbose
-@option_skip_tag_check
-def build_provider_packages(
- package_format: str,
- git_update: bool,
- version_suffix: str,
- package_id: str,
- verbose: bool,
- skip_tag_check: bool,
-):
- """Builds provider package.
-
- See `list-providers-packages` subcommand for the possible PACKAGE_ID values.
- """
-
- import tempfile
-
- # we cannot use context managers because if the directory gets deleted (which bdist_wheel does),
- # the context manager will throw an exception when trying to delete it again
- tmp_build_dir = tempfile.TemporaryDirectory().name
- tmp_dist_dir = tempfile.TemporaryDirectory().name
- try:
- provider_package_id = package_id
- with with_group(f"Prepare provider package for '{provider_package_id}'"):
- if not skip_tag_check and (version_suffix.startswith("rc") or version_suffix == ""):
- # For RC and official releases we check if the "officially released" version exists
- # and skip the released if it was. This allows to skip packages that have not been
- # marked for release. For "dev" suffixes, we always build all packages
- released_tag = get_current_tag(provider_package_id, "", git_update, verbose)
- if tag_exists_for_version(provider_package_id, released_tag, verbose):
- console.print(f"[yellow]The tag {released_tag} exists. Skipping the package.[/]")
- return False
- console.print(f"Changing directory to {TARGET_PROVIDER_PACKAGES_PATH}")
- os.chdir(TARGET_PROVIDER_PACKAGES_PATH)
- cleanup_remnants(verbose)
- provider_package = package_id
- verify_setup_cfg_prepared(provider_package)
-
- console.print(f"Building provider package: {provider_package} in format {package_format}")
- command: list[str] = ["python3", "setup.py", "build", "--build-temp", tmp_build_dir]
- if version_suffix is not None:
- command.extend(["egg_info", "--tag-build", version_suffix])
- if package_format in ["sdist", "both"]:
- command.append("sdist")
- if package_format in ["wheel", "both"]:
- command.extend(["bdist_wheel", "--bdist-dir", tmp_dist_dir])
- console.print(f"Executing command: '{' '.join(command)}'")
- try:
- subprocess.check_call(args=command, stdout=subprocess.DEVNULL)
- except subprocess.CalledProcessError as ex:
- console.print("[red]The command returned an error %s", ex)
- sys.exit(ex.returncode)
- console.print(
- f"[green]Prepared provider package {provider_package} in format {package_format}[/]"
- )
- finally:
- shutil.rmtree(tmp_build_dir, ignore_errors=True)
- shutil.rmtree(tmp_dist_dir, ignore_errors=True)
-
-
-if __name__ == "__main__":
- # The cli exit code is:
- # * 0 in case of success
- # * 1 in case of error
- # * 64 in case of skipped package
- # * 65 in case user decided to quit
- # * 66 in case package has doc-only changes
- try:
- cli()
- except KeyboardInterrupt:
- print("Interrupted")
- try:
- sys.exit(65)
- except SystemExit:
- os._exit(65)
diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg
index f465122c13..9bdb76be04 100644
--- a/images/breeze/output_release-management_prepare-provider-packages.svg
+++ b/images/breeze/output_release-management_prepare-provider-packages.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 708.8" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 830.8" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -43,7 +43,7 @@
<defs>
<clipPath id="breeze-release-management-prepare-provider-packages-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="657.8" />
+ <rect x="0" y="0" width="1463.0" height="779.8" />
</clipPath>
<clipPath id="breeze-release-management-prepare-provider-packages-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -123,9 +123,24 @@
<clipPath id="breeze-release-management-prepare-provider-packages-line-25">
<rect x="0" y="611.5" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-26">
+ <rect x="0" y="635.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-27">
+ <rect x="0" y="660.3" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-28">
+ <rect x="0" y="684.7" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-29">
+ <rect x="0" y="709.1" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="breeze-release-management-prepare-provider-packages-line-30">
+ <rect x="0" y="733.5" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="706.8" rx="8"/><text class="breeze-release-management-prepare-provider-packages-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management prepare-provider-packages</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="828.8" rx="8"/><text class="breeze-release-management-prepare-provider-packages-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: release-management prepare-provider-packages</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -151,17 +166,22 @@
</text><text class="breeze-release-management-prepare-provider-packages-r1" x="12.2" y="337.2" textLength="585.6" clip-path="url(#breeze-release-management-prepare-provider-packages-line-13)">Prepare sdist/whl packages of Airflow Providers.</text><text class="breeze-release-management-prepare-provider-packages-r1" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-13)">
</text><text class="breeze-release-management-prepare-provider-packages-r1" x="1464" y="361.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-14)">
</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="386" textLength="24.4" clip-path="url(#breeze-release-management-prepare-provider-packages-line-15)">╭─</text><text class="breeze-release-management-prepare-provider-packages-r5" x="24.4" y="386" textLength="183" clip-path="url(#breeze-release-management-prepare-provider-packages-line-15)"> Package flags </text><text class="breeze-release-management-prepare-provider-packages-r5" x="207.4" [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-16)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-16)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="410.4" textLength="97.6 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-17)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-17)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="434.8" textLength="97.6 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-18)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-18)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="459.2" textLength="97.6 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-19)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-19)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="483.6" textLength="73.2 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-20)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-20)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="508" textLength="85.4" clip [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="532.4" textLength="1464" clip-path="url(#breeze-release-management-prepare-provider-packages-line-21)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-provider-packages-r1" x="1464" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-21)">
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="556.8" textLength="24.4" clip-path="url(#breeze-release-management-prepare-provider-packages-line-22)">╭─</text><text class="breeze-release-management-prepare-provider-packages-r5" x="24.4" y="556.8" textLength="195.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-22)"> Common options </text><text class="breeze-release-management-prepare-provider-packages-r5" x=" [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-23)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-23)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="581.2" textLength="97.6 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-24)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-24)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="605.6" textLength="48.8 [...]
-</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="630" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="630" textLength="61" clip-p [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-16)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="410.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-16)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="410.4" textLength="97.6 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-17)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="434.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-17)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="434.8" textLength="97.6 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-18)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="459.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-18)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="459.2" textLength="73.2 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="483.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-19)">│</text><text class="breeze-release-management-prepare-provider-packages-r1" x="475.8" y="483.6" textLength="963.8" clip-path="url(#breeze-release-management-prepare-provider-packages-line-19)">multiple packages  in a clean environment       [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="508" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-20)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="508" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-20)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="508" textLength="61" clip-p [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-21)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="532.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-21)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="532.4" textLength="61" [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="556.8" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-22)">│</text><text class="breeze-release-management-prepare-provider-packages-r1" x="475.8" y="556.8" textLength="963.8" clip-path="url(#breeze-release-management-prepare-provider-packages-line-22)">debugging and developing changes to the build process.    [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-23)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="581.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-23)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="581.2" textLength="97.6 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-24)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="605.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-24)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="605.6" textLength="85.4 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="630" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">│</text><text class="breeze-release-management-prepare-provider-packages-r5" x="475.8" y="630" textLength="585.6" clip-path="url(#breeze-release-management-prepare-provider-packages-line-25)">[default: apache/airflow]               [...]
</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="654.4" textLength="1464" clip-path="url(#breeze-release-management-prepare-provider-packages-line-26)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-provider-packages-r1" x="1464" y="654.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-26)">
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="678.8" textLength="24.4" clip-path="url(#breeze-release-management-prepare-provider-packages-line-27)">╭─</text><text class="breeze-release-management-prepare-provider-packages-r5" x="24.4" y="678.8" textLength="195.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-27)"> Common options </text><text class="breeze-release-management-prepare-provider-packages-r5" x=" [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="703.2" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-28)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="703.2" textLength="97.6 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="727.6" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-29)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="727.6" textLength="48.8 [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="752" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">│</text><text class="breeze-release-management-prepare-provider-packages-r4" x="24.4" y="752" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-30)">-</text><text class="breeze-release-management-prepare-provider-packages-r4" x="36.6" y="752" textLength="61" clip-p [...]
+</text><text class="breeze-release-management-prepare-provider-packages-r5" x="0" y="776.4" textLength="1464" clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-release-management-prepare-provider-packages-r1" x="1464" y="776.4" textLength="12.2" clip-path="url(#breeze-release-management-prepare-provider-packages-line-31)">
</text>
</g>
</g>
diff --git a/images/breeze/output_release-management_prepare-provider-packages.txt b/images/breeze/output_release-management_prepare-provider-packages.txt
index 07c12cf7b5..872a3881ba 100644
--- a/images/breeze/output_release-management_prepare-provider-packages.txt
+++ b/images/breeze/output_release-management_prepare-provider-packages.txt
@@ -1 +1 @@
-b2d097fc51b60fd76a7c81d7ec46c811
+1b7d25cb5f379cee89c5345c60be1992
diff --git a/provider_packages/.gitignore b/provider_packages/.gitignore
deleted file mode 100644
index 07ae78b669..0000000000
--- a/provider_packages/.gitignore
+++ /dev/null
@@ -1,7 +0,0 @@
-*.egg-info
-setup.py
-CHANGELOG.txt
-README.md
-README.rst
-setup.cfg
-/airflow
diff --git a/provider_packages/INSTALL b/provider_packages/INSTALL
deleted file mode 100644
index bb516dba0a..0000000000
--- a/provider_packages/INSTALL
+++ /dev/null
@@ -1,15 +0,0 @@
-## INSTALL / BUILD instructions for Apache Airflow Provider packages
-
-NOTE! Those sources are only intended to be used to build Apache Airflow Provider packages,
-not the Apache Airflow. They have been generated using the unreleased version of Apache Airflow.
-(from main)
-
-The .tar.gz sdist package contains setup.py file that can be use to build the provider package using:
-
-`setup.py bdist_wheel sdist`
-
-Providing that you have all prerequisites described in 'setup_requires'.
-
-# [optional] run Apache RAT (release audit tool) to validate license headers
-# RAT docs here: https://creadur.apache.org/rat/. Requires Java and Apache Rat
-java -jar apache-rat.jar -d .
diff --git a/provider_packages/LICENSE b/provider_packages/LICENSE
deleted file mode 100644
index 11069edd79..0000000000
--- a/provider_packages/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
-2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
-3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
-4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
-5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
-6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
-8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
-Copyright [yyyy] [name of copyright owner]
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
diff --git a/provider_packages/NOTICE b/provider_packages/NOTICE
deleted file mode 100644
index 0ae7e64bab..0000000000
--- a/provider_packages/NOTICE
+++ /dev/null
@@ -1,6 +0,0 @@
-Apache Airflow
-Copyright 2016-2021 The Apache Software Foundation
-
-This product includes software developed at The Apache Software
-Foundation (http://www.apache.org/).
-=======================================================================
diff --git a/provider_packages/dist b/provider_packages/dist
deleted file mode 120000
index 56d4b041ce..0000000000
--- a/provider_packages/dist
+++ /dev/null
@@ -1 +0,0 @@
-../dist/
\ No newline at end of file
diff --git a/provider_packages/pyproject.toml b/provider_packages/pyproject.toml
deleted file mode 120000
index 1e11d78257..0000000000
--- a/provider_packages/pyproject.toml
+++ /dev/null
@@ -1 +0,0 @@
-../pyproject.toml
\ No newline at end of file
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index ed3f1a8b58..6be73d2936 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -16,9 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-OPTIONAL_VERBOSE_FLAG=()
-PROVIDER_PACKAGES_DIR="${AIRFLOW_SOURCES}/dev/provider_packages"
-
#######################################################################################################
#
# Adds trap to the traps already set.
@@ -373,24 +370,6 @@ function twine_check_provider_packages_from_sdist() {
twine check /dist/apache-airflow-providers-*.tar.gz
}
-function setup_provider_packages() {
- export PACKAGE_TYPE="regular"
- export PACKAGE_PREFIX_UPPERCASE=""
- export PACKAGE_PREFIX_LOWERCASE=""
- export PACKAGE_PREFIX_HYPHEN=""
- if [[ ${VERBOSE:="false"} == "true" || ${VERBOSE} == "True" ]]; then
- OPTIONAL_VERBOSE_FLAG+=("--verbose")
- fi
- if [[ ${ANSWER:=""} != "" ]]; then
- OPTIONAL_ANSWER_FLAG+=("--answer" "${ANSWER}")
- fi
- readonly PACKAGE_TYPE
- readonly PACKAGE_PREFIX_UPPERCASE
- readonly PACKAGE_PREFIX_LOWERCASE
- readonly PACKAGE_PREFIX_HYPHEN
-}
-
-
function install_supported_pip_version() {
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
@@ -400,16 +379,6 @@ function install_supported_pip_version() {
}
-function filename_to_python_module() {
- # Turn the file name into a python package name
- file="$1"
- no_leading_dotslash="${file#./}"
- no_py="${no_leading_dotslash/.py/}"
- no_init="${no_py/\/__init__/}"
- echo "${no_init//\//.}"
-}
-
-
function in_container_set_colors() {
COLOR_BLUE=$'\e[34m'
COLOR_GREEN=$'\e[32m'
@@ -424,71 +393,6 @@ function in_container_set_colors() {
}
-function check_missing_providers() {
- PACKAGE_ERROR="false"
-
- pushd "${AIRFLOW_SOURCES}/airflow/providers" >/dev/null 2>&1 || exit 1
-
- LIST_OF_DIRS_FILE=$(mktemp)
- find . -type d | sed 's!./!!; s!/!.!g' | grep -E 'hooks|operators|sensors|secrets|utils' \
- > "${LIST_OF_DIRS_FILE}"
-
- popd >/dev/null 2>&1 || exit 1
-
- # Check if all providers are included
- for PACKAGE in "${PROVIDER_PACKAGES[@]}"
- do
- if ! grep -E "^${PACKAGE}" <"${LIST_OF_DIRS_FILE}" >/dev/null; then
- echo "The package ${PACKAGE} is not available in providers dir"
- PACKAGE_ERROR="true"
- fi
- sed -i "/^${PACKAGE}.*/d" "${LIST_OF_DIRS_FILE}"
- done
-
- if [[ ${PACKAGE_ERROR} == "true" ]]; then
- echo
- echo "ERROR! Some packages from ${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py are missing in providers dir"
- exit 1
- fi
-
- if [[ $(wc -l < "${LIST_OF_DIRS_FILE}") != "0" ]]; then
- echo "ERROR! Some folders from providers package are not defined"
- echo " Please add them to ${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py:"
- echo
- cat "${LIST_OF_DIRS_FILE}"
- echo
-
- rm "$LIST_OF_DIRS_FILE"
- exit 1
- fi
- rm "$LIST_OF_DIRS_FILE"
-}
-
-function get_providers_to_act_on() {
- group_start "Get all providers"
- if [[ -z "$*" ]]; then
- while IFS='' read -r line; do PROVIDER_PACKAGES+=("$line"); done < <(
- python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \
- list-providers-packages
- )
- else
- if [[ "${1}" == "--help" ]]; then
- echo
- echo "Builds all provider packages."
- echo
- echo "You can provide list of packages to build out of:"
- echo
- python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \
- list-providers-packages \
- | tr '\n ' ' ' | fold -w 100 -s
- echo
- echo
- exit
- fi
- fi
- group_end
-}
-
# Starts group for GitHub Actions - makes logs much more readable
function group_start {
if [[ ${GITHUB_ACTIONS:="false"} == "true" || ${GITHUB_ACTIONS} == "True" ]]; then
diff --git a/scripts/in_container/run_prepare_provider_packages.sh b/scripts/in_container/run_prepare_provider_packages.sh
deleted file mode 100755
index 9da0472957..0000000000
--- a/scripts/in_container/run_prepare_provider_packages.sh
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-function copy_sources() {
- group_start "Copy sources"
- echo "==================================================================================="
- echo " Copying sources for provider packages"
- echo "==================================================================================="
- mkdir -pv "${AIRFLOW_SOURCES}/provider_packages/airflow/"
- rsync -avz --exclude '*node_modules*' --delete \
- "${AIRFLOW_SOURCES}/airflow" \
- "${AIRFLOW_SOURCES}/provider_packages/"
- group_end
-}
-
-
-function build_provider_packages() {
- rm -rf dist/*
- local package_format_args=()
- if [[ ${PACKAGE_FORMAT=} != "" ]]; then
- package_format_args=("--package-format" "${PACKAGE_FORMAT}")
- fi
-
- local prepared_packages=()
- local skipped_packages=()
- local error_packages=()
-
- echo "-----------------------------------------------------------------------------------"
- if [[ "${VERSION_SUFFIX_FOR_PYPI}" == '' ]]; then
- echo
- echo "Preparing official version of provider with no suffixes"
- echo
- else
- echo
- echo " Package Version of providers suffix set for PyPI version: ${VERSION_SUFFIX_FOR_PYPI}"
- echo
- fi
- echo "-----------------------------------------------------------------------------------"
-
- # Delete the remote, so that we fetch it and update it once, not once per package we build!
- git remote rm apache-https-for-providers 2>/dev/null || :
-
- local provider_package
- for provider_package in "${PROVIDER_PACKAGES[@]}"
- do
- rm -rf -- *.egg-info build/
- local res
- set +e
- python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \
- generate-setup-files \
- "${OPTIONAL_VERBOSE_FLAG[@]}" \
- --no-git-update \
- --version-suffix "${VERSION_SUFFIX_FOR_PYPI}" \
- "${provider_package}"
- res=$?
- set -e
- if [[ ${res} == "64" ]]; then
- skipped_packages+=("${provider_package}")
- continue
- fi
- if [[ ${res} != "0" ]]; then
- error_packages+=("${provider_package}")
- continue
- fi
- set +e
- package_suffix=""
- if [[ -n ${VERSION_SUFFIX_FOR_PYPI} ]]; then
- # only adds suffix to setup.py if version suffix for PyPI is set
- package_suffix="${VERSION_SUFFIX_FOR_PYPI}"
- fi
- python3 "${PROVIDER_PACKAGES_DIR}/prepare_provider_packages.py" \
- build-provider-packages \
- "${OPTIONAL_VERBOSE_FLAG[@]}" \
- --no-git-update \
- --version-suffix "${package_suffix}" \
- "${package_format_args[@]}" \
- "${provider_package}"
- res=$?
- set -e
- if [[ ${res} == "64" ]]; then
- skipped_packages+=("${provider_package}")
- continue
- fi
- if [[ ${res} != "0" ]]; then
- error_packages+=("${provider_package}")
- echo "${COLOR_RED}Error when preparing ${provider_package} package${COLOR_RESET}"
- continue
- fi
- prepared_packages+=("${provider_package}")
- done
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- echo
- echo "Summary of prepared packages:"
- echo
- if [[ "${#prepared_packages[@]}" != "0" ]]; then
- echo "${COLOR_GREEN} Prepared:${COLOR_RESET}"
- echo "${prepared_packages[*]}" | fold -w 100
- fi
- if [[ "${#skipped_packages[@]}" != "0" ]]; then
- echo "${COLOR_YELLOW} Skipped:${COLOR_RESET}"
- echo "${skipped_packages[*]}" | fold -w 100
- fi
- if [[ "${#error_packages[@]}" != "0" ]]; then
- echo "${COLOR_RED} Errors:${COLOR_RESET}"
- echo "${error_packages[*]}" | fold -w 100
- fi
- echo
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- ls -w 1 /dist
- echo "${COLOR_BLUE}===================================================================================${COLOR_RESET}"
- if [[ ${#error_packages[@]} != "0" ]]; then
- echo
- echo "${COLOR_RED}There were errors when preparing packages. Exiting! ${COLOR_RESET}"
- exit 1
- fi
-}
-
-setup_provider_packages
-
-cd "${PROVIDER_PACKAGES_DIR}" || exit 1
-
-install_supported_pip_version
-
-PROVIDER_PACKAGES=("${@}")
-get_providers_to_act_on "${@}"
-
-copy_sources
-build_provider_packages
-
-echo
-echo "${COLOR_GREEN}All good! Airflow packages are prepared in dist folder${COLOR_RESET}"
-echo