You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by el...@apache.org on 2023/07/04 06:02:39 UTC
[airflow] branch revert-32325-exclude-cross-providers-for-non-releasable-providers created (now 20565e7bc6)
This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a change to branch revert-32325-exclude-cross-providers-for-non-releasable-providers
in repository https://gitbox.apache.org/repos/asf/airflow.git
at 20565e7bc6 Revert "Exckude not-releaseable providers from extras not only from releasing (#32325)"
This branch includes the following new commits:
new 20565e7bc6 Revert "Exckude not-releaseable providers from extras not only from releasing (#32325)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airflow] 01/01: Revert "Exckude not-releaseable providers from extras not only from releasing (#32325)"
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch revert-32325-exclude-cross-providers-for-non-releasable-providers
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 20565e7bc66707bfa9089835927ec44f03fbee77
Author: eladkal <45...@users.noreply.github.com>
AuthorDate: Tue Jul 4 09:02:34 2023 +0300
Revert "Exckude not-releaseable providers from extras not only from releasing (#32325)"
This reverts commit bc212f80e89e3d460e6320db0983126cea9300e0.
---
dev/provider_packages/prepare_provider_packages.py | 14 +++++---------
generated/provider_dependencies.json | 4 +++-
.../pre_commit/pre_commit_update_providers_dependencies.py | 9 +--------
setup.py | 4 ----
4 files changed, 9 insertions(+), 22 deletions(-)
diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
index 7cf37d382b..8bf6e24852 100755
--- a/dev/provider_packages/prepare_provider_packages.py
+++ b/dev/provider_packages/prepare_provider_packages.py
@@ -123,11 +123,7 @@ ALL_DEPENDENCIES = json.loads(DEPENDENCIES_JSON_FILE_PATH.read_text())
# those imports need to come after the above sys.path.insert to make sure that Airflow
# sources are importable without having to add the airflow sources to the PYTHONPATH before
# running the script
-from setup import ( # type: ignore[attr-defined] # isort:skip # noqa
- ALL_PROVIDERS,
- NOT_RELEASED_YET_PROVIDERS,
- PREINSTALLED_PROVIDERS,
-)
+from setup import PREINSTALLED_PROVIDERS, ALL_PROVIDERS # type: ignore[attr-defined] # isort:skip # noqa
# Note - we do not test protocols as they are not really part of the official API of
# Apache Airflow
@@ -1679,10 +1675,10 @@ def verify_changelog_exists(package: str) -> str:
def list_providers_packages():
"""List all provider packages."""
providers = get_all_providers()
- # exclude providers we do not release yet
- for provider in NOT_RELEASED_YET_PROVIDERS:
- if provider in providers:
- providers.remove(provider)
+ # For now we should exclude open-lineage from being consider for releasing until it is ready to
+ # be released
+ if "openlineage" in providers:
+ providers.remove("openlineage")
for provider in providers:
console.print(provider)
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index aaa5033b9a..d80c3d084e 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -272,7 +272,9 @@
"apache-airflow>=2.4.0",
"sqlparse>=0.4.2"
],
- "cross-providers-deps": [],
+ "cross-providers-deps": [
+ "openlineage"
+ ],
"excluded-python-versions": []
},
"databricks": {
diff --git a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
index b5a3664991..0c489bad63 100755
--- a/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
+++ b/scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
@@ -52,7 +52,6 @@ ALL_DEPENDENCIES: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultd
ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict())
ALL_PROVIDER_FILES: list[Path] = []
-
# Allow AST to parse the files.
sys.path.append(str(AIRFLOW_SOURCES_ROOT))
@@ -169,15 +168,9 @@ def check_if_different_provider_used(file_path: Path) -> None:
imports = get_imports_from_file(file_path)
for import_name in imports:
imported_provider = get_provider_id_from_import(import_name, file_path)
- from setup import NOT_RELEASED_YET_PROVIDERS
-
if imported_provider is not None and imported_provider not in ALL_PROVIDERS:
warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.")
- elif (
- imported_provider
- and file_provider != imported_provider
- and imported_provider not in NOT_RELEASED_YET_PROVIDERS
- ):
+ elif imported_provider and file_provider != imported_provider:
ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider)
diff --git a/setup.py b/setup.py
index 32d5c54447..547e1221d3 100644
--- a/setup.py
+++ b/setup.py
@@ -60,10 +60,6 @@ CROSS_PROVIDERS_DEPS = "cross-providers-deps"
DEPS = "deps"
CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
-# we do not yet release some providers so we should skip them when releasing and when referring to
-# in cross-provider-dependencies.
-NOT_RELEASED_YET_PROVIDERS = ["openlineage"]
-
#
# NOTE! IN Airflow 2.4.+ dependencies for providers are maintained in `provider.yaml` files for each