You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ra...@apache.org on 2026/08/05 08:50:33 UTC
(airflow) branch constraints-build-not-ready-providers-from-source created (now 54494526956)
This is an automated email from the ASF dual-hosted git repository.
vatsrahul1001 pushed a change to branch constraints-build-not-ready-providers-from-source
in repository https://gitbox.apache.org/repos/asf/airflow.git
at 54494526956 Build not-ready providers from source when generating PyPI constraints
This branch includes the following new commits:
new 54494526956 Build not-ready providers from source when generating PyPI constraints
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 01/01: Build not-ready providers from source when generating PyPI constraints
Posted by ra...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatsrahul1001 pushed a commit to branch constraints-build-not-ready-providers-from-source
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 544945269566244979d17bc060939bf7db3c08c0
Author: Rahul Vats <ra...@gmail.com>
AuthorDate: Wed Aug 5 14:19:05 2026 +0530
Build not-ready providers from source when generating PyPI constraints
The release constraints resolution installs apache-airflow[all], which depends
on every non-suspended provider. not-ready providers (currently ibm.mq and
common.dataquality) are never published to PyPI, so the PyPI-only release path
cannot satisfy them and fails with "No solution found". Install them from local
source so the resolution stays solvable; pip freeze renders a path install as a
file:// line that is already dropped from the constraints file, so a not-ready
provider is never pinned in released constraints.
---
scripts/in_container/run_generate_constraints.py | 27 +++++++++++++++++
.../in_container/test_run_generate_constraints.py | 34 ++++++++++++++++++++++
2 files changed, 61 insertions(+)
diff --git a/scripts/in_container/run_generate_constraints.py b/scripts/in_container/run_generate_constraints.py
index ba23a0042b0..2247c22cbb7 100755
--- a/scripts/in_container/run_generate_constraints.py
+++ b/scripts/in_container/run_generate_constraints.py
@@ -459,6 +459,30 @@ def get_all_active_provider_distributions(python_version: str | None = None) ->
]
+def get_not_ready_provider_source_paths(python_version: str | None = None) -> list[str]:
+ """
+ Get local source paths of the ``not-ready`` providers that ``apache-airflow[all]`` requires.
+
+ ``[all]`` depends on every non-suspended provider, but ``not-ready`` providers are never
+ published to PyPI, so a PyPI-only resolution cannot satisfy them and fails with "No solution
+ found". The release path (``pypi-providers-only``) resolves from PyPI and skips building
+ providers into ``dist`` first, so these have nowhere to resolve from; installing them from local
+ source keeps the resolution solvable. ``pip freeze`` renders a path install as a
+ ``@ file://...`` line, which ``freeze_distributions_to_file`` already drops, so a not-ready
+ provider never lands in the released constraints file.
+ """
+ all_provider_dependencies = json.loads(GENERATED_PROVIDER_DEPENDENCIES_FILE.read_text())
+ return [
+ f"./providers/{provider.replace('.', '/')}"
+ for provider in all_provider_dependencies.keys()
+ if all_provider_dependencies[provider]["state"] == "not-ready"
+ and (
+ python_version is None
+ or python_version not in all_provider_dependencies[provider]["excluded-python-versions"]
+ )
+ ]
+
+
def build_provider_pre_release_requirements(python_version: str) -> list[str]:
"""Requirements that let only the providers resolve to a pre-release.
@@ -577,6 +601,9 @@ def generate_constraints_pypi_providers(config_params: ConfigParams) -> None:
f"apache-airflow-core[all]=={AIRFLOW_CORE_VERSION}",
f"apache-airflow-task-sdk=={AIRFLOW_TASK_SDK_VERSION}",
"./airflow-ctl",
+ # not-ready providers are required by [all] but never published to PyPI - build them
+ # from local source so the resolution is solvable (they are dropped from the file below).
+ *get_not_ready_provider_source_paths(config_params.python),
*additional_constraints_for_highest_resolution,
*pre_release_requirements,
*pre_release_strategy,
diff --git a/scripts/tests/in_container/test_run_generate_constraints.py b/scripts/tests/in_container/test_run_generate_constraints.py
index 88af9e84102..b6a2060ec5e 100644
--- a/scripts/tests/in_container/test_run_generate_constraints.py
+++ b/scripts/tests/in_container/test_run_generate_constraints.py
@@ -148,3 +148,37 @@ class TestBuildProviderPreReleaseRequirements:
m.build_provider_pre_release_requirements("3.14")
assert seen == ["3.14"]
+
+
+class TestGetNotReadyProviderSourcePaths:
+ """Only ``not-ready`` providers are built from local source; they are never on PyPI."""
+
+ @pytest.fixture
+ def dependencies_file(self, tmp_path, monkeypatch):
+ path = tmp_path / "provider_dependencies.json"
+ path.write_text(
+ json.dumps(
+ {
+ "amazon": {"state": "ready", "excluded-python-versions": []},
+ "ibm.mq": {"state": "not-ready", "excluded-python-versions": []},
+ "common.dataquality": {"state": "not-ready", "excluded-python-versions": []},
+ "apache.beam": {"state": "suspended", "excluded-python-versions": []},
+ "legacy.thing": {"state": "not-ready", "excluded-python-versions": ["3.10"]},
+ }
+ )
+ )
+ monkeypatch.setattr(m, "GENERATED_PROVIDER_DEPENDENCIES_FILE", path)
+ return path
+
+ def test_returns_only_not_ready_providers_as_source_paths(self, dependencies_file):
+ assert m.get_not_ready_provider_source_paths("3.12") == [
+ "./providers/ibm/mq",
+ "./providers/common/dataquality",
+ "./providers/legacy/thing",
+ ]
+
+ def test_excludes_provider_not_supported_on_the_python_version(self, dependencies_file):
+ assert m.get_not_ready_provider_source_paths("3.10") == [
+ "./providers/ibm/mq",
+ "./providers/common/dataquality",
+ ]