You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2025/04/24 12:33:47 UTC
(airflow) branch v3-0-test updated: Check for task concurrency parameters inside encoded object & partial_kwargs to handle dynamic tasks. (#49708)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new b4519ad5f69 Check for task concurrency parameters inside encoded object & partial_kwargs to handle dynamic tasks. (#49708)
b4519ad5f69 is described below
commit b4519ad5f69097b69fdf79b0702dd90c4d133b0d
Author: Karthikeyan Singaravelan <ti...@gmail.com>
AuthorDate: Thu Apr 24 18:02:57 2025 +0530
Check for task concurrency parameters inside encoded object & partial_kwargs to handle dynamic tasks. (#49708)
Closes #49696
(cherry picked from commit c2bd43a2c37f1d9578fa861e5fb2466659b9169e)
---
.../airflow/serialization/serialized_objects.py | 6 +++-
.../unit/serialization/test_serialized_objects.py | 39 ++++++++++++++++++++--
2 files changed, 42 insertions(+), 3 deletions(-)
diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py b/airflow-core/src/airflow/serialization/serialized_objects.py
index 8428765e2be..948111b2c17 100644
--- a/airflow-core/src/airflow/serialization/serialized_objects.py
+++ b/airflow-core/src/airflow/serialization/serialized_objects.py
@@ -2098,7 +2098,11 @@ class LazyDeserializedDAG(pydantic.BaseModel):
@property
def has_task_concurrency_limits(self) -> bool:
return any(
- task[Encoding.VAR].get("max_active_tis_per_dag") is not None for task in self.data["dag"]["tasks"]
+ task[Encoding.VAR].get("max_active_tis_per_dag") is not None
+ or task[Encoding.VAR].get("max_active_tis_per_dagrun") is not None
+ or task[Encoding.VAR].get("partial_kwargs", {}).get("max_active_tis_per_dag") is not None
+ or task[Encoding.VAR].get("partial_kwargs", {}).get("max_active_tis_per_dagrun") is not None
+ for task in self.data["dag"]["tasks"]
)
@property
diff --git a/airflow-core/tests/unit/serialization/test_serialized_objects.py b/airflow-core/tests/unit/serialization/test_serialized_objects.py
index 3f666249428..6f287289673 100644
--- a/airflow-core/tests/unit/serialization/test_serialized_objects.py
+++ b/airflow-core/tests/unit/serialization/test_serialized_objects.py
@@ -44,6 +44,7 @@ from airflow.providers.standard.operators.empty import EmptyOperator
from airflow.providers.standard.operators.python import PythonOperator
from airflow.providers.standard.triggers.file import FileDeleteTrigger
from airflow.sdk.definitions.asset import Asset, AssetAlias, AssetAliasEvent, AssetUniqueKey, AssetWatcher
+from airflow.sdk.definitions.decorators import task
from airflow.sdk.definitions.param import Param
from airflow.sdk.execution_time.context import OutletEventAccessor, OutletEventAccessors
from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding
@@ -448,10 +449,44 @@ def test_serialized_dag_to_dict_and_from_dict_gives_same_result_in_tasks(dag_mak
assert dag2["dag"]["tasks"][0]["__var"].keys() == dag1["dag"]["tasks"][0]["__var"].keys()
+@pytest.mark.parametrize(
+ "concurrency_parameter",
+ [
+ "max_active_tis_per_dag",
+ "max_active_tis_per_dagrun",
+ ],
+)
+@pytest.mark.db_test
+def test_serialized_dag_has_task_concurrency_limits(dag_maker, concurrency_parameter):
+ with dag_maker() as dag:
+ BashOperator(task_id="task1", bash_command="echo 1", **{concurrency_parameter: 1})
+
+ ser_dict = SerializedDAG.to_dict(dag)
+ lazy_serialized_dag = LazyDeserializedDAG(data=ser_dict)
+
+ assert lazy_serialized_dag.has_task_concurrency_limits
+
+
+@pytest.mark.parametrize(
+ "concurrency_parameter",
+ [
+ "max_active_tis_per_dag",
+ "max_active_tis_per_dagrun",
+ ],
+)
@pytest.mark.db_test
-def test_serialized_dag_has_task_concurrency_limits(dag_maker):
+def test_serialized_dag_mapped_task_has_task_concurrency_limits(dag_maker, concurrency_parameter):
with dag_maker() as dag:
- BashOperator(task_id="task1", bash_command="echo 1", max_active_tis_per_dag=1)
+
+ @task
+ def my_task():
+ return [1, 2, 3, 4, 5, 6, 7]
+
+ @task(**{concurrency_parameter: 1})
+ def map_me_but_slowly(a):
+ pass
+
+ map_me_but_slowly.expand(a=my_task())
ser_dict = SerializedDAG.to_dict(dag)
lazy_serialized_dag = LazyDeserializedDAG(data=ser_dict)