You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/04/27 19:53:35 UTC
[airflow] 02/05: Revert "EdgeModifier refactoring (#21404)" (#23291)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a83b6d4630fec00e2e48f00f20d67ddcfe0b9f88
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Wed Apr 27 19:12:08 2022 +0100
Revert "EdgeModifier refactoring (#21404)" (#23291)
This reverts commit ace8c6e942ff5554639801468b971915b7c0e9b9.
Fixes apache/airflow#23285
This dag breaks with a cycle as group.end has itself as a downstream otherwise:
```python
from pendulum import datetime
from airflow.decorators import dag, task, task_group
from airflow.utils.edgemodifier import Label
@task
def begin():
...
@task
def end():
...
@dag(start_date=datetime(2022, 1, 1), schedule_interval=None)
def task_groups_with_edge_labels():
@task_group
def group():
begin() >> end()
group()
_ = task_groups_with_edge_labels()
```
(cherry picked from commit 3182303ce50bda6d5d27a6ef4e19450fb4e47eea)
---
airflow/models/taskmixin.py | 21 +---
airflow/utils/edgemodifier.py | 107 ++++++++++----------
airflow/utils/task_group.py | 17 +---
tests/utils/test_edgemodifier.py | 205 ---------------------------------------
4 files changed, 60 insertions(+), 290 deletions(-)
diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py
index 1a7129f66e..1d66a719a6 100644
--- a/airflow/models/taskmixin.py
+++ b/airflow/models/taskmixin.py
@@ -55,29 +55,16 @@ class DependencyMixin:
raise NotImplementedError()
@abstractmethod
- def set_upstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
- ):
+ def set_upstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
"""Set a task or a task list to be directly upstream from the current task."""
raise NotImplementedError()
@abstractmethod
- def set_downstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
- ):
+ def set_downstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
"""Set a task or a task list to be directly downstream from the current task."""
raise NotImplementedError()
- def update_relative(
- self,
- other: "DependencyMixin",
- upstream=True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ def update_relative(self, other: "DependencyMixin", upstream=True) -> None:
"""
Update relationship information about another TaskMixin. Default is no-op.
Override if necessary.
@@ -184,7 +171,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
task_list: List[Operator] = []
for task_object in task_or_task_list:
- task_object.update_relative(self, not upstream, edge_modifier=edge_modifier)
+ task_object.update_relative(self, not upstream)
relatives = task_object.leaves if upstream else task_object.roots
for task in relatives:
if not isinstance(task, (BaseOperator, MappedOperator)):
diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py
index 109199fc7c..a59616c76b 100644
--- a/airflow/utils/edgemodifier.py
+++ b/airflow/utils/edgemodifier.py
@@ -14,9 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, Optional, Sequence, Union
+from typing import TYPE_CHECKING, List, Optional, Sequence, Union
-from airflow.models.taskmixin import DAGNode, DependencyMixin
+from airflow.models.taskmixin import DependencyMixin
+
+if TYPE_CHECKING:
+ from airflow.models.baseoperator import BaseOperator
class EdgeModifier(DependencyMixin):
@@ -39,8 +42,8 @@ class EdgeModifier(DependencyMixin):
def __init__(self, label: Optional[str] = None):
self.label = label
- self._upstream: List["DependencyMixin"] = []
- self._downstream: List["DependencyMixin"] = []
+ self._upstream: List["BaseOperator"] = []
+ self._downstream: List["BaseOperator"] = []
@property
def roots(self):
@@ -50,38 +53,8 @@ class EdgeModifier(DependencyMixin):
def leaves(self):
return self._upstream
- @staticmethod
- def _make_list(item_or_list):
- if not isinstance(item_or_list, Sequence):
- return [item_or_list]
- return item_or_list
-
- def _save_nodes(
- self,
- nodes: Union["DependencyMixin", Sequence["DependencyMixin"]],
- stream: List["DependencyMixin"],
- ):
- from airflow.models.xcom_arg import XComArg
- from airflow.utils.task_group import TaskGroup
-
- for node in self._make_list(nodes):
- if isinstance(node, (TaskGroup, XComArg)):
- stream.append(node)
- elif isinstance(node, DAGNode):
- if node.task_group and not node.task_group.is_root:
- stream.append(node.task_group)
- else:
- stream.append(node)
- else:
- raise TypeError(
- f"Cannot use edge labels with {type(node).__name__}, "
- f"only tasks, XComArg or TaskGroups"
- )
-
def set_upstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
+ self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
):
"""
Sets the given task/list onto the upstream attribute, and then checks if
@@ -89,14 +62,30 @@ class EdgeModifier(DependencyMixin):
Providing this also provides << via DependencyMixin.
"""
- self._save_nodes(other, self._upstream)
- for node in self._downstream:
- node.set_upstream(other, edge_modifier=self)
+ from airflow.models.baseoperator import BaseOperator
+
+ # Ensure we have a list, even if it's just one item
+ if isinstance(task_or_task_list, DependencyMixin):
+ task_or_task_list = [task_or_task_list]
+ # Unfurl it into actual operators
+ operators: List[BaseOperator] = []
+ for task in task_or_task_list:
+ for root in task.roots:
+ if not isinstance(root, BaseOperator):
+ raise TypeError(f"Cannot use edge labels with {type(root).__name__}, only operators")
+ operators.append(root)
+ # For each already-declared downstream, pair off with each new upstream
+ # item and store the edge info.
+ for operator in operators:
+ for downstream in self._downstream:
+ self.add_edge_info(operator.dag, operator.task_id, downstream.task_id)
+ if chain:
+ operator.set_downstream(downstream)
+ # Add the new tasks to our list of ones we've seen
+ self._upstream.extend(operators)
def set_downstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
+ self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
):
"""
Sets the given task/list onto the downstream attribute, and then checks if
@@ -104,24 +93,36 @@ class EdgeModifier(DependencyMixin):
Providing this also provides >> via DependencyMixin.
"""
- self._save_nodes(other, self._downstream)
- for node in self._upstream:
- node.set_downstream(other, edge_modifier=self)
-
- def update_relative(
- self,
- other: "DependencyMixin",
- upstream: bool = True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ from airflow.models.baseoperator import BaseOperator
+
+ # Ensure we have a list, even if it's just one item
+ if isinstance(task_or_task_list, DependencyMixin):
+ task_or_task_list = [task_or_task_list]
+ # Unfurl it into actual operators
+ operators: List[BaseOperator] = []
+ for task in task_or_task_list:
+ for leaf in task.leaves:
+ if not isinstance(leaf, BaseOperator):
+ raise TypeError(f"Cannot use edge labels with {type(leaf).__name__}, only operators")
+ operators.append(leaf)
+ # Pair them off with existing
+ for operator in operators:
+ for upstream in self._upstream:
+ self.add_edge_info(upstream.dag, upstream.task_id, operator.task_id)
+ if chain:
+ upstream.set_downstream(operator)
+ # Add the new tasks to our list of ones we've seen
+ self._downstream.extend(operators)
+
+ def update_relative(self, other: DependencyMixin, upstream: bool = True) -> None:
"""
Called if we're not the "main" side of a relationship; we still run the
same logic, though.
"""
if upstream:
- self.set_upstream(other)
+ self.set_upstream(other, chain=False)
else:
- self.set_downstream(other)
+ self.set_downstream(other, chain=False)
def add_edge_info(self, dag, upstream_id: str, downstream_id: str):
"""
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index 11ee806beb..7b53a521db 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -229,12 +229,7 @@ class TaskGroup(DAGNode):
"""group_id excluding parent's group_id used as the node label in UI."""
return self._group_id
- def update_relative(
- self,
- other: DependencyMixin,
- upstream=True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ def update_relative(self, other: DependencyMixin, upstream=True) -> None:
"""
Overrides TaskMixin.update_relative.
@@ -259,18 +254,10 @@ class TaskGroup(DAGNode):
f"or operators; received {task.__class__.__name__}"
)
- # Do not set a relationship between a TaskGroup and a Label's roots
- if self == task:
- continue
-
if upstream:
self.upstream_task_ids.add(task.node_id)
- if edge_modifier:
- edge_modifier.add_edge_info(self.dag, task.node_id, self.upstream_join_id)
else:
self.downstream_task_ids.add(task.node_id)
- if edge_modifier:
- edge_modifier.add_edge_info(self.dag, self.downstream_join_id, task.node_id)
def _set_relatives(
self,
@@ -293,7 +280,7 @@ class TaskGroup(DAGNode):
task_or_task_list = [task_or_task_list]
for task_like in task_or_task_list:
- self.update_relative(task_like, upstream, edge_modifier=edge_modifier)
+ self.update_relative(task_like, upstream)
def __enter__(self) -> "TaskGroup":
TaskGroupContext.push_context_managed_task_group(self)
diff --git a/tests/utils/test_edgemodifier.py b/tests/utils/test_edgemodifier.py
index da3291fad1..21573b3cf6 100644
--- a/tests/utils/test_edgemodifier.py
+++ b/tests/utils/test_edgemodifier.py
@@ -21,11 +21,9 @@ import pytest
from airflow import DAG
from airflow.models.xcom_arg import XComArg
-from airflow.operators.empty import EmptyOperator
from airflow.operators.python import PythonOperator
from airflow.utils.edgemodifier import Label
from airflow.utils.task_group import TaskGroup
-from airflow.www.views import dag_edges
DEFAULT_ARGS = {
"owner": "test",
@@ -64,125 +62,6 @@ def test_taskgroup_dag():
return dag, group, (op1, op2, op3, op4)
-@pytest.fixture
-def test_complex_taskgroup_dag():
- """Creates a test DAG with many operators and a task group."""
-
- def f(task_id):
- return f"OP:{task_id}"
-
- with DAG(dag_id="test_complex_dag", default_args=DEFAULT_ARGS) as dag:
- with TaskGroup("group_1") as group:
- group_dm1 = EmptyOperator(task_id="group_dummy1")
- group_dm2 = EmptyOperator(task_id="group_dummy2")
- group_dm3 = EmptyOperator(task_id="group_dummy3")
- dm_in1 = EmptyOperator(task_id="dummy_in1")
- dm_in2 = EmptyOperator(task_id="dummy_in2")
- dm_in3 = EmptyOperator(task_id="dummy_in3")
- dm_in4 = EmptyOperator(task_id="dummy_in4")
- dm_out1 = EmptyOperator(task_id="dummy_out1")
- dm_out2 = EmptyOperator(task_id="dummy_out2")
- dm_out3 = EmptyOperator(task_id="dummy_out3")
- dm_out4 = EmptyOperator(task_id="dummy_out4")
- op_in1 = PythonOperator(python_callable=f, task_id="op_in1")
- op_out1 = PythonOperator(python_callable=f, task_id="op_out1")
-
- return (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- )
-
-
-@pytest.fixture
-def simple_dag_expected_edges():
- return [
- {'source_id': 'group_1.downstream_join_id', 'target_id': 'test_op_4'},
- {'source_id': 'group_1.test_op_2', 'target_id': 'group_1.downstream_join_id'},
- {'source_id': 'group_1.test_op_3', 'target_id': 'group_1.downstream_join_id'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_2'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_3'},
- {'label': 'Label', 'source_id': 'test_op_1', 'target_id': 'group_1.upstream_join_id'},
- ]
-
-
-@pytest.fixture
-def complex_dag_expected_edges():
- return [
- {'source_id': 'dummy_in1', 'target_id': 'group_1.upstream_join_id'},
- {
- 'label': 'label dm_in2 <=> group',
- 'source_id': 'dummy_in2',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label dm_in3/dm_in4 <=> group',
- 'source_id': 'dummy_in3',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label dm_in3/dm_in4 <=> group',
- 'source_id': 'dummy_in4',
- 'target_id': 'group_1.upstream_join_id',
- },
- {'source_id': 'group_1.downstream_join_id', 'target_id': 'dummy_out1'},
- {
- 'label': 'label group <=> dm_out2',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out2',
- },
- {
- 'label': 'label group <=> dm_out3/dm_out4',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out3',
- },
- {
- 'label': 'label group <=> dm_out3/dm_out4',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out4',
- },
- {
- 'source_id': 'group_1.group_dummy1',
- 'target_id': 'group_1.downstream_join_id',
- },
- {'source_id': 'group_1.group_dummy2', 'target_id': 'group_1.group_dummy1'},
- {'source_id': 'group_1.group_dummy3', 'target_id': 'group_1.group_dummy1'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy2'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy3'},
- {
- 'label': 'label op_in1 <=> group',
- 'source_id': 'op_in1',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label group <=> op_out1',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'op_out1',
- },
- ]
-
-
-def compare_dag_edges(current, expected):
- assert len(current) == len(expected)
-
- for i in current:
- assert current.count(i) == expected.count(i), f'The unexpected DAG edge: {i}'
-
-
class TestEdgeModifierBuilding:
"""
Tests that EdgeModifiers work when composed with Tasks (either via >>
@@ -283,87 +162,3 @@ class TestEdgeModifierBuilding:
assert dag.get_edge_info(op1.task_id, op2.task_id) == {"label": "Group label"}
assert dag.get_edge_info(op1.task_id, op3.task_id) == {"label": "Group label"}
assert dag.get_edge_info(op3.task_id, op4.task_id) == {}
-
- def test_simple_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
- """Tests the simple dag with a TaskGroup and a Label"""
- dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
- op1 >> Label("Label") >> group >> op4
- compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
- def test_simple_reversed_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
- """Tests the simple reversed dag with a TaskGroup and a Label"""
- dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
- op4 << group << Label("Label") << op1
- compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
- def test_complex_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
- """Tests the complex dag with a TaskGroup and a Label"""
- (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- ) = test_complex_taskgroup_dag
-
- [group_dm2, group_dm3] >> group_dm1
-
- dm_in1 >> group
- dm_in2 >> Label('label dm_in2 <=> group') >> group
- [dm_in3, dm_in4] >> Label('label dm_in3/dm_in4 <=> group') >> group
- XComArg(op_in1, 'test_key') >> Label('label op_in1 <=> group') >> group
-
- group >> dm_out1
- group >> Label('label group <=> dm_out2') >> dm_out2
- group >> Label('label group <=> dm_out3/dm_out4') >> [dm_out3, dm_out4]
- group >> Label('label group <=> op_out1') >> XComArg(op_out1, 'test_key')
-
- compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)
-
- def test_complex_reversed_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
- """Tests the complex reversed dag with a TaskGroup and a Label"""
- (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- ) = test_complex_taskgroup_dag
-
- group_dm1 << [group_dm2, group_dm3]
-
- group << dm_in1
- group << Label('label dm_in2 <=> group') << dm_in2
- group << Label('label dm_in3/dm_in4 <=> group') << [dm_in3, dm_in4]
- group << Label('label op_in1 <=> group') << XComArg(op_in1, 'test_key')
-
- dm_out1 << group
- dm_out2 << Label('label group <=> dm_out2') << group
- [dm_out3, dm_out4] << Label('label group <=> dm_out3/dm_out4') << group
- XComArg(op_out1, 'test_key') << Label('label group <=> op_out1') << group
-
- compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)