You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/04/27 19:53:35 UTC

[airflow] 02/05: Revert "EdgeModifier refactoring (#21404)" (#23291)

This is an automated email from the ASF dual-hosted git repository.

ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit a83b6d4630fec00e2e48f00f20d67ddcfe0b9f88
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Wed Apr 27 19:12:08 2022 +0100

    Revert "EdgeModifier refactoring (#21404)" (#23291)
    
    This reverts commit ace8c6e942ff5554639801468b971915b7c0e9b9.
    
    Fixes apache/airflow#23285
    
    This dag breaks with a cycle as group.end has itself as a downstream otherwise:
    
    ```python
    from pendulum import datetime
    
    from airflow.decorators import dag, task, task_group
    from airflow.utils.edgemodifier import Label
    
    @task
    def begin():
        ...
    
    @task
    def end():
        ...
    
    @dag(start_date=datetime(2022, 1, 1), schedule_interval=None)
    def task_groups_with_edge_labels():
        @task_group
        def group():
            begin() >> end()
    
        group()
    
    _ = task_groups_with_edge_labels()
    ```
    
    (cherry picked from commit 3182303ce50bda6d5d27a6ef4e19450fb4e47eea)
---
 airflow/models/taskmixin.py      |  21 +---
 airflow/utils/edgemodifier.py    | 107 ++++++++++----------
 airflow/utils/task_group.py      |  17 +---
 tests/utils/test_edgemodifier.py | 205 ---------------------------------------
 4 files changed, 60 insertions(+), 290 deletions(-)

diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py
index 1a7129f66e..1d66a719a6 100644
--- a/airflow/models/taskmixin.py
+++ b/airflow/models/taskmixin.py
@@ -55,29 +55,16 @@ class DependencyMixin:
         raise NotImplementedError()
 
     @abstractmethod
-    def set_upstream(
-        self,
-        other: Union["DependencyMixin", Sequence["DependencyMixin"]],
-        edge_modifier: Optional["EdgeModifier"] = None,
-    ):
+    def set_upstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
         """Set a task or a task list to be directly upstream from the current task."""
         raise NotImplementedError()
 
     @abstractmethod
-    def set_downstream(
-        self,
-        other: Union["DependencyMixin", Sequence["DependencyMixin"]],
-        edge_modifier: Optional["EdgeModifier"] = None,
-    ):
+    def set_downstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
         """Set a task or a task list to be directly downstream from the current task."""
         raise NotImplementedError()
 
-    def update_relative(
-        self,
-        other: "DependencyMixin",
-        upstream=True,
-        edge_modifier: Optional["EdgeModifier"] = None,
-    ) -> None:
+    def update_relative(self, other: "DependencyMixin", upstream=True) -> None:
         """
         Update relationship information about another TaskMixin. Default is no-op.
         Override if necessary.
@@ -184,7 +171,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
 
         task_list: List[Operator] = []
         for task_object in task_or_task_list:
-            task_object.update_relative(self, not upstream, edge_modifier=edge_modifier)
+            task_object.update_relative(self, not upstream)
             relatives = task_object.leaves if upstream else task_object.roots
             for task in relatives:
                 if not isinstance(task, (BaseOperator, MappedOperator)):
diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py
index 109199fc7c..a59616c76b 100644
--- a/airflow/utils/edgemodifier.py
+++ b/airflow/utils/edgemodifier.py
@@ -14,9 +14,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from typing import List, Optional, Sequence, Union
+from typing import TYPE_CHECKING, List, Optional, Sequence, Union
 
-from airflow.models.taskmixin import DAGNode, DependencyMixin
+from airflow.models.taskmixin import DependencyMixin
+
+if TYPE_CHECKING:
+    from airflow.models.baseoperator import BaseOperator
 
 
 class EdgeModifier(DependencyMixin):
@@ -39,8 +42,8 @@ class EdgeModifier(DependencyMixin):
 
     def __init__(self, label: Optional[str] = None):
         self.label = label
-        self._upstream: List["DependencyMixin"] = []
-        self._downstream: List["DependencyMixin"] = []
+        self._upstream: List["BaseOperator"] = []
+        self._downstream: List["BaseOperator"] = []
 
     @property
     def roots(self):
@@ -50,38 +53,8 @@ class EdgeModifier(DependencyMixin):
     def leaves(self):
         return self._upstream
 
-    @staticmethod
-    def _make_list(item_or_list):
-        if not isinstance(item_or_list, Sequence):
-            return [item_or_list]
-        return item_or_list
-
-    def _save_nodes(
-        self,
-        nodes: Union["DependencyMixin", Sequence["DependencyMixin"]],
-        stream: List["DependencyMixin"],
-    ):
-        from airflow.models.xcom_arg import XComArg
-        from airflow.utils.task_group import TaskGroup
-
-        for node in self._make_list(nodes):
-            if isinstance(node, (TaskGroup, XComArg)):
-                stream.append(node)
-            elif isinstance(node, DAGNode):
-                if node.task_group and not node.task_group.is_root:
-                    stream.append(node.task_group)
-                else:
-                    stream.append(node)
-            else:
-                raise TypeError(
-                    f"Cannot use edge labels with {type(node).__name__}, "
-                    f"only tasks, XComArg or TaskGroups"
-                )
-
     def set_upstream(
-        self,
-        other: Union["DependencyMixin", Sequence["DependencyMixin"]],
-        edge_modifier: Optional["EdgeModifier"] = None,
+        self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
     ):
         """
         Sets the given task/list onto the upstream attribute, and then checks if
@@ -89,14 +62,30 @@ class EdgeModifier(DependencyMixin):
 
         Providing this also provides << via DependencyMixin.
         """
-        self._save_nodes(other, self._upstream)
-        for node in self._downstream:
-            node.set_upstream(other, edge_modifier=self)
+        from airflow.models.baseoperator import BaseOperator
+
+        # Ensure we have a list, even if it's just one item
+        if isinstance(task_or_task_list, DependencyMixin):
+            task_or_task_list = [task_or_task_list]
+        # Unfurl it into actual operators
+        operators: List[BaseOperator] = []
+        for task in task_or_task_list:
+            for root in task.roots:
+                if not isinstance(root, BaseOperator):
+                    raise TypeError(f"Cannot use edge labels with {type(root).__name__}, only operators")
+                operators.append(root)
+        # For each already-declared downstream, pair off with each new upstream
+        # item and store the edge info.
+        for operator in operators:
+            for downstream in self._downstream:
+                self.add_edge_info(operator.dag, operator.task_id, downstream.task_id)
+                if chain:
+                    operator.set_downstream(downstream)
+        # Add the new tasks to our list of ones we've seen
+        self._upstream.extend(operators)
 
     def set_downstream(
-        self,
-        other: Union["DependencyMixin", Sequence["DependencyMixin"]],
-        edge_modifier: Optional["EdgeModifier"] = None,
+        self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
     ):
         """
         Sets the given task/list onto the downstream attribute, and then checks if
@@ -104,24 +93,36 @@ class EdgeModifier(DependencyMixin):
 
         Providing this also provides >> via DependencyMixin.
         """
-        self._save_nodes(other, self._downstream)
-        for node in self._upstream:
-            node.set_downstream(other, edge_modifier=self)
-
-    def update_relative(
-        self,
-        other: "DependencyMixin",
-        upstream: bool = True,
-        edge_modifier: Optional["EdgeModifier"] = None,
-    ) -> None:
+        from airflow.models.baseoperator import BaseOperator
+
+        # Ensure we have a list, even if it's just one item
+        if isinstance(task_or_task_list, DependencyMixin):
+            task_or_task_list = [task_or_task_list]
+        # Unfurl it into actual operators
+        operators: List[BaseOperator] = []
+        for task in task_or_task_list:
+            for leaf in task.leaves:
+                if not isinstance(leaf, BaseOperator):
+                    raise TypeError(f"Cannot use edge labels with {type(leaf).__name__}, only operators")
+                operators.append(leaf)
+        # Pair them off with existing
+        for operator in operators:
+            for upstream in self._upstream:
+                self.add_edge_info(upstream.dag, upstream.task_id, operator.task_id)
+                if chain:
+                    upstream.set_downstream(operator)
+        # Add the new tasks to our list of ones we've seen
+        self._downstream.extend(operators)
+
+    def update_relative(self, other: DependencyMixin, upstream: bool = True) -> None:
         """
         Called if we're not the "main" side of a relationship; we still run the
         same logic, though.
         """
         if upstream:
-            self.set_upstream(other)
+            self.set_upstream(other, chain=False)
         else:
-            self.set_downstream(other)
+            self.set_downstream(other, chain=False)
 
     def add_edge_info(self, dag, upstream_id: str, downstream_id: str):
         """
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index 11ee806beb..7b53a521db 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -229,12 +229,7 @@ class TaskGroup(DAGNode):
         """group_id excluding parent's group_id used as the node label in UI."""
         return self._group_id
 
-    def update_relative(
-        self,
-        other: DependencyMixin,
-        upstream=True,
-        edge_modifier: Optional["EdgeModifier"] = None,
-    ) -> None:
+    def update_relative(self, other: DependencyMixin, upstream=True) -> None:
         """
         Overrides TaskMixin.update_relative.
 
@@ -259,18 +254,10 @@ class TaskGroup(DAGNode):
                         f"or operators; received {task.__class__.__name__}"
                     )
 
-                # Do not set a relationship between a TaskGroup and a Label's roots
-                if self == task:
-                    continue
-
                 if upstream:
                     self.upstream_task_ids.add(task.node_id)
-                    if edge_modifier:
-                        edge_modifier.add_edge_info(self.dag, task.node_id, self.upstream_join_id)
                 else:
                     self.downstream_task_ids.add(task.node_id)
-                    if edge_modifier:
-                        edge_modifier.add_edge_info(self.dag, self.downstream_join_id, task.node_id)
 
     def _set_relatives(
         self,
@@ -293,7 +280,7 @@ class TaskGroup(DAGNode):
             task_or_task_list = [task_or_task_list]
 
         for task_like in task_or_task_list:
-            self.update_relative(task_like, upstream, edge_modifier=edge_modifier)
+            self.update_relative(task_like, upstream)
 
     def __enter__(self) -> "TaskGroup":
         TaskGroupContext.push_context_managed_task_group(self)
diff --git a/tests/utils/test_edgemodifier.py b/tests/utils/test_edgemodifier.py
index da3291fad1..21573b3cf6 100644
--- a/tests/utils/test_edgemodifier.py
+++ b/tests/utils/test_edgemodifier.py
@@ -21,11 +21,9 @@ import pytest
 
 from airflow import DAG
 from airflow.models.xcom_arg import XComArg
-from airflow.operators.empty import EmptyOperator
 from airflow.operators.python import PythonOperator
 from airflow.utils.edgemodifier import Label
 from airflow.utils.task_group import TaskGroup
-from airflow.www.views import dag_edges
 
 DEFAULT_ARGS = {
     "owner": "test",
@@ -64,125 +62,6 @@ def test_taskgroup_dag():
             return dag, group, (op1, op2, op3, op4)
 
 
-@pytest.fixture
-def test_complex_taskgroup_dag():
-    """Creates a test DAG with many operators and a task group."""
-
-    def f(task_id):
-        return f"OP:{task_id}"
-
-    with DAG(dag_id="test_complex_dag", default_args=DEFAULT_ARGS) as dag:
-        with TaskGroup("group_1") as group:
-            group_dm1 = EmptyOperator(task_id="group_dummy1")
-            group_dm2 = EmptyOperator(task_id="group_dummy2")
-            group_dm3 = EmptyOperator(task_id="group_dummy3")
-        dm_in1 = EmptyOperator(task_id="dummy_in1")
-        dm_in2 = EmptyOperator(task_id="dummy_in2")
-        dm_in3 = EmptyOperator(task_id="dummy_in3")
-        dm_in4 = EmptyOperator(task_id="dummy_in4")
-        dm_out1 = EmptyOperator(task_id="dummy_out1")
-        dm_out2 = EmptyOperator(task_id="dummy_out2")
-        dm_out3 = EmptyOperator(task_id="dummy_out3")
-        dm_out4 = EmptyOperator(task_id="dummy_out4")
-        op_in1 = PythonOperator(python_callable=f, task_id="op_in1")
-        op_out1 = PythonOperator(python_callable=f, task_id="op_out1")
-
-        return (
-            dag,
-            group,
-            (
-                group_dm1,
-                group_dm2,
-                group_dm3,
-                dm_in1,
-                dm_in2,
-                dm_in3,
-                dm_in4,
-                dm_out1,
-                dm_out2,
-                dm_out3,
-                dm_out4,
-                op_in1,
-                op_out1,
-            ),
-        )
-
-
-@pytest.fixture
-def simple_dag_expected_edges():
-    return [
-        {'source_id': 'group_1.downstream_join_id', 'target_id': 'test_op_4'},
-        {'source_id': 'group_1.test_op_2', 'target_id': 'group_1.downstream_join_id'},
-        {'source_id': 'group_1.test_op_3', 'target_id': 'group_1.downstream_join_id'},
-        {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_2'},
-        {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_3'},
-        {'label': 'Label', 'source_id': 'test_op_1', 'target_id': 'group_1.upstream_join_id'},
-    ]
-
-
-@pytest.fixture
-def complex_dag_expected_edges():
-    return [
-        {'source_id': 'dummy_in1', 'target_id': 'group_1.upstream_join_id'},
-        {
-            'label': 'label dm_in2 <=> group',
-            'source_id': 'dummy_in2',
-            'target_id': 'group_1.upstream_join_id',
-        },
-        {
-            'label': 'label dm_in3/dm_in4 <=> group',
-            'source_id': 'dummy_in3',
-            'target_id': 'group_1.upstream_join_id',
-        },
-        {
-            'label': 'label dm_in3/dm_in4 <=> group',
-            'source_id': 'dummy_in4',
-            'target_id': 'group_1.upstream_join_id',
-        },
-        {'source_id': 'group_1.downstream_join_id', 'target_id': 'dummy_out1'},
-        {
-            'label': 'label group <=> dm_out2',
-            'source_id': 'group_1.downstream_join_id',
-            'target_id': 'dummy_out2',
-        },
-        {
-            'label': 'label group <=> dm_out3/dm_out4',
-            'source_id': 'group_1.downstream_join_id',
-            'target_id': 'dummy_out3',
-        },
-        {
-            'label': 'label group <=> dm_out3/dm_out4',
-            'source_id': 'group_1.downstream_join_id',
-            'target_id': 'dummy_out4',
-        },
-        {
-            'source_id': 'group_1.group_dummy1',
-            'target_id': 'group_1.downstream_join_id',
-        },
-        {'source_id': 'group_1.group_dummy2', 'target_id': 'group_1.group_dummy1'},
-        {'source_id': 'group_1.group_dummy3', 'target_id': 'group_1.group_dummy1'},
-        {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy2'},
-        {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy3'},
-        {
-            'label': 'label op_in1 <=> group',
-            'source_id': 'op_in1',
-            'target_id': 'group_1.upstream_join_id',
-        },
-        {
-            'label': 'label group <=> op_out1',
-            'source_id': 'group_1.downstream_join_id',
-            'target_id': 'op_out1',
-        },
-    ]
-
-
-def compare_dag_edges(current, expected):
-    assert len(current) == len(expected)
-
-    for i in current:
-        assert current.count(i) == expected.count(i), f'The unexpected DAG edge: {i}'
-
-
 class TestEdgeModifierBuilding:
     """
     Tests that EdgeModifiers work when composed with Tasks (either via >>
@@ -283,87 +162,3 @@ class TestEdgeModifierBuilding:
         assert dag.get_edge_info(op1.task_id, op2.task_id) == {"label": "Group label"}
         assert dag.get_edge_info(op1.task_id, op3.task_id) == {"label": "Group label"}
         assert dag.get_edge_info(op3.task_id, op4.task_id) == {}
-
-    def test_simple_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
-        """Tests the simple dag with a TaskGroup and a Label"""
-        dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
-        op1 >> Label("Label") >> group >> op4
-        compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
-    def test_simple_reversed_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
-        """Tests the simple reversed dag with a TaskGroup and a Label"""
-        dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
-        op4 << group << Label("Label") << op1
-        compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
-    def test_complex_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
-        """Tests the complex dag with a TaskGroup and a Label"""
-        (
-            dag,
-            group,
-            (
-                group_dm1,
-                group_dm2,
-                group_dm3,
-                dm_in1,
-                dm_in2,
-                dm_in3,
-                dm_in4,
-                dm_out1,
-                dm_out2,
-                dm_out3,
-                dm_out4,
-                op_in1,
-                op_out1,
-            ),
-        ) = test_complex_taskgroup_dag
-
-        [group_dm2, group_dm3] >> group_dm1
-
-        dm_in1 >> group
-        dm_in2 >> Label('label dm_in2 <=> group') >> group
-        [dm_in3, dm_in4] >> Label('label dm_in3/dm_in4 <=> group') >> group
-        XComArg(op_in1, 'test_key') >> Label('label op_in1 <=> group') >> group
-
-        group >> dm_out1
-        group >> Label('label group <=> dm_out2') >> dm_out2
-        group >> Label('label group <=> dm_out3/dm_out4') >> [dm_out3, dm_out4]
-        group >> Label('label group <=> op_out1') >> XComArg(op_out1, 'test_key')
-
-        compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)
-
-    def test_complex_reversed_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
-        """Tests the complex reversed dag with a TaskGroup and a Label"""
-        (
-            dag,
-            group,
-            (
-                group_dm1,
-                group_dm2,
-                group_dm3,
-                dm_in1,
-                dm_in2,
-                dm_in3,
-                dm_in4,
-                dm_out1,
-                dm_out2,
-                dm_out3,
-                dm_out4,
-                op_in1,
-                op_out1,
-            ),
-        ) = test_complex_taskgroup_dag
-
-        group_dm1 << [group_dm2, group_dm3]
-
-        group << dm_in1
-        group << Label('label dm_in2 <=> group') << dm_in2
-        group << Label('label dm_in3/dm_in4 <=> group') << [dm_in3, dm_in4]
-        group << Label('label op_in1 <=> group') << XComArg(op_in1, 'test_key')
-
-        dm_out1 << group
-        dm_out2 << Label('label group <=> dm_out2') << group
-        [dm_out3, dm_out4] << Label('label group <=> dm_out3/dm_out4') << group
-        XComArg(op_out1, 'test_key') << Label('label group <=> op_out1') << group
-
-        compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)