You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/12/08 16:27:46 UTC

[airflow] branch master updated: Moved subdag_operator.py to subdag.py (#11307)

This is an automated email from the ASF dual-hosted git repository.

kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/master by this push:
     new bfbd4bb  Moved subdag_operator.py to subdag.py (#11307)
bfbd4bb is described below

commit bfbd4bbb706d8c358f310c98470a613090d709ad
Author: Kishore Vancheeshwaran <24...@users.noreply.github.com>
AuthorDate: Tue Dec 8 21:56:39 2020 +0530

    Moved subdag_operator.py to subdag.py (#11307)
    
    Part of #11178
---
 airflow/api/common/experimental/mark_tasks.py      |   2 +-
 airflow/example_dags/example_subdag_operator.py    |   2 +-
 airflow/executors/celery_executor.py               |   2 +-
 airflow/models/dag.py                              |   4 +-
 .../operators/{subdag_operator.py => subdag.py}    |   5 +-
 airflow/operators/subdag_operator.py               | 203 +--------------------
 docs/apache-airflow/concepts.rst                   |  12 +-
 docs/apache-airflow/operators-and-hooks-ref.rst    |   2 +-
 tests/dags/test_clear_subdag.py                    |   2 +-
 tests/dags/test_impersonation_subdag.py            |   2 +-
 tests/dags/test_subdag.py                          |   2 +-
 tests/deprecated_classes.py                        |   4 +
 tests/models/test_dag.py                           |   2 +-
 tests/models/test_dagbag.py                        |   6 +-
 tests/operators/test_subdag_operator.py            |   6 +-
 15 files changed, 35 insertions(+), 221 deletions(-)

diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py
index 852038d..04215dc 100644
--- a/airflow/api/common/experimental/mark_tasks.py
+++ b/airflow/api/common/experimental/mark_tasks.py
@@ -25,7 +25,7 @@ from sqlalchemy import or_
 from airflow.models.baseoperator import BaseOperator
 from airflow.models.dagrun import DagRun
 from airflow.models.taskinstance import TaskInstance
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 from airflow.utils import timezone
 from airflow.utils.session import provide_session
 from airflow.utils.state import State
diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py
index de2853d..6e38b60 100644
--- a/airflow/example_dags/example_subdag_operator.py
+++ b/airflow/example_dags/example_subdag_operator.py
@@ -22,7 +22,7 @@
 from airflow import DAG
 from airflow.example_dags.subdags.subdag import subdag
 from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 from airflow.utils.dates import days_ago
 
 DAG_NAME = 'example_subdag_operator'
diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py
index 9fb9f63..f4f3412 100644
--- a/airflow/executors/celery_executor.py
+++ b/airflow/executors/celery_executor.py
@@ -184,7 +184,7 @@ def on_celery_import_modules(*args, **kwargs):
     import airflow.macros
     import airflow.operators.bash
     import airflow.operators.python
-    import airflow.operators.subdag_operator  # noqa: F401
+    import airflow.operators.subdag  # noqa: F401
 
     try:
         import kubernetes.client  # noqa: F401
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index 4f9333e..be6403c 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -952,7 +952,7 @@ class DAG(LoggingMixin):
     def subdags(self):
         """Returns a list of the subdag objects associated to this DAG"""
         # Check SubDag for class but don't check class directly
-        from airflow.operators.subdag_operator import SubDagOperator
+        from airflow.operators.subdag import SubDagOperator
 
         subdag_lst = []
         for task in self.tasks:
@@ -1061,7 +1061,7 @@ class DAG(LoggingMixin):
         :param include_subdag_tasks: whether to include tasks in subdags, default to False
         :return: list of tasks in topological order
         """
-        from airflow.operators.subdag_operator import SubDagOperator  # Avoid circular import
+        from airflow.operators.subdag import SubDagOperator  # Avoid circular import
 
         # convert into an OrderedDict to speedup lookup while keeping order the same
         graph_unsorted = OrderedDict((task.task_id, task) for task in self.tasks)
diff --git a/airflow/operators/subdag_operator.py b/airflow/operators/subdag.py
similarity index 99%
copy from airflow/operators/subdag_operator.py
copy to airflow/operators/subdag.py
index f5c147e..538d7cc 100644
--- a/airflow/operators/subdag_operator.py
+++ b/airflow/operators/subdag.py
@@ -45,7 +45,6 @@ class SubDagOperator(BaseSensorOperator):
     """
     This runs a sub dag. By convention, a sub dag's dag_id
     should be prefixed by its parent and a dot. As in `parent.child`.
-
     Although SubDagOperator can occupy a pool/concurrency slot,
     user can specify the mode=reschedule so that the slot will be
     released periodically to avoid potential deadlock.
@@ -55,7 +54,8 @@ class SubDagOperator(BaseSensorOperator):
     :param conf: Configuration for the subdag
     :type conf: dict
     :param propagate_skipped_state: by setting this argument you can define
-        whether the skipped state of leaf task(s) should be propagated to the parent dag's downstream task.
+        whether the skipped state of leaf task(s) should be propagated to the
+        parent dag's downstream task.
     """
 
     ui_color = '#555'
@@ -123,7 +123,6 @@ class SubDagOperator(BaseSensorOperator):
         """
         Set the DagRun state to RUNNING and set the failed TaskInstances to None state
         for scheduler to pick up.
-
         :param dag_run: DAG run
         :param execution_date: Execution date
         :return: None
diff --git a/airflow/operators/subdag_operator.py b/airflow/operators/subdag_operator.py
index f5c147e..de6c607 100644
--- a/airflow/operators/subdag_operator.py
+++ b/airflow/operators/subdag_operator.py
@@ -15,202 +15,13 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""The module which provides a way to nest your DAGs and so your levels of complexity."""
-from enum import Enum
-from typing import Dict, Optional
+"""This module is deprecated. Please use `airflow.operators.subdag`."""
 
-from sqlalchemy.orm.session import Session
+import warnings
 
-from airflow.api.common.experimental.get_task_instance import get_task_instance
-from airflow.exceptions import AirflowException, TaskInstanceNotFound
-from airflow.models import DagRun
-from airflow.models.dag import DAG, DagContext
-from airflow.models.pool import Pool
-from airflow.models.taskinstance import TaskInstance
-from airflow.sensors.base_sensor_operator import BaseSensorOperator
-from airflow.utils.decorators import apply_defaults
-from airflow.utils.session import create_session, provide_session
-from airflow.utils.state import State
-from airflow.utils.types import DagRunType
+# pylint: disable=unused-import
+from airflow.operators.subdag import SkippedStatePropagationOptions, SubDagOperator  # noqa
 
-
-class SkippedStatePropagationOptions(Enum):
-    """Available options for skipped state propagation of subdag's tasks to parent dag tasks."""
-
-    ALL_LEAVES = 'all_leaves'
-    ANY_LEAF = 'any_leaf'
-
-
-class SubDagOperator(BaseSensorOperator):
-    """
-    This runs a sub dag. By convention, a sub dag's dag_id
-    should be prefixed by its parent and a dot. As in `parent.child`.
-
-    Although SubDagOperator can occupy a pool/concurrency slot,
-    user can specify the mode=reschedule so that the slot will be
-    released periodically to avoid potential deadlock.
-
-    :param subdag: the DAG object to run as a subdag of the current DAG.
-    :param session: sqlalchemy session
-    :param conf: Configuration for the subdag
-    :type conf: dict
-    :param propagate_skipped_state: by setting this argument you can define
-        whether the skipped state of leaf task(s) should be propagated to the parent dag's downstream task.
-    """
-
-    ui_color = '#555'
-    ui_fgcolor = '#fff'
-
-    @provide_session
-    @apply_defaults
-    def __init__(
-        self,
-        *,
-        subdag: DAG,
-        session: Optional[Session] = None,
-        conf: Optional[Dict] = None,
-        propagate_skipped_state: Optional[SkippedStatePropagationOptions] = None,
-        **kwargs,
-    ) -> None:
-        super().__init__(**kwargs)
-        self.subdag = subdag
-        self.conf = conf
-        self.propagate_skipped_state = propagate_skipped_state
-
-        self._validate_dag(kwargs)
-        self._validate_pool(session)
-
-    def _validate_dag(self, kwargs):
-        dag = kwargs.get('dag') or DagContext.get_current_dag()
-
-        if not dag:
-            raise AirflowException('Please pass in the `dag` param or call within a DAG context manager')
-
-        if dag.dag_id + '.' + kwargs['task_id'] != self.subdag.dag_id:
-            raise AirflowException(
-                "The subdag's dag_id should have the form '{{parent_dag_id}}.{{this_task_id}}'. "
-                "Expected '{d}.{t}'; received '{rcvd}'.".format(
-                    d=dag.dag_id, t=kwargs['task_id'], rcvd=self.subdag.dag_id
-                )
-            )
-
-    def _validate_pool(self, session):
-        if self.pool:
-            conflicts = [t for t in self.subdag.tasks if t.pool == self.pool]
-            if conflicts:
-                # only query for pool conflicts if one may exist
-                pool = session.query(Pool).filter(Pool.slots == 1).filter(Pool.pool == self.pool).first()
-                if pool and any(t.pool == self.pool for t in self.subdag.tasks):
-                    raise AirflowException(
-                        'SubDagOperator {sd} and subdag task{plural} {t} both '
-                        'use pool {p}, but the pool only has 1 slot. The '
-                        'subdag tasks will never run.'.format(
-                            sd=self.task_id,
-                            plural=len(conflicts) > 1,
-                            t=', '.join(t.task_id for t in conflicts),
-                            p=self.pool,
-                        )
-                    )
-
-    def _get_dagrun(self, execution_date):
-        dag_runs = DagRun.find(
-            dag_id=self.subdag.dag_id,
-            execution_date=execution_date,
-        )
-        return dag_runs[0] if dag_runs else None
-
-    def _reset_dag_run_and_task_instances(self, dag_run, execution_date):
-        """
-        Set the DagRun state to RUNNING and set the failed TaskInstances to None state
-        for scheduler to pick up.
-
-        :param dag_run: DAG run
-        :param execution_date: Execution date
-        :return: None
-        """
-        with create_session() as session:
-            dag_run.state = State.RUNNING
-            session.merge(dag_run)
-            failed_task_instances = (
-                session.query(TaskInstance)
-                .filter(TaskInstance.dag_id == self.subdag.dag_id)
-                .filter(TaskInstance.execution_date == execution_date)
-                .filter(TaskInstance.state.in_([State.FAILED, State.UPSTREAM_FAILED]))
-            )
-
-            for task_instance in failed_task_instances:
-                task_instance.state = State.NONE
-                session.merge(task_instance)
-            session.commit()
-
-    def pre_execute(self, context):
-        execution_date = context['execution_date']
-        dag_run = self._get_dagrun(execution_date)
-
-        if dag_run is None:
-            dag_run = self.subdag.create_dagrun(
-                run_type=DagRunType.SCHEDULED,
-                execution_date=execution_date,
-                state=State.RUNNING,
-                conf=self.conf,
-                external_trigger=True,
-            )
-            self.log.info("Created DagRun: %s", dag_run.run_id)
-        else:
-            self.log.info("Found existing DagRun: %s", dag_run.run_id)
-            if dag_run.state == State.FAILED:
-                self._reset_dag_run_and_task_instances(dag_run, execution_date)
-
-    def poke(self, context):
-        execution_date = context['execution_date']
-        dag_run = self._get_dagrun(execution_date=execution_date)
-        return dag_run.state != State.RUNNING
-
-    def post_execute(self, context, result=None):
-        execution_date = context['execution_date']
-        dag_run = self._get_dagrun(execution_date=execution_date)
-        self.log.info("Execution finished. State is %s", dag_run.state)
-
-        if dag_run.state != State.SUCCESS:
-            raise AirflowException(f"Expected state: SUCCESS. Actual state: {dag_run.state}")
-
-        if self.propagate_skipped_state and self._check_skipped_states(context):
-            self._skip_downstream_tasks(context)
-
-    def _check_skipped_states(self, context):
-        leaves_tis = self._get_leaves_tis(context['execution_date'])
-
-        if self.propagate_skipped_state == SkippedStatePropagationOptions.ANY_LEAF:
-            return any(ti.state == State.SKIPPED for ti in leaves_tis)
-        if self.propagate_skipped_state == SkippedStatePropagationOptions.ALL_LEAVES:
-            return all(ti.state == State.SKIPPED for ti in leaves_tis)
-        raise AirflowException(
-            f'Unimplemented SkippedStatePropagationOptions {self.propagate_skipped_state} used.'
-        )
-
-    def _get_leaves_tis(self, execution_date):
-        leaves_tis = []
-        for leaf in self.subdag.leaves:
-            try:
-                ti = get_task_instance(
-                    dag_id=self.subdag.dag_id, task_id=leaf.task_id, execution_date=execution_date
-                )
-                leaves_tis.append(ti)
-            except TaskInstanceNotFound:
-                continue
-        return leaves_tis
-
-    def _skip_downstream_tasks(self, context):
-        self.log.info(
-            'Skipping downstream tasks because propagate_skipped_state is set to %s '
-            'and skipped task(s) were found.',
-            self.propagate_skipped_state,
-        )
-
-        downstream_tasks = context['task'].downstream_list
-        self.log.debug('Downstream task_ids %s', downstream_tasks)
-
-        if downstream_tasks:
-            self.skip(context['dag_run'], context['execution_date'], downstream_tasks)
-
-        self.log.info('Done.')
+warnings.warn(
+    "This module is deprecated. Please use `airflow.operators.subdag`.", DeprecationWarning, stacklevel=2
+)
diff --git a/docs/apache-airflow/concepts.rst b/docs/apache-airflow/concepts.rst
index 30fec4d..d92ae81 100644
--- a/docs/apache-airflow/concepts.rst
+++ b/docs/apache-airflow/concepts.rst
@@ -82,7 +82,7 @@ scope.
     my_function()
 
 Sometimes this can be put to good use. For example, a common pattern with
-:class:`~airflow.operators.subdag_operator.SubDagOperator` is to define the subdag inside a function so that Airflow
+:class:`~airflow.operators.subdag.SubDagOperator` is to define the subdag inside a function so that Airflow
 doesn't try to load it as a standalone DAG.
 
 .. _default-args:
@@ -1024,7 +1024,7 @@ This SubDAG can then be referenced in your main DAG file:
     :start-after: [START example_subdag_operator]
     :end-before: [END example_subdag_operator]
 
-You can zoom into a :class:`~airflow.operators.subdag_operator.SubDagOperator` from the graph view of the main DAG to show
+You can zoom into a :class:`~airflow.operators.subdag.SubDagOperator` from the graph view of the main DAG to show
 the tasks contained within the SubDAG:
 
 .. image:: img/subdag_zoom.png
@@ -1038,8 +1038,8 @@ Some other tips when using SubDAGs:
 -  SubDAGs must have a schedule and be enabled. If the SubDAG's schedule is
    set to ``None`` or ``@once``, the SubDAG will succeed without having done
    anything
--  clearing a :class:`~airflow.operators.subdag_operator.SubDagOperator` also clears the state of the tasks within
--  marking success on a :class:`~airflow.operators.subdag_operator.SubDagOperator` does not affect the state of the tasks
+-  clearing a :class:`~airflow.operators.subdag.SubDagOperator` also clears the state of the tasks within
+-  marking success on a :class:`~airflow.operators.subdag.SubDagOperator` does not affect the state of the tasks
    within
 -  refrain from using ``depends_on_past=True`` in tasks within the SubDAG as
    this can be confusing
@@ -1051,7 +1051,7 @@ Some other tips when using SubDAGs:
 
 See ``airflow/example_dags`` for a demonstration.
 
-Note that airflow pool is not honored by :class:`~airflow.operators.subdag_operator.SubDagOperator`. Hence
+Note that airflow pool is not honored by :class:`~airflow.operators.subdag.SubDagOperator`. Hence
 resources could be consumed by SubdagOperators.
 
 
@@ -1059,7 +1059,7 @@ TaskGroup
 =========
 TaskGroup can be used to organize tasks into hierarchical groups in Graph View. It is
 useful for creating repeating patterns and cutting down visual clutter. Unlike
-:class:`~airflow.operators.subdag_operator.SubDagOperator`, TaskGroup is a UI grouping concept.
+:class:`~airflow.operators.subdag.SubDagOperator`, TaskGroup is a UI grouping concept.
 Tasks in TaskGroups live on the same original DAG. They honor all the pool configurations.
 
 Dependency relationships can be applied across all tasks in a TaskGroup with the ``>>`` and ``<<``
diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst
index fc10238..8439a47 100644
--- a/docs/apache-airflow/operators-and-hooks-ref.rst
+++ b/docs/apache-airflow/operators-and-hooks-ref.rst
@@ -74,7 +74,7 @@ Airflow has many more integrations available for separate installation as a prov
    * - :mod:`airflow.operators.python`
      - :doc:`How to use <howto/operator/python>`
 
-   * - :mod:`airflow.operators.subdag_operator`
+   * - :mod:`airflow.operators.subdag`
      -
 
    * - :mod:`airflow.operators.sql`
diff --git a/tests/dags/test_clear_subdag.py b/tests/dags/test_clear_subdag.py
index 4a59126..bc08187 100644
--- a/tests/dags/test_clear_subdag.py
+++ b/tests/dags/test_clear_subdag.py
@@ -21,7 +21,7 @@ import datetime
 
 from airflow.models import DAG
 from airflow.operators.bash import BashOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 
 
 def create_subdag_opt(main_dag):
diff --git a/tests/dags/test_impersonation_subdag.py b/tests/dags/test_impersonation_subdag.py
index 3ea5200..22432f2 100644
--- a/tests/dags/test_impersonation_subdag.py
+++ b/tests/dags/test_impersonation_subdag.py
@@ -21,7 +21,7 @@ from datetime import datetime
 from airflow.models import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.python import PythonOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 
 DEFAULT_DATE = datetime(2016, 1, 1)
 
diff --git a/tests/dags/test_subdag.py b/tests/dags/test_subdag.py
index cc06095..f161268 100644
--- a/tests/dags/test_subdag.py
+++ b/tests/dags/test_subdag.py
@@ -25,7 +25,7 @@ from datetime import datetime, timedelta
 
 from airflow.models.dag import DAG
 from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 
 DAG_NAME = 'test_subdag_operator'
 
diff --git a/tests/deprecated_classes.py b/tests/deprecated_classes.py
index f79ba77..7b9d7b1 100644
--- a/tests/deprecated_classes.py
+++ b/tests/deprecated_classes.py
@@ -1324,6 +1324,10 @@ OPERATORS = [
         "airflow.operators.latest_only.LatestOnlyOperator",
         "airflow.operators.latest_only_operator.LatestOnlyOperator",
     ),
+    (
+        "airflow.operators.subdag.SubDagOperator",
+        "airflow.operators.subdag_operator.SubDagOperator",
+    ),
 ]
 
 SECRETS = [
diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py
index 7b09ba8..53717d4 100644
--- a/tests/models/test_dag.py
+++ b/tests/models/test_dag.py
@@ -46,7 +46,7 @@ from airflow.models.dagparam import DagParam
 from airflow.operators.bash import BashOperator
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python import task as task_decorator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
 from airflow.security import permissions
 from airflow.utils import timezone
 from airflow.utils.file import list_py_file_paths
diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py
index 1151478..c7e3728 100644
--- a/tests/models/test_dagbag.py
+++ b/tests/models/test_dagbag.py
@@ -320,7 +320,7 @@ class TestDagBag(unittest.TestCase):
 
             from airflow.models import DAG
             from airflow.operators.dummy_operator import DummyOperator
-            from airflow.operators.subdag_operator import SubDagOperator
+            from airflow.operators.subdag import SubDagOperator
 
             dag_name = 'master'
             default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
@@ -371,7 +371,7 @@ class TestDagBag(unittest.TestCase):
 
             from airflow.models import DAG
             from airflow.operators.dummy_operator import DummyOperator
-            from airflow.operators.subdag_operator import SubDagOperator
+            from airflow.operators.subdag import SubDagOperator
 
             dag_name = 'master'
             default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
@@ -491,7 +491,7 @@ class TestDagBag(unittest.TestCase):
 
             from airflow.models import DAG
             from airflow.operators.dummy_operator import DummyOperator
-            from airflow.operators.subdag_operator import SubDagOperator
+            from airflow.operators.subdag import SubDagOperator
 
             dag_name = 'nested_cycle'
             default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
diff --git a/tests/operators/test_subdag_operator.py b/tests/operators/test_subdag_operator.py
index bd2b578..fe59078 100644
--- a/tests/operators/test_subdag_operator.py
+++ b/tests/operators/test_subdag_operator.py
@@ -26,7 +26,7 @@ import airflow
 from airflow.exceptions import AirflowException
 from airflow.models import DAG, DagRun, TaskInstance
 from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SkippedStatePropagationOptions, SubDagOperator
+from airflow.operators.subdag import SkippedStatePropagationOptions, SubDagOperator
 from airflow.utils.session import create_session
 from airflow.utils.state import State
 from airflow.utils.timezone import datetime
@@ -270,8 +270,8 @@ class TestSubDagOperator(unittest.TestCase):
             (None, [State.SKIPPED, State.SKIPPED], False),
         ]
     )
-    @mock.patch('airflow.operators.subdag_operator.SubDagOperator.skip')
-    @mock.patch('airflow.operators.subdag_operator.get_task_instance')
+    @mock.patch('airflow.operators.subdag.SubDagOperator.skip')
+    @mock.patch('airflow.operators.subdag.get_task_instance')
     def test_subdag_with_propagate_skipped_state(
         self, propagate_option, states, skip_parent, mock_get_task_instance, mock_skip
     ):