You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/12/08 16:27:46 UTC
[airflow] branch master updated: Moved subdag_operator.py to
subdag.py (#11307)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new bfbd4bb Moved subdag_operator.py to subdag.py (#11307)
bfbd4bb is described below
commit bfbd4bbb706d8c358f310c98470a613090d709ad
Author: Kishore Vancheeshwaran <24...@users.noreply.github.com>
AuthorDate: Tue Dec 8 21:56:39 2020 +0530
Moved subdag_operator.py to subdag.py (#11307)
Part of #11178
---
airflow/api/common/experimental/mark_tasks.py | 2 +-
airflow/example_dags/example_subdag_operator.py | 2 +-
airflow/executors/celery_executor.py | 2 +-
airflow/models/dag.py | 4 +-
.../operators/{subdag_operator.py => subdag.py} | 5 +-
airflow/operators/subdag_operator.py | 203 +--------------------
docs/apache-airflow/concepts.rst | 12 +-
docs/apache-airflow/operators-and-hooks-ref.rst | 2 +-
tests/dags/test_clear_subdag.py | 2 +-
tests/dags/test_impersonation_subdag.py | 2 +-
tests/dags/test_subdag.py | 2 +-
tests/deprecated_classes.py | 4 +
tests/models/test_dag.py | 2 +-
tests/models/test_dagbag.py | 6 +-
tests/operators/test_subdag_operator.py | 6 +-
15 files changed, 35 insertions(+), 221 deletions(-)
diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py
index 852038d..04215dc 100644
--- a/airflow/api/common/experimental/mark_tasks.py
+++ b/airflow/api/common/experimental/mark_tasks.py
@@ -25,7 +25,7 @@ from sqlalchemy import or_
from airflow.models.baseoperator import BaseOperator
from airflow.models.dagrun import DagRun
from airflow.models.taskinstance import TaskInstance
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
from airflow.utils import timezone
from airflow.utils.session import provide_session
from airflow.utils.state import State
diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py
index de2853d..6e38b60 100644
--- a/airflow/example_dags/example_subdag_operator.py
+++ b/airflow/example_dags/example_subdag_operator.py
@@ -22,7 +22,7 @@
from airflow import DAG
from airflow.example_dags.subdags.subdag import subdag
from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
from airflow.utils.dates import days_ago
DAG_NAME = 'example_subdag_operator'
diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py
index 9fb9f63..f4f3412 100644
--- a/airflow/executors/celery_executor.py
+++ b/airflow/executors/celery_executor.py
@@ -184,7 +184,7 @@ def on_celery_import_modules(*args, **kwargs):
import airflow.macros
import airflow.operators.bash
import airflow.operators.python
- import airflow.operators.subdag_operator # noqa: F401
+ import airflow.operators.subdag # noqa: F401
try:
import kubernetes.client # noqa: F401
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index 4f9333e..be6403c 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -952,7 +952,7 @@ class DAG(LoggingMixin):
def subdags(self):
"""Returns a list of the subdag objects associated to this DAG"""
# Check SubDag for class but don't check class directly
- from airflow.operators.subdag_operator import SubDagOperator
+ from airflow.operators.subdag import SubDagOperator
subdag_lst = []
for task in self.tasks:
@@ -1061,7 +1061,7 @@ class DAG(LoggingMixin):
:param include_subdag_tasks: whether to include tasks in subdags, default to False
:return: list of tasks in topological order
"""
- from airflow.operators.subdag_operator import SubDagOperator # Avoid circular import
+ from airflow.operators.subdag import SubDagOperator # Avoid circular import
# convert into an OrderedDict to speedup lookup while keeping order the same
graph_unsorted = OrderedDict((task.task_id, task) for task in self.tasks)
diff --git a/airflow/operators/subdag_operator.py b/airflow/operators/subdag.py
similarity index 99%
copy from airflow/operators/subdag_operator.py
copy to airflow/operators/subdag.py
index f5c147e..538d7cc 100644
--- a/airflow/operators/subdag_operator.py
+++ b/airflow/operators/subdag.py
@@ -45,7 +45,6 @@ class SubDagOperator(BaseSensorOperator):
"""
This runs a sub dag. By convention, a sub dag's dag_id
should be prefixed by its parent and a dot. As in `parent.child`.
-
Although SubDagOperator can occupy a pool/concurrency slot,
user can specify the mode=reschedule so that the slot will be
released periodically to avoid potential deadlock.
@@ -55,7 +54,8 @@ class SubDagOperator(BaseSensorOperator):
:param conf: Configuration for the subdag
:type conf: dict
:param propagate_skipped_state: by setting this argument you can define
- whether the skipped state of leaf task(s) should be propagated to the parent dag's downstream task.
+ whether the skipped state of leaf task(s) should be propagated to the
+ parent dag's downstream task.
"""
ui_color = '#555'
@@ -123,7 +123,6 @@ class SubDagOperator(BaseSensorOperator):
"""
Set the DagRun state to RUNNING and set the failed TaskInstances to None state
for scheduler to pick up.
-
:param dag_run: DAG run
:param execution_date: Execution date
:return: None
diff --git a/airflow/operators/subdag_operator.py b/airflow/operators/subdag_operator.py
index f5c147e..de6c607 100644
--- a/airflow/operators/subdag_operator.py
+++ b/airflow/operators/subdag_operator.py
@@ -15,202 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""The module which provides a way to nest your DAGs and so your levels of complexity."""
-from enum import Enum
-from typing import Dict, Optional
+"""This module is deprecated. Please use `airflow.operators.subdag`."""
-from sqlalchemy.orm.session import Session
+import warnings
-from airflow.api.common.experimental.get_task_instance import get_task_instance
-from airflow.exceptions import AirflowException, TaskInstanceNotFound
-from airflow.models import DagRun
-from airflow.models.dag import DAG, DagContext
-from airflow.models.pool import Pool
-from airflow.models.taskinstance import TaskInstance
-from airflow.sensors.base_sensor_operator import BaseSensorOperator
-from airflow.utils.decorators import apply_defaults
-from airflow.utils.session import create_session, provide_session
-from airflow.utils.state import State
-from airflow.utils.types import DagRunType
+# pylint: disable=unused-import
+from airflow.operators.subdag import SkippedStatePropagationOptions, SubDagOperator # noqa
-
-class SkippedStatePropagationOptions(Enum):
- """Available options for skipped state propagation of subdag's tasks to parent dag tasks."""
-
- ALL_LEAVES = 'all_leaves'
- ANY_LEAF = 'any_leaf'
-
-
-class SubDagOperator(BaseSensorOperator):
- """
- This runs a sub dag. By convention, a sub dag's dag_id
- should be prefixed by its parent and a dot. As in `parent.child`.
-
- Although SubDagOperator can occupy a pool/concurrency slot,
- user can specify the mode=reschedule so that the slot will be
- released periodically to avoid potential deadlock.
-
- :param subdag: the DAG object to run as a subdag of the current DAG.
- :param session: sqlalchemy session
- :param conf: Configuration for the subdag
- :type conf: dict
- :param propagate_skipped_state: by setting this argument you can define
- whether the skipped state of leaf task(s) should be propagated to the parent dag's downstream task.
- """
-
- ui_color = '#555'
- ui_fgcolor = '#fff'
-
- @provide_session
- @apply_defaults
- def __init__(
- self,
- *,
- subdag: DAG,
- session: Optional[Session] = None,
- conf: Optional[Dict] = None,
- propagate_skipped_state: Optional[SkippedStatePropagationOptions] = None,
- **kwargs,
- ) -> None:
- super().__init__(**kwargs)
- self.subdag = subdag
- self.conf = conf
- self.propagate_skipped_state = propagate_skipped_state
-
- self._validate_dag(kwargs)
- self._validate_pool(session)
-
- def _validate_dag(self, kwargs):
- dag = kwargs.get('dag') or DagContext.get_current_dag()
-
- if not dag:
- raise AirflowException('Please pass in the `dag` param or call within a DAG context manager')
-
- if dag.dag_id + '.' + kwargs['task_id'] != self.subdag.dag_id:
- raise AirflowException(
- "The subdag's dag_id should have the form '{{parent_dag_id}}.{{this_task_id}}'. "
- "Expected '{d}.{t}'; received '{rcvd}'.".format(
- d=dag.dag_id, t=kwargs['task_id'], rcvd=self.subdag.dag_id
- )
- )
-
- def _validate_pool(self, session):
- if self.pool:
- conflicts = [t for t in self.subdag.tasks if t.pool == self.pool]
- if conflicts:
- # only query for pool conflicts if one may exist
- pool = session.query(Pool).filter(Pool.slots == 1).filter(Pool.pool == self.pool).first()
- if pool and any(t.pool == self.pool for t in self.subdag.tasks):
- raise AirflowException(
- 'SubDagOperator {sd} and subdag task{plural} {t} both '
- 'use pool {p}, but the pool only has 1 slot. The '
- 'subdag tasks will never run.'.format(
- sd=self.task_id,
- plural=len(conflicts) > 1,
- t=', '.join(t.task_id for t in conflicts),
- p=self.pool,
- )
- )
-
- def _get_dagrun(self, execution_date):
- dag_runs = DagRun.find(
- dag_id=self.subdag.dag_id,
- execution_date=execution_date,
- )
- return dag_runs[0] if dag_runs else None
-
- def _reset_dag_run_and_task_instances(self, dag_run, execution_date):
- """
- Set the DagRun state to RUNNING and set the failed TaskInstances to None state
- for scheduler to pick up.
-
- :param dag_run: DAG run
- :param execution_date: Execution date
- :return: None
- """
- with create_session() as session:
- dag_run.state = State.RUNNING
- session.merge(dag_run)
- failed_task_instances = (
- session.query(TaskInstance)
- .filter(TaskInstance.dag_id == self.subdag.dag_id)
- .filter(TaskInstance.execution_date == execution_date)
- .filter(TaskInstance.state.in_([State.FAILED, State.UPSTREAM_FAILED]))
- )
-
- for task_instance in failed_task_instances:
- task_instance.state = State.NONE
- session.merge(task_instance)
- session.commit()
-
- def pre_execute(self, context):
- execution_date = context['execution_date']
- dag_run = self._get_dagrun(execution_date)
-
- if dag_run is None:
- dag_run = self.subdag.create_dagrun(
- run_type=DagRunType.SCHEDULED,
- execution_date=execution_date,
- state=State.RUNNING,
- conf=self.conf,
- external_trigger=True,
- )
- self.log.info("Created DagRun: %s", dag_run.run_id)
- else:
- self.log.info("Found existing DagRun: %s", dag_run.run_id)
- if dag_run.state == State.FAILED:
- self._reset_dag_run_and_task_instances(dag_run, execution_date)
-
- def poke(self, context):
- execution_date = context['execution_date']
- dag_run = self._get_dagrun(execution_date=execution_date)
- return dag_run.state != State.RUNNING
-
- def post_execute(self, context, result=None):
- execution_date = context['execution_date']
- dag_run = self._get_dagrun(execution_date=execution_date)
- self.log.info("Execution finished. State is %s", dag_run.state)
-
- if dag_run.state != State.SUCCESS:
- raise AirflowException(f"Expected state: SUCCESS. Actual state: {dag_run.state}")
-
- if self.propagate_skipped_state and self._check_skipped_states(context):
- self._skip_downstream_tasks(context)
-
- def _check_skipped_states(self, context):
- leaves_tis = self._get_leaves_tis(context['execution_date'])
-
- if self.propagate_skipped_state == SkippedStatePropagationOptions.ANY_LEAF:
- return any(ti.state == State.SKIPPED for ti in leaves_tis)
- if self.propagate_skipped_state == SkippedStatePropagationOptions.ALL_LEAVES:
- return all(ti.state == State.SKIPPED for ti in leaves_tis)
- raise AirflowException(
- f'Unimplemented SkippedStatePropagationOptions {self.propagate_skipped_state} used.'
- )
-
- def _get_leaves_tis(self, execution_date):
- leaves_tis = []
- for leaf in self.subdag.leaves:
- try:
- ti = get_task_instance(
- dag_id=self.subdag.dag_id, task_id=leaf.task_id, execution_date=execution_date
- )
- leaves_tis.append(ti)
- except TaskInstanceNotFound:
- continue
- return leaves_tis
-
- def _skip_downstream_tasks(self, context):
- self.log.info(
- 'Skipping downstream tasks because propagate_skipped_state is set to %s '
- 'and skipped task(s) were found.',
- self.propagate_skipped_state,
- )
-
- downstream_tasks = context['task'].downstream_list
- self.log.debug('Downstream task_ids %s', downstream_tasks)
-
- if downstream_tasks:
- self.skip(context['dag_run'], context['execution_date'], downstream_tasks)
-
- self.log.info('Done.')
+warnings.warn(
+ "This module is deprecated. Please use `airflow.operators.subdag`.", DeprecationWarning, stacklevel=2
+)
diff --git a/docs/apache-airflow/concepts.rst b/docs/apache-airflow/concepts.rst
index 30fec4d..d92ae81 100644
--- a/docs/apache-airflow/concepts.rst
+++ b/docs/apache-airflow/concepts.rst
@@ -82,7 +82,7 @@ scope.
my_function()
Sometimes this can be put to good use. For example, a common pattern with
-:class:`~airflow.operators.subdag_operator.SubDagOperator` is to define the subdag inside a function so that Airflow
+:class:`~airflow.operators.subdag.SubDagOperator` is to define the subdag inside a function so that Airflow
doesn't try to load it as a standalone DAG.
.. _default-args:
@@ -1024,7 +1024,7 @@ This SubDAG can then be referenced in your main DAG file:
:start-after: [START example_subdag_operator]
:end-before: [END example_subdag_operator]
-You can zoom into a :class:`~airflow.operators.subdag_operator.SubDagOperator` from the graph view of the main DAG to show
+You can zoom into a :class:`~airflow.operators.subdag.SubDagOperator` from the graph view of the main DAG to show
the tasks contained within the SubDAG:
.. image:: img/subdag_zoom.png
@@ -1038,8 +1038,8 @@ Some other tips when using SubDAGs:
- SubDAGs must have a schedule and be enabled. If the SubDAG's schedule is
set to ``None`` or ``@once``, the SubDAG will succeed without having done
anything
-- clearing a :class:`~airflow.operators.subdag_operator.SubDagOperator` also clears the state of the tasks within
-- marking success on a :class:`~airflow.operators.subdag_operator.SubDagOperator` does not affect the state of the tasks
+- clearing a :class:`~airflow.operators.subdag.SubDagOperator` also clears the state of the tasks within
+- marking success on a :class:`~airflow.operators.subdag.SubDagOperator` does not affect the state of the tasks
within
- refrain from using ``depends_on_past=True`` in tasks within the SubDAG as
this can be confusing
@@ -1051,7 +1051,7 @@ Some other tips when using SubDAGs:
See ``airflow/example_dags`` for a demonstration.
-Note that airflow pool is not honored by :class:`~airflow.operators.subdag_operator.SubDagOperator`. Hence
+Note that airflow pool is not honored by :class:`~airflow.operators.subdag.SubDagOperator`. Hence
resources could be consumed by SubdagOperators.
@@ -1059,7 +1059,7 @@ TaskGroup
=========
TaskGroup can be used to organize tasks into hierarchical groups in Graph View. It is
useful for creating repeating patterns and cutting down visual clutter. Unlike
-:class:`~airflow.operators.subdag_operator.SubDagOperator`, TaskGroup is a UI grouping concept.
+:class:`~airflow.operators.subdag.SubDagOperator`, TaskGroup is a UI grouping concept.
Tasks in TaskGroups live on the same original DAG. They honor all the pool configurations.
Dependency relationships can be applied across all tasks in a TaskGroup with the ``>>`` and ``<<``
diff --git a/docs/apache-airflow/operators-and-hooks-ref.rst b/docs/apache-airflow/operators-and-hooks-ref.rst
index fc10238..8439a47 100644
--- a/docs/apache-airflow/operators-and-hooks-ref.rst
+++ b/docs/apache-airflow/operators-and-hooks-ref.rst
@@ -74,7 +74,7 @@ Airflow has many more integrations available for separate installation as a prov
* - :mod:`airflow.operators.python`
- :doc:`How to use <howto/operator/python>`
- * - :mod:`airflow.operators.subdag_operator`
+ * - :mod:`airflow.operators.subdag`
-
* - :mod:`airflow.operators.sql`
diff --git a/tests/dags/test_clear_subdag.py b/tests/dags/test_clear_subdag.py
index 4a59126..bc08187 100644
--- a/tests/dags/test_clear_subdag.py
+++ b/tests/dags/test_clear_subdag.py
@@ -21,7 +21,7 @@ import datetime
from airflow.models import DAG
from airflow.operators.bash import BashOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
def create_subdag_opt(main_dag):
diff --git a/tests/dags/test_impersonation_subdag.py b/tests/dags/test_impersonation_subdag.py
index 3ea5200..22432f2 100644
--- a/tests/dags/test_impersonation_subdag.py
+++ b/tests/dags/test_impersonation_subdag.py
@@ -21,7 +21,7 @@ from datetime import datetime
from airflow.models import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
DEFAULT_DATE = datetime(2016, 1, 1)
diff --git a/tests/dags/test_subdag.py b/tests/dags/test_subdag.py
index cc06095..f161268 100644
--- a/tests/dags/test_subdag.py
+++ b/tests/dags/test_subdag.py
@@ -25,7 +25,7 @@ from datetime import datetime, timedelta
from airflow.models.dag import DAG
from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
DAG_NAME = 'test_subdag_operator'
diff --git a/tests/deprecated_classes.py b/tests/deprecated_classes.py
index f79ba77..7b9d7b1 100644
--- a/tests/deprecated_classes.py
+++ b/tests/deprecated_classes.py
@@ -1324,6 +1324,10 @@ OPERATORS = [
"airflow.operators.latest_only.LatestOnlyOperator",
"airflow.operators.latest_only_operator.LatestOnlyOperator",
),
+ (
+ "airflow.operators.subdag.SubDagOperator",
+ "airflow.operators.subdag_operator.SubDagOperator",
+ ),
]
SECRETS = [
diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py
index 7b09ba8..53717d4 100644
--- a/tests/models/test_dag.py
+++ b/tests/models/test_dag.py
@@ -46,7 +46,7 @@ from airflow.models.dagparam import DagParam
from airflow.operators.bash import BashOperator
from airflow.operators.dummy_operator import DummyOperator
from airflow.operators.python import task as task_decorator
-from airflow.operators.subdag_operator import SubDagOperator
+from airflow.operators.subdag import SubDagOperator
from airflow.security import permissions
from airflow.utils import timezone
from airflow.utils.file import list_py_file_paths
diff --git a/tests/models/test_dagbag.py b/tests/models/test_dagbag.py
index 1151478..c7e3728 100644
--- a/tests/models/test_dagbag.py
+++ b/tests/models/test_dagbag.py
@@ -320,7 +320,7 @@ class TestDagBag(unittest.TestCase):
from airflow.models import DAG
from airflow.operators.dummy_operator import DummyOperator
- from airflow.operators.subdag_operator import SubDagOperator
+ from airflow.operators.subdag import SubDagOperator
dag_name = 'master'
default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
@@ -371,7 +371,7 @@ class TestDagBag(unittest.TestCase):
from airflow.models import DAG
from airflow.operators.dummy_operator import DummyOperator
- from airflow.operators.subdag_operator import SubDagOperator
+ from airflow.operators.subdag import SubDagOperator
dag_name = 'master'
default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
@@ -491,7 +491,7 @@ class TestDagBag(unittest.TestCase):
from airflow.models import DAG
from airflow.operators.dummy_operator import DummyOperator
- from airflow.operators.subdag_operator import SubDagOperator
+ from airflow.operators.subdag import SubDagOperator
dag_name = 'nested_cycle'
default_args = {'owner': 'owner1', 'start_date': datetime.datetime(2016, 1, 1)}
diff --git a/tests/operators/test_subdag_operator.py b/tests/operators/test_subdag_operator.py
index bd2b578..fe59078 100644
--- a/tests/operators/test_subdag_operator.py
+++ b/tests/operators/test_subdag_operator.py
@@ -26,7 +26,7 @@ import airflow
from airflow.exceptions import AirflowException
from airflow.models import DAG, DagRun, TaskInstance
from airflow.operators.dummy_operator import DummyOperator
-from airflow.operators.subdag_operator import SkippedStatePropagationOptions, SubDagOperator
+from airflow.operators.subdag import SkippedStatePropagationOptions, SubDagOperator
from airflow.utils.session import create_session
from airflow.utils.state import State
from airflow.utils.timezone import datetime
@@ -270,8 +270,8 @@ class TestSubDagOperator(unittest.TestCase):
(None, [State.SKIPPED, State.SKIPPED], False),
]
)
- @mock.patch('airflow.operators.subdag_operator.SubDagOperator.skip')
- @mock.patch('airflow.operators.subdag_operator.get_task_instance')
+ @mock.patch('airflow.operators.subdag.SubDagOperator.skip')
+ @mock.patch('airflow.operators.subdag.get_task_instance')
def test_subdag_with_propagate_skipped_state(
self, propagate_option, states, skip_parent, mock_get_task_instance, mock_skip
):