You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ds...@apache.org on 2024/10/16 15:07:20 UTC

(airflow) branch main updated: Remove DAG.get_num_active_runs (#43067)

This is an automated email from the ASF dual-hosted git repository.

dstandish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 598e0ee8d71 Remove DAG.get_num_active_runs (#43067)
598e0ee8d71 is described below

commit 598e0ee8d71f73133a187651be92cd482d4dcbd5
Author: Daniel Standish <15...@users.noreply.github.com>
AuthorDate: Wed Oct 16 08:07:13 2024 -0700

    Remove DAG.get_num_active_runs (#43067)
    
    We don't need this function.  There's already an almost-identical function on DagRun that we can use, namely DagRun.active_runs_of_dags.
    
    Also simplified that one.
---
 airflow/jobs/scheduler_job_runner.py | 13 ++++++-------
 airflow/models/dag.py                | 22 ----------------------
 airflow/models/dagrun.py             | 25 ++++++++++++-------------
 newsfragments/43067.significant.rst  |  4 ++++
 tests/jobs/test_scheduler_job.py     |  4 ++--
 5 files changed, 24 insertions(+), 44 deletions(-)

diff --git a/airflow/jobs/scheduler_job_runner.py b/airflow/jobs/scheduler_job_runner.py
index 7958adb64cf..f583cb414ec 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -1337,9 +1337,8 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
             .all()
         )
 
-        active_runs_of_dags = Counter(
-            DagRun.active_runs_of_dags(dag_ids=(dm.dag_id for dm in dag_models), session=session),
-        )
+        dag_ids = (dm.dag_id for dm in dag_models)
+        active_runs_of_dags = Counter(DagRun.active_runs_of_dags(dag_ids=dag_ids, session=session))
 
         for dag_model in dag_models:
             dag = self.dagbag.get_dag(dag_model.dag_id, session=session)
@@ -1512,11 +1511,11 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
         if not dag.timetable.can_be_scheduled:
             return False
 
-        # get active dag runs from DB if not available
-        if not total_active_runs:
-            total_active_runs = dag.get_num_active_runs(only_running=False, session=session)
+        if total_active_runs is None:
+            runs_dict = DagRun.active_runs_of_dags(dag_ids=[dag.dag_id], session=session)
+            total_active_runs = runs_dict.get(dag.dag_id, 0)
 
-        if total_active_runs and total_active_runs >= dag.max_active_runs:
+        if total_active_runs >= dag.max_active_runs:
             self.log.info(
                 "DAG %s is at (or above) max_active_runs (%d of %d), not creating any more runs",
                 dag_model.dag_id,
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index f8d9f55e56e..f0a7d7f56be 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -1283,28 +1283,6 @@ class DAG(LoggingMixin):
 
         return active_dates
 
-    @provide_session
-    def get_num_active_runs(self, external_trigger=None, only_running=True, session=NEW_SESSION):
-        """
-        Return the number of active "running" dag runs.
-
-        :param external_trigger: True for externally triggered active dag runs
-        :param session:
-        :return: number greater than 0 for active dag runs
-        """
-        query = select(func.count()).where(DagRun.dag_id == self.dag_id)
-        if only_running:
-            query = query.where(DagRun.state == DagRunState.RUNNING)
-        else:
-            query = query.where(DagRun.state.in_({DagRunState.RUNNING, DagRunState.QUEUED}))
-
-        if external_trigger is not None:
-            query = query.where(
-                DagRun.external_trigger == (expression.true() if external_trigger else expression.false())
-            )
-
-        return session.scalar(query)
-
     @staticmethod
     @internal_api_call
     @provide_session
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
index cad82e72b8b..4fd689d616b 100644
--- a/airflow/models/dagrun.py
+++ b/airflow/models/dagrun.py
@@ -386,21 +386,20 @@ class DagRun(Base, LoggingMixin):
     @provide_session
     def active_runs_of_dags(
         cls,
-        dag_ids: Iterable[str] | None = None,
-        only_running: bool = False,
+        dag_ids: Iterable[str],
         session: Session = NEW_SESSION,
     ) -> dict[str, int]:
-        """Get the number of active dag runs for each dag."""
-        query = select(cls.dag_id, func.count("*"))
-        if dag_ids is not None:
-            # 'set' called to avoid duplicate dag_ids, but converted back to 'list'
-            # because SQLAlchemy doesn't accept a set here.
-            query = query.where(cls.dag_id.in_(set(dag_ids)))
-        if only_running:
-            query = query.where(cls.state == DagRunState.RUNNING)
-        else:
-            query = query.where(cls.state.in_((DagRunState.RUNNING, DagRunState.QUEUED)))
-        query = query.group_by(cls.dag_id)
+        """
+        Get the number of active dag runs for each dag.
+
+        :meta private:
+        """
+        query = (
+            select(cls.dag_id, func.count("*"))
+            .where(cls.dag_id.in_(set(dag_ids)))
+            .where(cls.state.in_((DagRunState.RUNNING, DagRunState.QUEUED)))
+            .group_by(cls.dag_id)
+        )
         return dict(iter(session.execute(query)))
 
     @classmethod
diff --git a/newsfragments/43067.significant.rst b/newsfragments/43067.significant.rst
new file mode 100644
index 00000000000..d57fec6be9e
--- /dev/null
+++ b/newsfragments/43067.significant.rst
@@ -0,0 +1,4 @@
+Remove DAG.get_num_active_runs
+
+We don't need this function. There's already an almost-identical function on DagRun that we can use, namely DagRun.active_runs_of_dags.
+Also, make DagRun.active_runs_of_dags private.
diff --git a/tests/jobs/test_scheduler_job.py b/tests/jobs/test_scheduler_job.py
index 01d1c5fe7a3..13ccfc74a0f 100644
--- a/tests/jobs/test_scheduler_job.py
+++ b/tests/jobs/test_scheduler_job.py
@@ -4477,14 +4477,14 @@ class TestSchedulerJob:
         model: DagModel = session.get(DagModel, dag.dag_id)
 
         # Pre-condition
-        assert DagRun.active_runs_of_dags(session=session) == {"test_dag": 3}
+        assert DagRun.active_runs_of_dags(dag_ids=["test_dag"], session=session) == {"test_dag": 3}
 
         assert model.next_dagrun == timezone.DateTime(2016, 1, 3, tzinfo=UTC)
         assert model.next_dagrun_create_after is None
 
         complete_one_dagrun()
 
-        assert DagRun.active_runs_of_dags(session=session) == {"test_dag": 3}
+        assert DagRun.active_runs_of_dags(dag_ids=["test_dag"], session=session) == {"test_dag": 3}
 
         for _ in range(5):
             self.job_runner._do_scheduling(session)