You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2022/09/23 17:25:51 UTC

[airflow] 02/02: Fix Deferrable stuck as "scheduled" during backfill (#26205)

This is an automated email from the ASF dual-hosted git repository.

jedcunningham pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 53f0ffd7676c0c0a8c351568b534c4da93d6e817
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Fri Sep 23 06:08:28 2022 +0100

    Fix Deferrable stuck as "scheduled" during backfill (#26205)
    
    Co-authored-by: Tzu-ping Chung <ur...@gmail.com>
    (cherry picked from commit 3396d1f822caac7cbeb14e1e67679b8378a84a6c)
---
 airflow/jobs/backfill_job.py    |  6 ++++++
 tests/jobs/test_backfill_job.py | 24 ++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/airflow/jobs/backfill_job.py b/airflow/jobs/backfill_job.py
index 4a2d3e7bd4..62bc8c2bb3 100644
--- a/airflow/jobs/backfill_job.py
+++ b/airflow/jobs/backfill_job.py
@@ -217,6 +217,12 @@ class BackfillJob(BaseJob):
                 tis_to_be_scheduled.append(ti)
                 ti_status.running.pop(reduced_key)
                 ti_status.to_run[ti.key] = ti
+            # special case: Deferrable task can go from DEFERRED to SCHEDULED;
+            # when that happens, we need to put it back as in UP_FOR_RESCHEDULE
+            elif ti.state == TaskInstanceState.SCHEDULED:
+                self.log.debug("Task instance %s is resumed from deferred state", ti)
+                ti_status.running.pop(ti.key)
+                ti_status.to_run[ti.key] = ti
 
         # Batch schedule of task instances
         if tis_to_be_scheduled:
diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py
index 31ea238f5e..18fa630835 100644
--- a/tests/jobs/test_backfill_job.py
+++ b/tests/jobs/test_backfill_job.py
@@ -1415,6 +1415,30 @@ class TestBackfillJob:
 
         ti_status.to_run.clear()
 
+        # test for scheduled
+        ti.set_state(State.SCHEDULED)
+        # Deferred tasks are put into scheduled by the triggerer
+        # Check that they are put into to_run
+        ti_status.running[ti.key] = ti
+        job._update_counters(ti_status=ti_status, session=session)
+        assert len(ti_status.running) == 0
+        assert len(ti_status.succeeded) == 0
+        assert len(ti_status.skipped) == 0
+        assert len(ti_status.failed) == 0
+        assert len(ti_status.to_run) == 1
+
+        ti_status.to_run.clear()
+        # test for deferred
+        # if a task is deferred and it's not yet time for the triggerer
+        # to reschedule it, we should leave it in ti_status.running
+        ti.set_state(State.DEFERRED)
+        ti_status.running[ti.key] = ti
+        job._update_counters(ti_status=ti_status, session=session)
+        assert len(ti_status.running) == 1
+        assert len(ti_status.succeeded) == 0
+        assert len(ti_status.skipped) == 0
+        assert len(ti_status.failed) == 0
+        assert len(ti_status.to_run) == 0
         session.close()
 
     def test_dag_dagrun_infos_between(self, dag_maker):