You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2022/09/23 17:25:51 UTC
[airflow] 02/02: Fix Deferrable stuck as "scheduled" during backfill (#26205)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 53f0ffd7676c0c0a8c351568b534c4da93d6e817
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Fri Sep 23 06:08:28 2022 +0100
Fix Deferrable stuck as "scheduled" during backfill (#26205)
Co-authored-by: Tzu-ping Chung <ur...@gmail.com>
(cherry picked from commit 3396d1f822caac7cbeb14e1e67679b8378a84a6c)
---
airflow/jobs/backfill_job.py | 6 ++++++
tests/jobs/test_backfill_job.py | 24 ++++++++++++++++++++++++
2 files changed, 30 insertions(+)
diff --git a/airflow/jobs/backfill_job.py b/airflow/jobs/backfill_job.py
index 4a2d3e7bd4..62bc8c2bb3 100644
--- a/airflow/jobs/backfill_job.py
+++ b/airflow/jobs/backfill_job.py
@@ -217,6 +217,12 @@ class BackfillJob(BaseJob):
tis_to_be_scheduled.append(ti)
ti_status.running.pop(reduced_key)
ti_status.to_run[ti.key] = ti
+ # special case: Deferrable task can go from DEFERRED to SCHEDULED;
+ # when that happens, we need to put it back as in UP_FOR_RESCHEDULE
+ elif ti.state == TaskInstanceState.SCHEDULED:
+ self.log.debug("Task instance %s is resumed from deferred state", ti)
+ ti_status.running.pop(ti.key)
+ ti_status.to_run[ti.key] = ti
# Batch schedule of task instances
if tis_to_be_scheduled:
diff --git a/tests/jobs/test_backfill_job.py b/tests/jobs/test_backfill_job.py
index 31ea238f5e..18fa630835 100644
--- a/tests/jobs/test_backfill_job.py
+++ b/tests/jobs/test_backfill_job.py
@@ -1415,6 +1415,30 @@ class TestBackfillJob:
ti_status.to_run.clear()
+ # test for scheduled
+ ti.set_state(State.SCHEDULED)
+ # Deferred tasks are put into scheduled by the triggerer
+ # Check that they are put into to_run
+ ti_status.running[ti.key] = ti
+ job._update_counters(ti_status=ti_status, session=session)
+ assert len(ti_status.running) == 0
+ assert len(ti_status.succeeded) == 0
+ assert len(ti_status.skipped) == 0
+ assert len(ti_status.failed) == 0
+ assert len(ti_status.to_run) == 1
+
+ ti_status.to_run.clear()
+ # test for deferred
+ # if a task is deferred and it's not yet time for the triggerer
+ # to reschedule it, we should leave it in ti_status.running
+ ti.set_state(State.DEFERRED)
+ ti_status.running[ti.key] = ti
+ job._update_counters(ti_status=ti_status, session=session)
+ assert len(ti_status.running) == 1
+ assert len(ti_status.succeeded) == 0
+ assert len(ti_status.skipped) == 0
+ assert len(ti_status.failed) == 0
+ assert len(ti_status.to_run) == 0
session.close()
def test_dag_dagrun_infos_between(self, dag_maker):