You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/05/19 11:59:17 UTC
[airflow] branch v2-3-test updated (10675af30b -> d07392fe15)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 10675af30b Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674)
new 0ddc90f120 Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674)
new 99202020fc Fix task log is not captured (#23684)
new 27762afd9f Fix auto upstream dep when expanding non-templated field (#23771)
new d07392fe15 Fix dag-processor fetch metabase config (#23575)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (10675af30b)
\
N -- N -- N refs/heads/v2-3-test (d07392fe15)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/cli/commands/dag_processor_command.py | 2 +-
airflow/models/mappedoperator.py | 3 +--
airflow/models/taskinstance.py | 6 ++++--
tests/cli/commands/test_task_command.py | 2 +-
tests/models/test_taskinstance.py | 26 ++++++++++++++++++++++++++
5 files changed, 33 insertions(+), 6 deletions(-)
[airflow] 01/04: Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0ddc90f12020cd7197189a80ea56a7ec0a81273b
Author: David Caron <dc...@gmail.com>
AuthorDate: Tue May 17 08:18:29 2022 -0400
Add 'reschedule' to the serialized fields for the BaseSensorOperator (#23674)
fix #23411
(cherry picked from commit f9e2a3051cd3a5b6fcf33bca4c929d220cf5661e)
---
airflow/sensors/base.py | 4 ++++
tests/serialization/test_dag_serialization.py | 1 +
tests/ti_deps/deps/test_ready_to_reschedule_dep.py | 7 ++++++-
3 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/airflow/sensors/base.py b/airflow/sensors/base.py
index 7f1cd87c3d..f00b3a6761 100644
--- a/airflow/sensors/base.py
+++ b/airflow/sensors/base.py
@@ -339,6 +339,10 @@ class BaseSensorOperator(BaseOperator, SkipMixin):
"""Define mode rescheduled sensors."""
return self.mode == 'reschedule'
+ @classmethod
+ def get_serialized_fields(cls):
+ return super().get_serialized_fields() | {"reschedule"}
+
def poke_mode_only(cls):
"""
diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py
index 0144501f1a..fe9fc7c7e5 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -1462,6 +1462,7 @@ class TestStringifiedDAGs:
assert "deps" in blob
serialized_op = SerializedBaseOperator.deserialize_operator(blob)
+ assert serialized_op.reschedule == (mode == "reschedule")
assert op.deps == serialized_op.deps
@pytest.mark.parametrize(
diff --git a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
index 470166db21..99416bbbc8 100644
--- a/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
+++ b/tests/ti_deps/deps/test_ready_to_reschedule_dep.py
@@ -31,7 +31,7 @@ from airflow.utils.timezone import utcnow
class TestNotInReschedulePeriodDep(unittest.TestCase):
def _get_task_instance(self, state):
dag = DAG('test_dag')
- task = Mock(dag=dag)
+ task = Mock(dag=dag, reschedule=True)
ti = TaskInstance(task=task, state=state, run_id=None)
return ti
@@ -52,6 +52,11 @@ class TestNotInReschedulePeriodDep(unittest.TestCase):
dep_context = DepContext(ignore_in_reschedule_period=True)
assert ReadyToRescheduleDep().is_met(ti=ti, dep_context=dep_context)
+ def test_should_pass_if_not_reschedule_mode(self):
+ ti = self._get_task_instance(State.UP_FOR_RESCHEDULE)
+ del ti.task.reschedule
+ assert ReadyToRescheduleDep().is_met(ti=ti)
+
def test_should_pass_if_not_in_none_state(self):
ti = self._get_task_instance(State.UP_FOR_RETRY)
assert ReadyToRescheduleDep().is_met(ti=ti)
[airflow] 03/04: Fix auto upstream dep when expanding non-templated field (#23771)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 27762afd9f01519d0dd53a769eae968b96ac2580
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed May 18 13:43:16 2022 -0600
Fix auto upstream dep when expanding non-templated field (#23771)
If you tried to expand via xcom into a non-templated field without
explicitly setting the upstream task dependency, the scheduler would
crash because the upstream task dependency wasn't being set
automatically. It was being set only for templated fields, but now we do
it for both.
(cherry picked from commit 3849ebb8d22bbc229d464c4171c9b5ff960cd089)
---
airflow/models/mappedoperator.py | 3 +--
tests/models/test_taskinstance.py | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py
index b63e26ec9e..c522cefb2c 100644
--- a/airflow/models/mappedoperator.py
+++ b/airflow/models/mappedoperator.py
@@ -300,8 +300,7 @@ class MappedOperator(AbstractOperator):
if self.dag:
self.dag.add_task(self)
for k, v in self.mapped_kwargs.items():
- if k in self.template_fields:
- XComArg.apply_upstream_relationship(self, v)
+ XComArg.apply_upstream_relationship(self, v)
for k, v in self.partial_kwargs.items():
if k in self.template_fields:
XComArg.apply_upstream_relationship(self, v)
diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py
index 4bb3b2bae4..c520fe70bd 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -2855,3 +2855,23 @@ def test_ti_mapped_depends_on_mapped_xcom_arg_XXX(dag_maker, session):
ti.refresh_from_task(dag.get_task("add_one"))
with pytest.raises(XComForMappingNotPushed):
ti.run()
+
+
+def test_expand_non_templated_field(dag_maker, session):
+ """Test expand on non-templated fields sets upstream deps properly."""
+
+ class SimpleBashOperator(BashOperator):
+ template_fields = ()
+
+ with dag_maker(dag_id="product_same_types", session=session) as dag:
+
+ @dag.task
+ def get_extra_env():
+ return [{"foo": "bar"}, {"foo": "biz"}]
+
+ SimpleBashOperator.partial(task_id="echo", bash_command="echo $FOO").expand(env=get_extra_env())
+
+ dag_maker.create_dagrun()
+
+ echo_task = dag.get_task("echo")
+ assert "get_extra_env" in echo_task.upstream_task_ids
[airflow] 02/04: Fix task log is not captured (#23684)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 99202020fcaff1260247fc63f77cd48fb6a7924d
Author: Ping Zhang <pi...@umich.edu>
AuthorDate: Tue May 17 09:29:52 2022 -0700
Fix task log is not captured (#23684)
when StandardTaskRunner runs tasks with exec
Issue: https://github.com/apache/airflow/issues/23540
(cherry picked from commit e453e68158b65177572a3a1f95d8542c3a38d4e7)
---
airflow/models/taskinstance.py | 6 ++++--
tests/cli/commands/test_task_command.py | 2 +-
tests/models/test_taskinstance.py | 6 ++++++
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index e1e87ec8e8..0bc90636ff 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -548,7 +548,8 @@ class TaskInstance(Base, LoggingMixin):
self.task_id = task.task_id
self.map_index = map_index
self.refresh_from_task(task)
- self._log = logging.getLogger("airflow.task")
+ # init_on_load will config the log
+ self.init_on_load()
if run_id is None and execution_date is not None:
from airflow.models.dagrun import DagRun # Avoid circular import
@@ -591,7 +592,6 @@ class TaskInstance(Base, LoggingMixin):
if state:
self.state = state
self.hostname = ''
- self.init_on_load()
# Is this TaskInstance being currently running within `airflow tasks run --raw`.
# Not persisted to the database so only valid for the current process
self.raw = False
@@ -622,6 +622,8 @@ class TaskInstance(Base, LoggingMixin):
@reconstructor
def init_on_load(self):
"""Initialize the attributes that aren't stored in the DB"""
+ # correctly config the ti log
+ self._log = logging.getLogger("airflow.task")
self.test_mode = False # can be changed when calling 'run'
@property
diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py
index e539592a64..08ecebf525 100644
--- a/tests/cli/commands/test_task_command.py
+++ b/tests/cli/commands/test_task_command.py
@@ -110,7 +110,7 @@ class TestCliTasks(unittest.TestCase):
args = self.parser.parse_args(["tasks", "test", self.dag_id, task_id, DEFAULT_DATE.isoformat()])
- with self.assertLogs('airflow.models', level='INFO') as cm:
+ with self.assertLogs('airflow.task', level='INFO') as cm:
task_command.task_test(args)
assert any(
[
diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py
index 57355a3ad5..4bb3b2bae4 100644
--- a/tests/models/test_taskinstance.py
+++ b/tests/models/test_taskinstance.py
@@ -264,6 +264,12 @@ class TestTaskInstance:
assert op2 in op1.downstream_list
assert op2 in op3.downstream_list
+ def test_init_on_load(self, create_task_instance):
+ ti = create_task_instance()
+ # ensure log is correctly created for ORM ti
+ assert ti.log.name == 'airflow.task'
+ assert not ti.test_mode
+
@patch.object(DAG, 'get_concurrency_reached')
def test_requeue_over_dag_concurrency(self, mock_concurrency_reached, create_task_instance):
mock_concurrency_reached.return_value = True
[airflow] 04/04: Fix dag-processor fetch metabase config (#23575)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d07392fe15d30b61631de30c55b91dd0843bfef3
Author: Andrey Anshin <An...@river-island.com>
AuthorDate: Mon May 9 11:50:33 2022 +0300
Fix dag-processor fetch metabase config (#23575)
(cherry picked from commit 9837e6d813744e3c5861c32e87b3aeb496d0f88d)
---
airflow/cli/commands/dag_processor_command.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/cli/commands/dag_processor_command.py b/airflow/cli/commands/dag_processor_command.py
index a07035be50..183b77dbe2 100644
--- a/airflow/cli/commands/dag_processor_command.py
+++ b/airflow/cli/commands/dag_processor_command.py
@@ -50,7 +50,7 @@ def dag_processor(args):
if not conf.getboolean("scheduler", "standalone_dag_processor"):
raise SystemExit('The option [scheduler/standalone_dag_processor] must be True.')
- sql_conn: str = conf.get('core', 'sql_alchemy_conn').lower()
+ sql_conn: str = conf.get('database', 'sql_alchemy_conn').lower()
if sql_conn.startswith('sqlite'):
raise SystemExit('Standalone DagProcessor is not supported when using sqlite.')