You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/04/27 19:53:33 UTC
[airflow] branch v2-3-test updated (aca1c0c896 -> efdb945505)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
from aca1c0c896 Update default branches for 2-3
new 5617ca009c We weren't checking .jsx files for licenses and missed one (#23289)
new a83b6d4630 Revert "EdgeModifier refactoring (#21404)" (#23291)
new f22dab0f90 Add missing licenses and update `.rat-excludes` (#23296)
new 0adcfa7ca1 Fix retrieval of the right branch in pre-commits (#23297)
new efdb945505 update release note
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.pre-commit-config.yaml | 2 +-
.rat-excludes | 5 +
RELEASE_NOTES.rst | 1 -
airflow/models/taskmixin.py | 21 +--
airflow/utils/edgemodifier.py | 107 +++++------
airflow/utils/task_group.py | 17 +-
airflow/www/static/js/tree/Clipboard.jsx | 19 ++
chart/newsfragments/config.toml | 16 ++
dev/CHERRY_PICK_SUMMARY.txt.jinja2 | 18 ++
dev/ISSUE_TEMPLATE.md.jinja2 | 18 ++
dev/templates/example_dags_template.txt.jinja2 | 18 ++
dev/templates/system_test_template.txt.jinja2 | 18 ++
newsfragments/config.toml | 16 ++
scripts/ci/docker-compose/devcontainer.env | 16 ++
scripts/ci/pre_commit/pre_commit_flake8.py | 6 +-
.../pre_commit/pre_commit_migration_reference.py | 5 +-
scripts/ci/pre_commit/pre_commit_mypy.py | 7 +-
scripts/ci/pre_commit/pre_commit_ui_lint.py | 4 +
scripts/ci/pre_commit/pre_commit_www_lint.py | 4 +
tests/utils/test_edgemodifier.py | 205 ---------------------
20 files changed, 228 insertions(+), 295 deletions(-)
[airflow] 02/05: Revert "EdgeModifier refactoring (#21404)" (#23291)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a83b6d4630fec00e2e48f00f20d67ddcfe0b9f88
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Wed Apr 27 19:12:08 2022 +0100
Revert "EdgeModifier refactoring (#21404)" (#23291)
This reverts commit ace8c6e942ff5554639801468b971915b7c0e9b9.
Fixes apache/airflow#23285
This dag breaks with a cycle as group.end has itself as a downstream otherwise:
```python
from pendulum import datetime
from airflow.decorators import dag, task, task_group
from airflow.utils.edgemodifier import Label
@task
def begin():
...
@task
def end():
...
@dag(start_date=datetime(2022, 1, 1), schedule_interval=None)
def task_groups_with_edge_labels():
@task_group
def group():
begin() >> end()
group()
_ = task_groups_with_edge_labels()
```
(cherry picked from commit 3182303ce50bda6d5d27a6ef4e19450fb4e47eea)
---
airflow/models/taskmixin.py | 21 +---
airflow/utils/edgemodifier.py | 107 ++++++++++----------
airflow/utils/task_group.py | 17 +---
tests/utils/test_edgemodifier.py | 205 ---------------------------------------
4 files changed, 60 insertions(+), 290 deletions(-)
diff --git a/airflow/models/taskmixin.py b/airflow/models/taskmixin.py
index 1a7129f66e..1d66a719a6 100644
--- a/airflow/models/taskmixin.py
+++ b/airflow/models/taskmixin.py
@@ -55,29 +55,16 @@ class DependencyMixin:
raise NotImplementedError()
@abstractmethod
- def set_upstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
- ):
+ def set_upstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
"""Set a task or a task list to be directly upstream from the current task."""
raise NotImplementedError()
@abstractmethod
- def set_downstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
- ):
+ def set_downstream(self, other: Union["DependencyMixin", Sequence["DependencyMixin"]]):
"""Set a task or a task list to be directly downstream from the current task."""
raise NotImplementedError()
- def update_relative(
- self,
- other: "DependencyMixin",
- upstream=True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ def update_relative(self, other: "DependencyMixin", upstream=True) -> None:
"""
Update relationship information about another TaskMixin. Default is no-op.
Override if necessary.
@@ -184,7 +171,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
task_list: List[Operator] = []
for task_object in task_or_task_list:
- task_object.update_relative(self, not upstream, edge_modifier=edge_modifier)
+ task_object.update_relative(self, not upstream)
relatives = task_object.leaves if upstream else task_object.roots
for task in relatives:
if not isinstance(task, (BaseOperator, MappedOperator)):
diff --git a/airflow/utils/edgemodifier.py b/airflow/utils/edgemodifier.py
index 109199fc7c..a59616c76b 100644
--- a/airflow/utils/edgemodifier.py
+++ b/airflow/utils/edgemodifier.py
@@ -14,9 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, Optional, Sequence, Union
+from typing import TYPE_CHECKING, List, Optional, Sequence, Union
-from airflow.models.taskmixin import DAGNode, DependencyMixin
+from airflow.models.taskmixin import DependencyMixin
+
+if TYPE_CHECKING:
+ from airflow.models.baseoperator import BaseOperator
class EdgeModifier(DependencyMixin):
@@ -39,8 +42,8 @@ class EdgeModifier(DependencyMixin):
def __init__(self, label: Optional[str] = None):
self.label = label
- self._upstream: List["DependencyMixin"] = []
- self._downstream: List["DependencyMixin"] = []
+ self._upstream: List["BaseOperator"] = []
+ self._downstream: List["BaseOperator"] = []
@property
def roots(self):
@@ -50,38 +53,8 @@ class EdgeModifier(DependencyMixin):
def leaves(self):
return self._upstream
- @staticmethod
- def _make_list(item_or_list):
- if not isinstance(item_or_list, Sequence):
- return [item_or_list]
- return item_or_list
-
- def _save_nodes(
- self,
- nodes: Union["DependencyMixin", Sequence["DependencyMixin"]],
- stream: List["DependencyMixin"],
- ):
- from airflow.models.xcom_arg import XComArg
- from airflow.utils.task_group import TaskGroup
-
- for node in self._make_list(nodes):
- if isinstance(node, (TaskGroup, XComArg)):
- stream.append(node)
- elif isinstance(node, DAGNode):
- if node.task_group and not node.task_group.is_root:
- stream.append(node.task_group)
- else:
- stream.append(node)
- else:
- raise TypeError(
- f"Cannot use edge labels with {type(node).__name__}, "
- f"only tasks, XComArg or TaskGroups"
- )
-
def set_upstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
+ self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
):
"""
Sets the given task/list onto the upstream attribute, and then checks if
@@ -89,14 +62,30 @@ class EdgeModifier(DependencyMixin):
Providing this also provides << via DependencyMixin.
"""
- self._save_nodes(other, self._upstream)
- for node in self._downstream:
- node.set_upstream(other, edge_modifier=self)
+ from airflow.models.baseoperator import BaseOperator
+
+ # Ensure we have a list, even if it's just one item
+ if isinstance(task_or_task_list, DependencyMixin):
+ task_or_task_list = [task_or_task_list]
+ # Unfurl it into actual operators
+ operators: List[BaseOperator] = []
+ for task in task_or_task_list:
+ for root in task.roots:
+ if not isinstance(root, BaseOperator):
+ raise TypeError(f"Cannot use edge labels with {type(root).__name__}, only operators")
+ operators.append(root)
+ # For each already-declared downstream, pair off with each new upstream
+ # item and store the edge info.
+ for operator in operators:
+ for downstream in self._downstream:
+ self.add_edge_info(operator.dag, operator.task_id, downstream.task_id)
+ if chain:
+ operator.set_downstream(downstream)
+ # Add the new tasks to our list of ones we've seen
+ self._upstream.extend(operators)
def set_downstream(
- self,
- other: Union["DependencyMixin", Sequence["DependencyMixin"]],
- edge_modifier: Optional["EdgeModifier"] = None,
+ self, task_or_task_list: Union[DependencyMixin, Sequence[DependencyMixin]], chain: bool = True
):
"""
Sets the given task/list onto the downstream attribute, and then checks if
@@ -104,24 +93,36 @@ class EdgeModifier(DependencyMixin):
Providing this also provides >> via DependencyMixin.
"""
- self._save_nodes(other, self._downstream)
- for node in self._upstream:
- node.set_downstream(other, edge_modifier=self)
-
- def update_relative(
- self,
- other: "DependencyMixin",
- upstream: bool = True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ from airflow.models.baseoperator import BaseOperator
+
+ # Ensure we have a list, even if it's just one item
+ if isinstance(task_or_task_list, DependencyMixin):
+ task_or_task_list = [task_or_task_list]
+ # Unfurl it into actual operators
+ operators: List[BaseOperator] = []
+ for task in task_or_task_list:
+ for leaf in task.leaves:
+ if not isinstance(leaf, BaseOperator):
+ raise TypeError(f"Cannot use edge labels with {type(leaf).__name__}, only operators")
+ operators.append(leaf)
+ # Pair them off with existing
+ for operator in operators:
+ for upstream in self._upstream:
+ self.add_edge_info(upstream.dag, upstream.task_id, operator.task_id)
+ if chain:
+ upstream.set_downstream(operator)
+ # Add the new tasks to our list of ones we've seen
+ self._downstream.extend(operators)
+
+ def update_relative(self, other: DependencyMixin, upstream: bool = True) -> None:
"""
Called if we're not the "main" side of a relationship; we still run the
same logic, though.
"""
if upstream:
- self.set_upstream(other)
+ self.set_upstream(other, chain=False)
else:
- self.set_downstream(other)
+ self.set_downstream(other, chain=False)
def add_edge_info(self, dag, upstream_id: str, downstream_id: str):
"""
diff --git a/airflow/utils/task_group.py b/airflow/utils/task_group.py
index 11ee806beb..7b53a521db 100644
--- a/airflow/utils/task_group.py
+++ b/airflow/utils/task_group.py
@@ -229,12 +229,7 @@ class TaskGroup(DAGNode):
"""group_id excluding parent's group_id used as the node label in UI."""
return self._group_id
- def update_relative(
- self,
- other: DependencyMixin,
- upstream=True,
- edge_modifier: Optional["EdgeModifier"] = None,
- ) -> None:
+ def update_relative(self, other: DependencyMixin, upstream=True) -> None:
"""
Overrides TaskMixin.update_relative.
@@ -259,18 +254,10 @@ class TaskGroup(DAGNode):
f"or operators; received {task.__class__.__name__}"
)
- # Do not set a relationship between a TaskGroup and a Label's roots
- if self == task:
- continue
-
if upstream:
self.upstream_task_ids.add(task.node_id)
- if edge_modifier:
- edge_modifier.add_edge_info(self.dag, task.node_id, self.upstream_join_id)
else:
self.downstream_task_ids.add(task.node_id)
- if edge_modifier:
- edge_modifier.add_edge_info(self.dag, self.downstream_join_id, task.node_id)
def _set_relatives(
self,
@@ -293,7 +280,7 @@ class TaskGroup(DAGNode):
task_or_task_list = [task_or_task_list]
for task_like in task_or_task_list:
- self.update_relative(task_like, upstream, edge_modifier=edge_modifier)
+ self.update_relative(task_like, upstream)
def __enter__(self) -> "TaskGroup":
TaskGroupContext.push_context_managed_task_group(self)
diff --git a/tests/utils/test_edgemodifier.py b/tests/utils/test_edgemodifier.py
index da3291fad1..21573b3cf6 100644
--- a/tests/utils/test_edgemodifier.py
+++ b/tests/utils/test_edgemodifier.py
@@ -21,11 +21,9 @@ import pytest
from airflow import DAG
from airflow.models.xcom_arg import XComArg
-from airflow.operators.empty import EmptyOperator
from airflow.operators.python import PythonOperator
from airflow.utils.edgemodifier import Label
from airflow.utils.task_group import TaskGroup
-from airflow.www.views import dag_edges
DEFAULT_ARGS = {
"owner": "test",
@@ -64,125 +62,6 @@ def test_taskgroup_dag():
return dag, group, (op1, op2, op3, op4)
-@pytest.fixture
-def test_complex_taskgroup_dag():
- """Creates a test DAG with many operators and a task group."""
-
- def f(task_id):
- return f"OP:{task_id}"
-
- with DAG(dag_id="test_complex_dag", default_args=DEFAULT_ARGS) as dag:
- with TaskGroup("group_1") as group:
- group_dm1 = EmptyOperator(task_id="group_dummy1")
- group_dm2 = EmptyOperator(task_id="group_dummy2")
- group_dm3 = EmptyOperator(task_id="group_dummy3")
- dm_in1 = EmptyOperator(task_id="dummy_in1")
- dm_in2 = EmptyOperator(task_id="dummy_in2")
- dm_in3 = EmptyOperator(task_id="dummy_in3")
- dm_in4 = EmptyOperator(task_id="dummy_in4")
- dm_out1 = EmptyOperator(task_id="dummy_out1")
- dm_out2 = EmptyOperator(task_id="dummy_out2")
- dm_out3 = EmptyOperator(task_id="dummy_out3")
- dm_out4 = EmptyOperator(task_id="dummy_out4")
- op_in1 = PythonOperator(python_callable=f, task_id="op_in1")
- op_out1 = PythonOperator(python_callable=f, task_id="op_out1")
-
- return (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- )
-
-
-@pytest.fixture
-def simple_dag_expected_edges():
- return [
- {'source_id': 'group_1.downstream_join_id', 'target_id': 'test_op_4'},
- {'source_id': 'group_1.test_op_2', 'target_id': 'group_1.downstream_join_id'},
- {'source_id': 'group_1.test_op_3', 'target_id': 'group_1.downstream_join_id'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_2'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.test_op_3'},
- {'label': 'Label', 'source_id': 'test_op_1', 'target_id': 'group_1.upstream_join_id'},
- ]
-
-
-@pytest.fixture
-def complex_dag_expected_edges():
- return [
- {'source_id': 'dummy_in1', 'target_id': 'group_1.upstream_join_id'},
- {
- 'label': 'label dm_in2 <=> group',
- 'source_id': 'dummy_in2',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label dm_in3/dm_in4 <=> group',
- 'source_id': 'dummy_in3',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label dm_in3/dm_in4 <=> group',
- 'source_id': 'dummy_in4',
- 'target_id': 'group_1.upstream_join_id',
- },
- {'source_id': 'group_1.downstream_join_id', 'target_id': 'dummy_out1'},
- {
- 'label': 'label group <=> dm_out2',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out2',
- },
- {
- 'label': 'label group <=> dm_out3/dm_out4',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out3',
- },
- {
- 'label': 'label group <=> dm_out3/dm_out4',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'dummy_out4',
- },
- {
- 'source_id': 'group_1.group_dummy1',
- 'target_id': 'group_1.downstream_join_id',
- },
- {'source_id': 'group_1.group_dummy2', 'target_id': 'group_1.group_dummy1'},
- {'source_id': 'group_1.group_dummy3', 'target_id': 'group_1.group_dummy1'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy2'},
- {'source_id': 'group_1.upstream_join_id', 'target_id': 'group_1.group_dummy3'},
- {
- 'label': 'label op_in1 <=> group',
- 'source_id': 'op_in1',
- 'target_id': 'group_1.upstream_join_id',
- },
- {
- 'label': 'label group <=> op_out1',
- 'source_id': 'group_1.downstream_join_id',
- 'target_id': 'op_out1',
- },
- ]
-
-
-def compare_dag_edges(current, expected):
- assert len(current) == len(expected)
-
- for i in current:
- assert current.count(i) == expected.count(i), f'The unexpected DAG edge: {i}'
-
-
class TestEdgeModifierBuilding:
"""
Tests that EdgeModifiers work when composed with Tasks (either via >>
@@ -283,87 +162,3 @@ class TestEdgeModifierBuilding:
assert dag.get_edge_info(op1.task_id, op2.task_id) == {"label": "Group label"}
assert dag.get_edge_info(op1.task_id, op3.task_id) == {"label": "Group label"}
assert dag.get_edge_info(op3.task_id, op4.task_id) == {}
-
- def test_simple_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
- """Tests the simple dag with a TaskGroup and a Label"""
- dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
- op1 >> Label("Label") >> group >> op4
- compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
- def test_simple_reversed_dag(self, test_taskgroup_dag, simple_dag_expected_edges):
- """Tests the simple reversed dag with a TaskGroup and a Label"""
- dag, group, (op1, op2, op3, op4) = test_taskgroup_dag
- op4 << group << Label("Label") << op1
- compare_dag_edges(dag_edges(dag), simple_dag_expected_edges)
-
- def test_complex_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
- """Tests the complex dag with a TaskGroup and a Label"""
- (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- ) = test_complex_taskgroup_dag
-
- [group_dm2, group_dm3] >> group_dm1
-
- dm_in1 >> group
- dm_in2 >> Label('label dm_in2 <=> group') >> group
- [dm_in3, dm_in4] >> Label('label dm_in3/dm_in4 <=> group') >> group
- XComArg(op_in1, 'test_key') >> Label('label op_in1 <=> group') >> group
-
- group >> dm_out1
- group >> Label('label group <=> dm_out2') >> dm_out2
- group >> Label('label group <=> dm_out3/dm_out4') >> [dm_out3, dm_out4]
- group >> Label('label group <=> op_out1') >> XComArg(op_out1, 'test_key')
-
- compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)
-
- def test_complex_reversed_dag(self, test_complex_taskgroup_dag, complex_dag_expected_edges):
- """Tests the complex reversed dag with a TaskGroup and a Label"""
- (
- dag,
- group,
- (
- group_dm1,
- group_dm2,
- group_dm3,
- dm_in1,
- dm_in2,
- dm_in3,
- dm_in4,
- dm_out1,
- dm_out2,
- dm_out3,
- dm_out4,
- op_in1,
- op_out1,
- ),
- ) = test_complex_taskgroup_dag
-
- group_dm1 << [group_dm2, group_dm3]
-
- group << dm_in1
- group << Label('label dm_in2 <=> group') << dm_in2
- group << Label('label dm_in3/dm_in4 <=> group') << [dm_in3, dm_in4]
- group << Label('label op_in1 <=> group') << XComArg(op_in1, 'test_key')
-
- dm_out1 << group
- dm_out2 << Label('label group <=> dm_out2') << group
- [dm_out3, dm_out4] << Label('label group <=> dm_out3/dm_out4') << group
- XComArg(op_out1, 'test_key') << Label('label group <=> op_out1') << group
-
- compare_dag_edges(dag_edges(dag), complex_dag_expected_edges)
[airflow] 03/05: Add missing licenses and update `.rat-excludes` (#23296)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f22dab0f90de607661276f204b6d659a144872c5
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed Apr 27 13:24:20 2022 -0600
Add missing licenses and update `.rat-excludes` (#23296)
(cherry picked from commit b3396558c4b045bbc15bb3d9ce9a29308cafba78)
---
.rat-excludes | 5 +++++
chart/newsfragments/config.toml | 16 ++++++++++++++++
dev/CHERRY_PICK_SUMMARY.txt.jinja2 | 18 ++++++++++++++++++
dev/ISSUE_TEMPLATE.md.jinja2 | 18 ++++++++++++++++++
dev/templates/example_dags_template.txt.jinja2 | 18 ++++++++++++++++++
dev/templates/system_test_template.txt.jinja2 | 18 ++++++++++++++++++
newsfragments/config.toml | 16 ++++++++++++++++
scripts/ci/docker-compose/devcontainer.env | 16 ++++++++++++++++
8 files changed, 125 insertions(+)
diff --git a/.rat-excludes b/.rat-excludes
index d3c4ac67aa..fa4663ce65 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -7,6 +7,7 @@
.gitrepo
.airflow_db_initialised
.airflowignore
+.babelrc
.coverage
.coveragerc
.codecov.yml
@@ -42,6 +43,7 @@ venv
files
airflow.iml
.gitmodules
+scripts/ci/installed_providers.txt
# Generated doc files
.*html
@@ -116,3 +118,6 @@ airflow/www/static/robots.txt
# Generated autocomplete files
dev/breeze/autocomplete/*
+
+# Newsfragments are snippets that will be, eventually, consumed into RELEASE_NOTES
+newsfragments/*
diff --git a/chart/newsfragments/config.toml b/chart/newsfragments/config.toml
index 7c83de868d..b00560d711 100644
--- a/chart/newsfragments/config.toml
+++ b/chart/newsfragments/config.toml
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
[tool.towncrier]
name = "Airflow Helm Chart"
filename = "RELEASE_NOTES.rst"
diff --git a/dev/CHERRY_PICK_SUMMARY.txt.jinja2 b/dev/CHERRY_PICK_SUMMARY.txt.jinja2
index a382e585f9..c91f9dd7d1 100644
--- a/dev/CHERRY_PICK_SUMMARY.txt.jinja2
+++ b/dev/CHERRY_PICK_SUMMARY.txt.jinja2
@@ -1,3 +1,21 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
---------------------------------------------------------
This is summary of the cherry-picks:
diff --git a/dev/ISSUE_TEMPLATE.md.jinja2 b/dev/ISSUE_TEMPLATE.md.jinja2
index d57fb0c6f1..be8fa5d74c 100644
--- a/dev/ISSUE_TEMPLATE.md.jinja2
+++ b/dev/ISSUE_TEMPLATE.md.jinja2
@@ -1,3 +1,21 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
We have a kind request for all the contributors to the latest [{{link_text}}]({{link}}).
diff --git a/dev/templates/example_dags_template.txt.jinja2 b/dev/templates/example_dags_template.txt.jinja2
index a8b2384020..52077a5dc8 100644
--- a/dev/templates/example_dags_template.txt.jinja2
+++ b/dev/templates/example_dags_template.txt.jinja2
@@ -1,3 +1,21 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
There is a new design of system tests that was introduced by the [AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests).
All current example dags need to be migrated and converted into system tests, so they can be run in the CI process automatically before releases.
diff --git a/dev/templates/system_test_template.txt.jinja2 b/dev/templates/system_test_template.txt.jinja2
index 7fa3815c78..061ec0b384 100644
--- a/dev/templates/system_test_template.txt.jinja2
+++ b/dev/templates/system_test_template.txt.jinja2
@@ -1,3 +1,21 @@
+{#
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-#}
There is a new design of system tests that was introduced by the [AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests).
All current system tests need to be migrated, so they can be run in the CI process automatically before releases.
diff --git a/newsfragments/config.toml b/newsfragments/config.toml
index ab30e3b52b..014284c133 100644
--- a/newsfragments/config.toml
+++ b/newsfragments/config.toml
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
[tool.towncrier]
name = "Airflow"
filename = "../RELEASE_NOTES.rst"
diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env
index 708b1a5496..3698ac4abe 100644
--- a/scripts/ci/docker-compose/devcontainer.env
+++ b/scripts/ci/docker-compose/devcontainer.env
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
HOME=
AIRFLOW_CI_IMAGE="ghcr.io/apache/airflow/main/ci/python3.7:latest"
ANSWER=
[airflow] 04/05: Fix retrieval of the right branch in pre-commits (#23297)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0adcfa7ca1e1782b9abbe1f9faf7d66e8bf54753
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed Apr 27 21:47:16 2022 +0200
Fix retrieval of the right branch in pre-commits (#23297)
(cherry picked from commit d93908deb7bfea2bc7cfe5a69830b01e4df06676)
---
scripts/ci/pre_commit/pre_commit_flake8.py | 6 +++++-
scripts/ci/pre_commit/pre_commit_migration_reference.py | 5 ++++-
scripts/ci/pre_commit/pre_commit_mypy.py | 7 ++++++-
scripts/ci/pre_commit/pre_commit_ui_lint.py | 4 ++++
scripts/ci/pre_commit/pre_commit_www_lint.py | 4 ++++
5 files changed, 23 insertions(+), 3 deletions(-)
diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py
index 163154ba47..19841b3347 100755
--- a/scripts/ci/pre_commit/pre_commit_flake8.py
+++ b/scripts/ci/pre_commit/pre_commit_flake8.py
@@ -30,9 +30,13 @@ if __name__ not in ("__main__", "__mp_main__"):
AIRFLOW_SOURCES = Path(__file__).parents[3].absolute()
GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
-AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7"
if __name__ == '__main__':
+ sys.path.insert(0, str(Path(__file__).parents[3].absolute() / "dev" / "breeze" / "src"))
+ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
+ AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7"
+
if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0:
print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n')
print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n")
diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py
index a0a0b7bed0..6e1fc15c19 100755
--- a/scripts/ci/pre_commit/pre_commit_migration_reference.py
+++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py
@@ -30,9 +30,12 @@ if __name__ not in ("__main__", "__mp_main__"):
AIRFLOW_SOURCES = Path(__file__).parents[3].absolute()
GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
-AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7"
if __name__ == '__main__':
+ sys.path.insert(0, str(Path(__file__).parents[3].absolute() / "dev" / "breeze" / "src"))
+ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
+ AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7"
if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0:
print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n')
print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n")
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py
index 93ff218f94..0c70b96d52 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -28,11 +28,16 @@ if __name__ not in ("__main__", "__mp_main__"):
f"To run this script, run the ./{__file__} command"
)
+
AIRFLOW_SOURCES = Path(__file__).parents[3].absolute()
GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
-AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7"
if __name__ == '__main__':
+ sys.path.insert(0, str(Path(__file__).parents[3].absolute() / "dev" / "breeze" / "src"))
+ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
+ AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7"
+
if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0:
print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n')
print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n")
diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py
index ab29d9707c..45360faa85 100755
--- a/scripts/ci/pre_commit/pre_commit_ui_lint.py
+++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py
@@ -33,6 +33,10 @@ GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7"
if __name__ == '__main__':
+ sys.path.insert(0, str(Path(__file__).parents[3].absolute() / "dev" / "breeze" / "src"))
+ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
+ AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7"
if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0:
print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n')
print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n")
diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py
index 2e64f4a54c..5e0d1f83ab 100755
--- a/scripts/ci/pre_commit/pre_commit_www_lint.py
+++ b/scripts/ci/pre_commit/pre_commit_www_lint.py
@@ -33,6 +33,10 @@ GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY', "apache/airflow")
AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/main/ci/python3.7"
if __name__ == '__main__':
+ sys.path.insert(0, str(Path(__file__).parents[3].absolute() / "dev" / "breeze" / "src"))
+ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH
+
+ AIRFLOW_CI_IMAGE = f"ghcr.io/{GITHUB_REPOSITORY}/{AIRFLOW_BRANCH}/ci/python3.7"
if subprocess.call(args=["docker", "inspect", AIRFLOW_CI_IMAGE], stdout=subprocess.DEVNULL) != 0:
print(f'[red]The image {AIRFLOW_CI_IMAGE} is not available.[/]\n')
print("\n[yellow]Please run at the earliest convenience:[/]\n\nbreeze build-image --python 3.7\n\n")
[airflow] 05/05: update release note
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit efdb945505c0378e1421c71149e21b110fbe7068
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Wed Apr 27 20:51:56 2022 +0100
update release note
---
RELEASE_NOTES.rst | 1 -
1 file changed, 1 deletion(-)
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index cb02280c42..7831eaa29a 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -453,7 +453,6 @@ Bug Fixes
- Don't validate that Params are JSON when NOTSET (#22000)
- Add per-DAG delete permissions (#21938)
- Fix handling some None parameters in kubernetes 23 libs. (#21905)
-- ``EdgeModifier`` refactoring (#21404)
- Fix handling of empty (None) tags in ``bulk_write_to_db`` (#21757)
- Fix DAG date range bug (#20507)
- Removed ``request.referrer`` from views.py (#21751)
[airflow] 01/05: We weren't checking .jsx files for licenses and missed one (#23289)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5617ca009c99e98cf762a05957a0196fa09b7589
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Wed Apr 27 18:33:36 2022 +0100
We weren't checking .jsx files for licenses and missed one (#23289)
(cherry picked from commit 726b27f86cf964924e5ee7b29a30aefe24dac45a)
---
.pre-commit-config.yaml | 2 +-
airflow/www/static/js/tree/Clipboard.jsx | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bc103e6c27..47147bec8e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -66,7 +66,7 @@ repos:
files: \.rst$
- id: insert-license
name: Add license for all CSS/JS/PUML/TS/TSX files
- files: \.(css|js|puml|ts|tsx)$
+ files: \.(css|js|puml|ts|tsx|jsx)$
exclude: ^\.github/.*$|^airflow/_vendor/
args:
- --comment-style
diff --git a/airflow/www/static/js/tree/Clipboard.jsx b/airflow/www/static/js/tree/Clipboard.jsx
index 27f34a9154..5fa645e61a 100644
--- a/airflow/www/static/js/tree/Clipboard.jsx
+++ b/airflow/www/static/js/tree/Clipboard.jsx
@@ -1,3 +1,22 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
import React from 'react';
import {
Button,