You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2022/12/01 07:29:47 UTC
[airflow] branch v2-5-test updated (4cb5c8d2f1 -> 2b78e5562d)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 4cb5c8d2f1 Fix failing test case for db clean in newer sqlalchemy (#28004)
new 963a5a3188 Pass in session appropriately to _clear_dag_tis (#28003)
new 52c2fa8824 Fix failing test case for db clean in newer sqlalchemy (#28004)
new c9a72469ab Apply more masking on audit logs (#27994)
new cd1aea4d21 allow scroll in triggered dag runs modal (#27965)
new 2b78e5562d Add release notes
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4cb5c8d2f1)
\
N -- N -- N refs/heads/v2-5-test (2b78e5562d)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
RELEASE_NOTES.rst | 5 ++-
airflow/models/dag.py | 2 +-
airflow/www/decorators.py | 42 +++++++++++++++++++
airflow/www/static/js/components/Table/Cells.tsx | 10 ++++-
airflow/www/views.py | 19 +++++----
tests/test_utils/www.py | 53 ++++++++++++++++++++++++
tests/www/views/test_views_connection.py | 14 ++++++-
tests/www/views/test_views_decorators.py | 21 ++++++++--
8 files changed, 149 insertions(+), 17 deletions(-)
[airflow] 04/05: allow scroll in triggered dag runs modal (#27965)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cd1aea4d211eb76abe10377f47aa4ac36190307a
Author: Brent Bovenzi <br...@astronomer.io>
AuthorDate: Mon Nov 28 19:16:04 2022 -0600
allow scroll in triggered dag runs modal (#27965)
Co-authored-by: Jed Cunningham <66...@users.noreply.github.com>
(cherry picked from commit 5e4f4a3556db5111c2ae36af1716719a8494efc7)
---
airflow/www/static/js/components/Table/Cells.tsx | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/airflow/www/static/js/components/Table/Cells.tsx b/airflow/www/static/js/components/Table/Cells.tsx
index 4850d0772e..6c34f5d9b1 100644
--- a/airflow/www/static/js/components/Table/Cells.tsx
+++ b/airflow/www/static/js/components/Table/Cells.tsx
@@ -110,7 +110,14 @@ export const TriggeredRuns = ({ cell: { value, row } }: CellProps) => {
return (
<Box>
<Text color="blue.600" cursor="pointer" onClick={onToggle}>{value.length}</Text>
- <Modal size="3xl" isOpen={isOpen} onClose={onClose} portalProps={{ containerRef }}>
+ <Modal
+ size="3xl"
+ isOpen={isOpen}
+ onClose={onClose}
+ scrollBehavior="inside"
+ blockScrollOnMount={false}
+ portalProps={{ containerRef }}
+ >
<ModalOverlay />
<ModalContent>
<ModalHeader>
@@ -127,6 +134,7 @@ export const TriggeredRuns = ({ cell: { value, row } }: CellProps) => {
<Table
data={data}
columns={columns}
+ pageSize={data.length}
/>
</ModalBody>
</ModalContent>
[airflow] 01/05: Pass in session appropriately to _clear_dag_tis (#28003)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 963a5a3188d2cfa9d894934643d930de4f78eab6
Author: Tzu-ping Chung <ur...@gmail.com>
AuthorDate: Wed Nov 30 18:27:00 2022 +0800
Pass in session appropriately to _clear_dag_tis (#28003)
This makes the session argument required instead, and pass it from the
dagrun_clear view correctly.
Some type annotations are added also to the function for future
maintainability.
(cherry picked from commit f43f50e3f11fa02a2025b4b68b8770d6456ba95d)
---
airflow/models/dag.py | 2 +-
airflow/www/views.py | 19 +++++++++++--------
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/airflow/models/dag.py b/airflow/models/dag.py
index c761dcadfa..210f455f06 100644
--- a/airflow/models/dag.py
+++ b/airflow/models/dag.py
@@ -1950,7 +1950,7 @@ class DAG(LoggingMixin):
@provide_session
def clear(
self,
- task_ids: Collection[str] | Collection[tuple[str, int]] | None = None,
+ task_ids: Collection[str | tuple[str, int]] | None = None,
start_date: datetime | None = None,
end_date: datetime | None = None,
only_failed: bool = False,
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 39f69d4410..a4126535ca 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -33,7 +33,7 @@ from datetime import datetime, timedelta
from functools import wraps
from json import JSONDecodeError
from operator import itemgetter
-from typing import Any, Callable
+from typing import Any, Callable, Collection
from urllib.parse import unquote, urljoin, urlsplit
import configupdater
@@ -2013,11 +2013,12 @@ class Airflow(AirflowBaseView):
dag: DAG,
start_date: datetime | None,
end_date: datetime | None,
- origin: str,
- task_ids=None,
- recursive=False,
- confirmed=False,
- only_failed=False,
+ *,
+ origin: str | None,
+ task_ids: Collection[str | tuple[str, int]] | None = None,
+ recursive: bool = False,
+ confirmed: bool = False,
+ only_failed: bool = False,
session: Session = NEW_SESSION,
):
if confirmed:
@@ -2144,7 +2145,7 @@ class Airflow(AirflowBaseView):
dag,
start_date,
end_date,
- origin,
+ origin=origin,
task_ids=task_ids,
recursive=recursive,
confirmed=confirmed,
@@ -2164,7 +2165,8 @@ class Airflow(AirflowBaseView):
]
)
@action_logging
- def dagrun_clear(self):
+ @provide_session
+ def dagrun_clear(self, *, session: Session = NEW_SESSION):
"""Clears the DagRun"""
dag_id = request.form.get("dag_id")
dag_run_id = request.form.get("dag_run_id")
@@ -2182,6 +2184,7 @@ class Airflow(AirflowBaseView):
origin=None,
recursive=True,
confirmed=confirmed,
+ session=session,
)
@expose("/blocked", methods=["POST"])
[airflow] 05/05: Add release notes
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2b78e5562d2890ea8299019e55861daac8e3d585
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Thu Nov 24 16:14:49 2022 +0100
Add release notes
---
RELEASE_NOTES.rst | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index 3bda987b96..05d30a55eb 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -54,9 +54,9 @@ New Features
- Add DagRun state change to the Listener plugin system(#27113)
- Metric for raw task return codes (#27155)
- Add logic for XComArg to pull specific map indexes (#27771)
-- Clear TaskGroup (#26658)
+- Clear TaskGroup (#26658, #28003)
- Add critical section query duration metric (#27700)
-- Add: #23880 :: Audit log for ``AirflowModelViews(Variables/Connection)`` (#24079)
+- Add: #23880 :: Audit log for ``AirflowModelViews(Variables/Connection)`` (#24079, #27994, #27923)
- Add postgres 15 support (#27444)
- Expand tasks in mapped group at run time (#27491)
- reset commits, clean submodules (#27560)
@@ -166,6 +166,7 @@ Bug Fixes
- template rendering issue fix (#26390)
- Clear ``autoregistered`` DAGs if there are any import errors (#26398)
- Fix ``from airflow import version`` lazy import (#26239)
+- allow scroll in triggered dag runs modal (#27965)
Misc/Internal
^^^^^^^^^^^^^
[airflow] 03/05: Apply more masking on audit logs (#27994)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c9a72469abc0a6f3f3123b435c1dfdc14e20eae9
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Wed Nov 30 12:13:02 2022 +0100
Apply more masking on audit logs (#27994)
This fixes variables val and connection extra field masking in the audit log table
(cherry picked from commit 924725855134288bae52f6568d2b8c4fad393c3d)
---
airflow/www/decorators.py | 42 +++++++++++++++++++++++++
tests/test_utils/www.py | 53 ++++++++++++++++++++++++++++++++
tests/www/views/test_views_connection.py | 14 ++++++++-
tests/www/views/test_views_decorators.py | 21 ++++++++++---
4 files changed, 125 insertions(+), 5 deletions(-)
diff --git a/airflow/www/decorators.py b/airflow/www/decorators.py
index 2bde2c83fb..29094969f7 100644
--- a/airflow/www/decorators.py
+++ b/airflow/www/decorators.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import functools
import gzip
+import json
import logging
from io import BytesIO as IO
from itertools import chain
@@ -37,6 +38,43 @@ T = TypeVar("T", bound=Callable)
logger = logging.getLogger(__name__)
+def _mask_variable_fields(extra_fields):
+ """
+ The variable requests values and args comes in this form:
+ [('key', 'key_content'),('val', 'val_content'), ('description', 'description_content')]
+ So we need to mask the 'val_content' field if 'key_content' is in the mask list.
+ """
+ result = []
+ keyname = None
+ for k, v in extra_fields:
+ if k == "key":
+ keyname = v
+ result.append((k, v))
+ elif keyname and k == "val":
+ x = secrets_masker.redact(v, keyname)
+ result.append((k, x))
+ keyname = None
+ else:
+ result.append((k, v))
+ return result
+
+
+def _mask_connection_fields(extra_fields):
+ """Mask connection fields"""
+ result = []
+ for k, v in extra_fields:
+ if k == "extra":
+ try:
+ extra = json.loads(v)
+ extra = [(k, secrets_masker.redact(v, k)) for k, v in extra.items()]
+ result.append((k, json.dumps(dict(extra))))
+ except json.JSONDecodeError:
+ result.append((k, "Encountered non-JSON in `extra` field"))
+ else:
+ result.append((k, secrets_masker.redact(v, k)))
+ return result
+
+
def action_logging(func: Callable | None = None, event: str | None = None) -> Callable[[T], T]:
"""Decorator to log user actions"""
@@ -57,6 +95,10 @@ def action_logging(func: Callable | None = None, event: str | None = None) -> Ca
for k, v in chain(request.values.items(multi=True), request.view_args.items())
if k not in fields_skip_logging
]
+ if event and event.startswith("variable."):
+ extra_fields = _mask_variable_fields(extra_fields)
+ if event and event.startswith("connection."):
+ extra_fields = _mask_connection_fields(extra_fields)
params = {k: v for k, v in chain(request.values.items(), request.view_args.items())}
diff --git a/tests/test_utils/www.py b/tests/test_utils/www.py
index 2f48c304b4..8491d54094 100644
--- a/tests/test_utils/www.py
+++ b/tests/test_utils/www.py
@@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations
+import ast
from unittest import mock
from airflow.models import Log
@@ -73,3 +74,55 @@ def _check_last_log(session, dag_id, event, execution_date):
assert len(logs) >= 1
assert logs[0].extra
session.query(Log).delete()
+
+
+def _check_last_log_masked_connection(session, dag_id, event, execution_date):
+ logs = (
+ session.query(
+ Log.dag_id,
+ Log.task_id,
+ Log.event,
+ Log.execution_date,
+ Log.owner,
+ Log.extra,
+ )
+ .filter(
+ Log.dag_id == dag_id,
+ Log.event == event,
+ Log.execution_date == execution_date,
+ )
+ .order_by(Log.dttm.desc())
+ .limit(5)
+ .all()
+ )
+ assert len(logs) >= 1
+ extra = ast.literal_eval(logs[0].extra)
+ for k, v in extra:
+ if k == "password":
+ assert v == "***"
+ if k == "extra":
+ assert v == '{"x_secret": "***", "y_secret": "***"}'
+
+
+def _check_last_log_masked_variable(session, dag_id, event, execution_date):
+ logs = (
+ session.query(
+ Log.dag_id,
+ Log.task_id,
+ Log.event,
+ Log.execution_date,
+ Log.owner,
+ Log.extra,
+ )
+ .filter(
+ Log.dag_id == dag_id,
+ Log.event == event,
+ Log.execution_date == execution_date,
+ )
+ .order_by(Log.dttm.desc())
+ .limit(5)
+ .all()
+ )
+ assert len(logs) >= 1
+ extra_dict = ast.literal_eval(logs[0].extra)
+ assert extra_dict == [("key", "x_secret"), ("val", "***")]
diff --git a/tests/www/views/test_views_connection.py b/tests/www/views/test_views_connection.py
index 28d3f9570f..a884bbe793 100644
--- a/tests/www/views/test_views_connection.py
+++ b/tests/www/views/test_views_connection.py
@@ -28,7 +28,7 @@ from airflow.models import Connection
from airflow.utils.session import create_session
from airflow.www.extensions import init_views
from airflow.www.views import ConnectionFormWidget, ConnectionModelView
-from tests.test_utils.www import _check_last_log, check_content_in_response
+from tests.test_utils.www import _check_last_log, _check_last_log_masked_connection, check_content_in_response
CONNECTION = {
"conn_id": "test_conn",
@@ -40,6 +40,12 @@ CONNECTION = {
"password": "admin",
}
+CONNECTION_WITH_EXTRA = CONNECTION.update(
+ {
+ "extra": '{"x_secret": "testsecret","y_secret": "test"}',
+ }
+)
+
@pytest.fixture(autouse=True)
def clear_connections():
@@ -54,6 +60,12 @@ def test_create_connection(admin_client, session):
_check_last_log(session, dag_id=None, event="connection.create", execution_date=None)
+def test_action_logging_connection_masked_secrets(session, admin_client):
+ init_views.init_connection_form()
+ admin_client.post("/connection/add", data=CONNECTION_WITH_EXTRA, follow_redirects=True)
+ _check_last_log_masked_connection(session, dag_id=None, event="connection.create", execution_date=None)
+
+
def test_prefill_form_null_extra():
mock_form = mock.Mock()
mock_form.data = {"conn_id": "test", "extra": None, "conn_type": "test"}
diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py
index 607249f04e..13ede6273d 100644
--- a/tests/www/views/test_views_decorators.py
+++ b/tests/www/views/test_views_decorators.py
@@ -28,8 +28,8 @@ from airflow.utils.state import State
from airflow.utils.types import DagRunType
from airflow.www import app
from airflow.www.views import action_has_dag_edit_access
-from tests.test_utils.db import clear_db_runs
-from tests.test_utils.www import _check_last_log, check_content_in_response
+from tests.test_utils.db import clear_db_runs, clear_db_variables
+from tests.test_utils.www import _check_last_log, _check_last_log_masked_variable, check_content_in_response
EXAMPLE_DAG_DEFAULT_DATE = timezone.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
@@ -86,6 +86,13 @@ def dagruns(bash_dag, sub_dag, xcom_dag):
clear_db_runs()
+@pytest.fixture(autouse=True)
+def clean_db():
+ clear_db_variables()
+ yield
+ clear_db_variables()
+
+
@action_has_dag_edit_access
def some_view_action_which_requires_dag_edit_access(*args) -> bool:
return True
@@ -156,11 +163,17 @@ def delete_variable(session, key):
def test_action_logging_variables_post(session, admin_client):
- form = dict(key="random", value="random")
+ form = dict(key="random", val="random")
admin_client.post("/variable/add", data=form)
session.commit()
_check_last_log(session, dag_id=None, event="variable.create", execution_date=None)
- delete_variable(session, key="random")
+
+
+def test_action_logging_variables_masked_secrets(session, admin_client):
+ form = dict(key="x_secret", val="randomval")
+ admin_client.post("/variable/add", data=form)
+ session.commit()
+ _check_last_log_masked_variable(session, dag_id=None, event="variable.create", execution_date=None)
def test_calendar(admin_client, dagruns):
[airflow] 02/05: Fix failing test case for db clean in newer sqlalchemy (#28004)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 52c2fa8824dbb90abb871775f0f7cd36a97eb59e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Nov 30 11:47:05 2022 +0100
Fix failing test case for db clean in newer sqlalchemy (#28004)
Fixing of ambiguity of treating "*" in SQLAlchemy 4.1.42 made our
tests to fail, because they were trying to execute the query
from aliased table as a standalone queryi for verificaiton,
where in "reality" the same query would be executed as part of "CREATE
AS" statement.
The tests started to fail with the new SQLAlchemy and the fix was
to change our tests to also run "CREATE AS" statement and count
number of rows in the created temporary table.
(cherry picked from commit 122d60b5b4547f7380d58eea148552607264122e)
---
tests/utils/test_db_cleanup.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/tests/utils/test_db_cleanup.py b/tests/utils/test_db_cleanup.py
index 422bec37f3..c505408a3b 100644
--- a/tests/utils/test_db_cleanup.py
+++ b/tests/utils/test_db_cleanup.py
@@ -30,7 +30,7 @@ from sqlalchemy.ext.declarative import DeclarativeMeta
from airflow.models import DagModel, DagRun, TaskInstance
from airflow.operators.python import PythonOperator
-from airflow.utils.db_cleanup import _build_query, _cleanup_table, config_dict, run_cleanup
+from airflow.utils.db_cleanup import CreateTableAs, _build_query, _cleanup_table, config_dict, run_cleanup
from airflow.utils.session import create_session
from tests.test_utils.db import clear_db_dags, clear_db_datasets, clear_db_runs, drop_tables_with_prefix
@@ -172,6 +172,7 @@ class TestDBCleanup:
num_tis=10,
external_trigger=external_trigger,
)
+ target_table_name = "_airflow_temp_table_name"
with create_session() as session:
clean_before_date = base_date.add(**date_add_kwargs)
query = _build_query(
@@ -179,7 +180,11 @@ class TestDBCleanup:
clean_before_timestamp=clean_before_date,
session=session,
)
- assert len(query.all()) == expected_to_delete
+ stmt = CreateTableAs(target_table_name, query.selectable)
+ session.execute(stmt)
+ res = session.execute(f"SELECT COUNT(1) FROM {target_table_name}")
+ for row in res:
+ assert row[0] == expected_to_delete
@pytest.mark.parametrize(
"table_name, date_add_kwargs, expected_to_delete, external_trigger",