You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/08/14 17:01:15 UTC
[airflow] branch openlineage-do-not-redact-proxy updated (1663fd409e -> 40cbbdc924)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch openlineage-do-not-redact-proxy
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 1663fd409e openlineage: do not try to redact Proxy objects from deprecated config
discard 1018746970 openlineage: don't run task instance listener in executor
new 40cbbdc924 openlineage: do not try to redact Proxy objects from deprecated config
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1663fd409e)
\
N -- N -- N refs/heads/openlineage-do-not-redact-proxy (40cbbdc924)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/providers/openlineage/plugins/listener.py | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
[airflow] 01/01: openlineage: do not try to redact Proxy objects from deprecated config
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch openlineage-do-not-redact-proxy
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 40cbbdc9244804b412f04d70fef4c6c9e611b484
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Mon Aug 14 18:19:16 2023 +0200
openlineage: do not try to redact Proxy objects from deprecated config
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/providers/openlineage/utils/utils.py | 60 ++++++++++++++---------
tests/providers/openlineage/plugins/test_utils.py | 6 +++
2 files changed, 42 insertions(+), 24 deletions(-)
diff --git a/airflow/providers/openlineage/utils/utils.py b/airflow/providers/openlineage/utils/utils.py
index 20b9afef49..3ff7610c4f 100644
--- a/airflow/providers/openlineage/utils/utils.py
+++ b/airflow/providers/openlineage/utils/utils.py
@@ -38,6 +38,7 @@ from airflow.providers.openlineage.plugins.facets import (
AirflowMappedTaskRunFacet,
AirflowRunFacet,
)
+from airflow.utils.context import AirflowContextDeprecationWarning
from airflow.utils.log.secrets_masker import Redactable, Redacted, SecretsMasker, should_hide_value_for_key
if TYPE_CHECKING:
@@ -345,29 +346,40 @@ class OpenLineageRedactor(SecretsMasker):
if depth > max_depth:
return item
try:
- if name and should_hide_value_for_key(name):
- return self._redact_all(item, depth, max_depth)
- if attrs.has(type(item)):
- # TODO: fixme when mypy gets compatible with new attrs
- for dict_key, subval in attrs.asdict(item, recurse=False).items(): # type: ignore[arg-type]
- if _is_name_redactable(dict_key, item):
- setattr(
- item,
- dict_key,
- self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
- )
- return item
- elif is_json_serializable(item) and hasattr(item, "__dict__"):
- for dict_key, subval in item.__dict__.items():
- if _is_name_redactable(dict_key, item):
- setattr(
- item,
- dict_key,
- self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
- )
- return item
- else:
- return super()._redact(item, name, depth, max_depth)
+ # It's impossible to check the type of variable in a dict without accessing it, and
+ # this already causes warning - so suppress it
+ with suppress(AirflowContextDeprecationWarning):
+ if type(item).__name__ == "Proxy":
+ # Those are deprecated values in _DEPRECATION_REPLACEMENTS
+ # in airflow.utils.context.Context
+ return "<<non-redactable: Proxy>>"
+ if name and should_hide_value_for_key(name):
+ return self._redact_all(item, depth, max_depth)
+ if attrs.has(type(item)):
+ # TODO: fixme when mypy gets compatible with new attrs
+ for dict_key, subval in attrs.asdict(
+ item, recurse=False # type: ignore[arg-type]
+ ).items():
+ if _is_name_redactable(dict_key, item):
+ setattr(
+ item,
+ dict_key,
+ self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
+ )
+ return item
+ elif is_json_serializable(item) and hasattr(item, "__dict__"):
+ for dict_key, subval in item.__dict__.items():
+ if type(subval).__name__ == "Proxy":
+ return "<<non-redactable: Proxy>>"
+ if _is_name_redactable(dict_key, item):
+ setattr(
+ item,
+ dict_key,
+ self._redact(subval, name=dict_key, depth=(depth + 1), max_depth=max_depth),
+ )
+ return item
+ else:
+ return super()._redact(item, name, depth, max_depth)
except Exception as e:
log.warning(
"Unable to redact %s. Error was: %s: %s",
@@ -375,7 +387,7 @@ class OpenLineageRedactor(SecretsMasker):
type(e).__name__,
str(e),
)
- return item
+ return item
def is_json_serializable(item):
diff --git a/tests/providers/openlineage/plugins/test_utils.py b/tests/providers/openlineage/plugins/test_utils.py
index afbc40ee4c..59abb43e09 100644
--- a/tests/providers/openlineage/plugins/test_utils.py
+++ b/tests/providers/openlineage/plugins/test_utils.py
@@ -171,6 +171,9 @@ def test_redact_with_exclusions(monkeypatch):
def __init__(self):
self.password = "passwd"
+ class Proxy:
+ pass
+
def default(self, o):
if isinstance(o, NotMixin):
return o.__dict__
@@ -180,6 +183,9 @@ def test_redact_with_exclusions(monkeypatch):
monkeypatch.setattr(JSONEncoder, "default", default)
assert redactor.redact(NotMixin()).password == "***"
+ assert redactor.redact(Proxy()) == "<<non-redactable: Proxy>>"
+ assert redactor.redact({"a": "a", "b": Proxy()}) == {"a": "a", "b": "<<non-redactable: Proxy>>"}
+
class Mixined(RedactMixin):
_skip_redact = ["password"]