You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by jh...@apache.org on 2021/06/10 20:57:18 UTC
[airflow] branch v2-1-test updated (779811e -> abc86c4)
This is an automated email from the ASF dual-hosted git repository.
jhtimmins pushed a change to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 779811e Fill the "job_id" field for `airflow task run` without `--local`/`--raw` for KubeExecutor (#16108)
new 789aace Ensure that we don't try to mask empty string in logs (#16057)
new fb714d6 set max tree width to 1200px (#16067)
new abc86c4 Don't fail to log if we can't redact something (#16118)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/utils/log/secrets_masker.py | 53 ++++++++++++++++++++++------------
airflow/www/static/js/tree.js | 4 ++-
tests/utils/log/test_secrets_masker.py | 24 +++++++++++++++
3 files changed, 62 insertions(+), 19 deletions(-)
[airflow] 01/03: Ensure that we don't try to mask empty string in
logs (#16057)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhtimmins pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 789aacea36820e1e72c7962b539d57c36134910c
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Tue May 25 19:31:22 2021 +0100
Ensure that we don't try to mask empty string in logs (#16057)
Although `Connection.password` being empty was guarded against, there
are other possible cases (such as an extra field) that wasn't guarded
against, which ended up with this in the logs:
WARNING - ***-***-***-*** ***L***o***g***g***i***n***g*** ***e***r***r***o***r*** ***-***-***-***
Oops!
(cherry picked from commit 8814a59a5bf54dd17aef21eefd0900703330c22c)
[airflow] 02/03: set max tree width to 1200px (#16067)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhtimmins pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit fb714d6db9e28de113014364365202c82eb0ce22
Author: Brent Bovenzi <br...@gmail.com>
AuthorDate: Tue May 25 16:20:31 2021 -0400
set max tree width to 1200px (#16067)
the totalwidth of the tree view will depend on the window size like before, but max out at 1200px
(cherry picked from commit f2aa9b58cb012a3bc347f43baeaa41ecdece4cbf)
---
airflow/www/static/js/tree.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/airflow/www/static/js/tree.js b/airflow/www/static/js/tree.js
index 04702a7..cc8276d 100644
--- a/airflow/www/static/js/tree.js
+++ b/airflow/www/static/js/tree.js
@@ -70,7 +70,9 @@ document.addEventListener('DOMContentLoaded', () => {
if (node.depth > treeDepth) treeDepth = node.depth;
});
treeDepth += 1;
- const squareX = window.innerWidth - (data.instances.length * squareSize) - (treeDepth * 50);
+
+ const innerWidth = window.innerWidth > 1200 ? 1200 : window.innerWidth;
+ const squareX = innerWidth - (data.instances.length * squareSize) - (treeDepth * 50);
const squareSpacing = 2;
const margin = {
[airflow] 03/03: Don't fail to log if we can't redact something
(#16118)
Posted by jh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jhtimmins pushed a commit to branch v2-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit abc86c4bd96318b40615f8f173e08208a4756905
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Mon Jun 7 09:27:01 2021 +0100
Don't fail to log if we can't redact something (#16118)
Rather than dying with an exception, catch it and warn about that,
asking users to report it to us.
Additionally handle the specific case where a file handle/IO object is
logged -- we definitely don't want to iterate over that!
(cherry picked from commit 57bd6fb2925a7d505a80b83140811b94b363f49c)
---
airflow/utils/log/secrets_masker.py | 53 ++++++++++++++++++++++------------
tests/utils/log/test_secrets_masker.py | 24 +++++++++++++++
2 files changed, 59 insertions(+), 18 deletions(-)
diff --git a/airflow/utils/log/secrets_masker.py b/airflow/utils/log/secrets_masker.py
index 6df8d39..b3ccfdb 100644
--- a/airflow/utils/log/secrets_masker.py
+++ b/airflow/utils/log/secrets_masker.py
@@ -16,6 +16,7 @@
# under the License.
"""Mask sensitive information from logs"""
import collections
+import io
import logging
import re
from typing import TYPE_CHECKING, Iterable, Optional, Set, TypeVar, Union
@@ -40,6 +41,10 @@ if TYPE_CHECKING:
RedactableItem = TypeVar('RedctableItem')
+
+log = logging.getLogger(__name__)
+
+
DEFAULT_SENSITIVE_FIELDS = frozenset(
{
'password',
@@ -186,24 +191,36 @@ class SecretsMasker(logging.Filter):
is redacted.
"""
- if name and should_hide_value_for_key(name):
- return self._redact_all(item)
-
- if isinstance(item, dict):
- return {dict_key: self.redact(subval, dict_key) for dict_key, subval in item.items()}
- elif isinstance(item, str):
- if self.replacer:
- # We can't replace specific values, but the key-based redacting
- # can still happen, so we can't short-circuit, we need to walk
- # the structure.
- return self.replacer.sub('***', item)
- return item
- elif isinstance(item, (tuple, set)):
- # Turn set in to tuple!
- return tuple(self.redact(subval) for subval in item)
- elif isinstance(item, Iterable):
- return list(self.redact(subval) for subval in item)
- else:
+ try:
+ if name and should_hide_value_for_key(name):
+ return self._redact_all(item)
+
+ if isinstance(item, dict):
+ return {dict_key: self.redact(subval, dict_key) for dict_key, subval in item.items()}
+ elif isinstance(item, str):
+ if self.replacer:
+ # We can't replace specific values, but the key-based redacting
+ # can still happen, so we can't short-circuit, we need to walk
+ # the structure.
+ return self.replacer.sub('***', item)
+ return item
+ elif isinstance(item, (tuple, set)):
+ # Turn set in to tuple!
+ return tuple(self.redact(subval) for subval in item)
+ elif isinstance(item, io.IOBase):
+ return item
+ elif isinstance(item, Iterable):
+ return list(self.redact(subval) for subval in item)
+ else:
+ return item
+ except Exception as e: # pylint: disable=broad-except
+ log.warning(
+ "Unable to redact %r, please report this via <https://github.com/apache/airflow/issues>. "
+ "Error was: %s: %s",
+ item,
+ type(e).__name__,
+ str(e),
+ )
return item
# pylint: enable=too-many-return-statements
diff --git a/tests/utils/log/test_secrets_masker.py b/tests/utils/log/test_secrets_masker.py
index 8c88bdd..24e86c1 100644
--- a/tests/utils/log/test_secrets_masker.py
+++ b/tests/utils/log/test_secrets_masker.py
@@ -72,6 +72,22 @@ class TestSecretsMasker:
assert caplog.text == "INFO Cannot connect to user:***\n"
+ def test_non_redactable(self, logger, caplog):
+ class NonReactable:
+ def __iter__(self):
+ raise RuntimeError("force fail")
+
+ def __repr__(self):
+ return "<NonRedactable>"
+
+ logger.info("Logging %s", NonReactable())
+
+ assert caplog.messages == [
+ "Unable to redact <NonRedactable>, please report this via "
+ + "<https://github.com/apache/airflow/issues>. Error was: RuntimeError: force fail",
+ "Logging <NonRedactable>",
+ ]
+
def test_extra(self, logger, caplog):
logger.handlers[0].formatter = ShortExcFormatter("%(levelname)s %(message)s %(conn)s")
logger.info("Cannot connect", extra={'conn': "user:password"})
@@ -202,6 +218,14 @@ class TestSecretsMasker:
assert filt.redact(value, name) == expected
+ def test_redact_filehandles(self, caplog):
+ filt = SecretsMasker()
+ with open("/dev/null", "w") as handle:
+ assert filt.redact(handle, None) == handle
+
+ # We shouldn't have logged a warning here
+ assert caplog.messages == []
+
class TestShouldHideValueForKey:
@pytest.mark.parametrize(