You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2021/12/08 18:03:27 UTC
[airflow] branch v2-2-test updated (44430b4 -> e627308)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a change to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 44430b4 Fix infinite recursion on redact log (#20039)
new 486dbae update upper bound for MarkupSafe (#19953)
new 2ddb431 Lift off upper bound for MarkupSafe (#20113)
new d2eab1d fixing #19028 by moving chown to use sudo (#20114)
new 6be9609 Fix log link in gantt view (#20121)
new e627308 Fix missing dot (#20141)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airflow/task/task_runner/base_task_runner.py | 15 +++++----------
airflow/www/templates/airflow/dags.html | 2 +-
airflow/www/views.py | 4 ++--
setup.cfg | 2 +-
tests/jobs/test_local_task_job.py | 6 ++++--
tests/task/task_runner/test_base_task_runner.py | 11 +++++------
tests/www/views/test_views_rendered.py | 18 +++++++++++-------
7 files changed, 29 insertions(+), 29 deletions(-)
[airflow] 04/05: Fix log link in gantt view (#20121)
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 6be960981ed486fb06f3e7d214e0a08a13dac0c1
Author: Jed Cunningham <66...@users.noreply.github.com>
AuthorDate: Wed Dec 8 09:11:43 2021 -0700
Fix log link in gantt view (#20121)
(cherry picked from commit f59decd391b75c509020e603e5857bb63ec891be)
---
airflow/www/views.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/airflow/www/views.py b/airflow/www/views.py
index a217758..5178283 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2875,7 +2875,7 @@ class Airflow(AirflowBaseView):
task_dict['end_date'] = task_dict['end_date'] or timezone.utcnow()
task_dict['extraLinks'] = dag.get_task(ti.task_id).extra_links
task_dict['try_number'] = try_count
- task_dict['execution_date'] = dttm
+ task_dict['execution_date'] = dttm.isoformat()
tasks.append(task_dict)
tf_count = 0
@@ -2897,7 +2897,7 @@ class Airflow(AirflowBaseView):
task_dict['operator'] = task.task_type
task_dict['try_number'] = try_count
task_dict['extraLinks'] = task.extra_links
- task_dict['execution_date'] = dttm
+ task_dict['execution_date'] = dttm.isoformat()
tasks.append(task_dict)
task_names = [ti.task_id for ti in tis]
[airflow] 05/05: Fix missing dot (#20141)
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e627308b31c80a02b5809f0d52a82fe4fbf4406b
Author: Bas Harenslak <Ba...@users.noreply.github.com>
AuthorDate: Wed Dec 8 17:35:58 2021 +0100
Fix missing dot (#20141)
Co-authored-by: Bas Harenslak <ba...@astronomer.io>
(cherry picked from commit 01c7c4d8884ebb4a26a2d41f7bd42a8c93378895)
---
airflow/www/templates/airflow/dags.html | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html
index 5c2a063..27ff989 100644
--- a/airflow/www/templates/airflow/dags.html
+++ b/airflow/www/templates/airflow/dags.html
@@ -63,7 +63,7 @@
{{ super() }}
{% if sqlite_warning | default(true) %}
{% call show_message(category='warning', dismissible=false) %}
- Do not use <b>SQLite</b> as metadata DB in production – it should only be used for dev/testing
+ Do not use <b>SQLite</b> as metadata DB in production – it should only be used for dev/testing.
We recommend using Postgres or MySQL.
<a href={{ get_docs_url("howto/set-up-database.html") }}><b>Click here</b></a> for more information.
{% endcall %}
[airflow] 02/05: Lift off upper bound for MarkupSafe (#20113)
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2ddb431161834073fd6a7ad755ee49f73a71961b
Author: Tajinder Singh <ts...@gmail.com>
AuthorDate: Tue Dec 7 20:26:31 2021 -0500
Lift off upper bound for MarkupSafe (#20113)
Per discussion and guidance from #19753, opening this PR for review. Based on if all the tests pass, this could be reviewed further. Resolves #19761.
(cherry picked from commit bcacc51a16697a656357c29c7a40240e422e4bf9)
---
setup.cfg | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.cfg b/setup.cfg
index ff95f29..fcd09b2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -122,7 +122,7 @@ install_requires =
lazy-object-proxy
lockfile>=0.12.2
markdown>=2.5.2, <4.0
- markupsafe>=1.1.1, <=2.0
+ markupsafe>=1.1.1
marshmallow-oneofschema>=2.0.1
# Required by vendored-in connexion
openapi-spec-validator>=0.2.4
[airflow] 01/05: update upper bound for MarkupSafe (#19953)
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 486dbae302afb307cb176464df239537ce021194
Author: Tajinder Singh <ts...@gmail.com>
AuthorDate: Mon Dec 6 09:28:14 2021 -0500
update upper bound for MarkupSafe (#19953)
Co-authored-by: Tzu-ping Chung <tp...@astronomer.io>
(cherry picked from commit ba6b7c7424f6b5ea2c1464304be8738ea482f8c1)
---
setup.cfg | 2 +-
tests/www/views/test_views_rendered.py | 18 +++++++++++-------
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/setup.cfg b/setup.cfg
index e234a04..ff95f29 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -122,7 +122,7 @@ install_requires =
lazy-object-proxy
lockfile>=0.12.2
markdown>=2.5.2, <4.0
- markupsafe>=1.1.1, <2.0
+ markupsafe>=1.1.1, <=2.0
marshmallow-oneofschema>=2.0.1
# Required by vendored-in connexion
openapi-spec-validator>=0.2.4
diff --git a/tests/www/views/test_views_rendered.py b/tests/www/views/test_views_rendered.py
index b749fa1..f88db34 100644
--- a/tests/www/views/test_views_rendered.py
+++ b/tests/www/views/test_views_rendered.py
@@ -28,7 +28,7 @@ from airflow.utils.session import create_session
from airflow.utils.state import DagRunState, TaskInstanceState
from airflow.utils.types import DagRunType
from tests.test_utils.db import clear_db_dags, clear_db_runs, clear_rendered_ti_fields
-from tests.test_utils.www import check_content_in_response, check_content_not_in_response
+from tests.test_utils.www import check_content_in_response
DEFAULT_DATE = timezone.datetime(2020, 3, 1)
@@ -154,13 +154,17 @@ def test_user_defined_filter_and_macros_raise_error(admin_client, create_dag_run
url = f'rendered-templates?task_id=task2&dag_id=testdag&execution_date={quote_plus(str(DEFAULT_DATE))}'
resp = admin_client.get(url, follow_redirects=True)
+ assert resp.status_code == 200
- check_content_not_in_response("echo Hello Apache Airflow", resp)
- check_content_in_response(
+ resp_html: str = resp.data.decode("utf-8")
+ assert "echo Hello Apache Airflow" in resp_html
+ assert (
"Webserver does not have access to User-defined Macros or Filters when "
"Dag Serialization is enabled. Hence for the task that have not yet "
"started running, please use 'airflow tasks render' for "
- "debugging the rendering of template_fields.<br><br>OriginalError: no "
- "filter named 'hello'",
- resp,
- )
+ "debugging the rendering of template_fields.<br><br>"
+ ) in resp_html
+
+ # MarkupSafe changed the exception detail from 'no filter named' to
+ # 'No filter named' in 2.0 (I think), so we normalize for comparison.
+ assert "originalerror: no filter named 'hello'" in resp_html.lower()
[airflow] 03/05: fixing #19028 by moving chown to use sudo (#20114)
Posted by je...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit d2eab1de0c3ee93e5e1c7830ffcfdb534657fc21
Author: Paul Lockaby <63...@users.noreply.github.com>
AuthorDate: Wed Dec 8 08:00:14 2021 -0800
fixing #19028 by moving chown to use sudo (#20114)
* fixing #19028 by having chown be in a sudo call
* removing unused import
* trying to clean up a test
* combine sudo chown calls
* force exception when chown fails
* Update tests/task/task_runner/test_base_task_runner.py
* Fix tests
* Fix formatting
Co-authored-by: Ash Berlin-Taylor <as...@firemirror.com>
Co-authored-by: Ash Berlin-Taylor <as...@apache.org>
(cherry picked from commit b37c0efabd29b9f20ba05c0e1281de22809e0624)
---
airflow/task/task_runner/base_task_runner.py | 15 +++++----------
tests/jobs/test_local_task_job.py | 6 ++++--
tests/task/task_runner/test_base_task_runner.py | 11 +++++------
3 files changed, 14 insertions(+), 18 deletions(-)
diff --git a/airflow/task/task_runner/base_task_runner.py b/airflow/task/task_runner/base_task_runner.py
index 28bb847..5551508 100644
--- a/airflow/task/task_runner/base_task_runner.py
+++ b/airflow/task/task_runner/base_task_runner.py
@@ -19,7 +19,6 @@
import os
import subprocess
import threading
-from pwd import getpwnam
from tempfile import NamedTemporaryFile
from typing import Optional, Union
@@ -58,6 +57,8 @@ class BaseTaskRunner(LoggingMixin):
except AirflowConfigException:
self.run_as_user = None
+ self._error_file = NamedTemporaryFile(delete=True)
+
# Add sudo commands to change user if we need to. Needed to handle SubDagOperator
# case using a SequentialExecutor.
self.log.debug("Planning to run as the %s user", self.run_as_user)
@@ -69,7 +70,9 @@ class BaseTaskRunner(LoggingMixin):
cfg_path = tmp_configuration_copy(chmod=0o600, include_env=True, include_cmds=True)
# Give ownership of file to user; only they can read and write
- subprocess.call(['sudo', 'chown', self.run_as_user, cfg_path], close_fds=True)
+ subprocess.check_call(
+ ['sudo', 'chown', self.run_as_user, cfg_path, self._error_file.name], close_fds=True
+ )
# propagate PYTHONPATH environment variable
pythonpath_value = os.environ.get(PYTHONPATH_VAR, '')
@@ -85,14 +88,6 @@ class BaseTaskRunner(LoggingMixin):
# - the runner can read/execute those values as it needs
cfg_path = tmp_configuration_copy(chmod=0o600, include_env=False, include_cmds=False)
- self._error_file = NamedTemporaryFile(delete=True)
- if self.run_as_user:
- try:
- os.chown(self._error_file.name, getpwnam(self.run_as_user).pw_uid, -1)
- except KeyError:
- # No user `run_as_user` found
- pass
-
self._cfg_path = cfg_path
self._command = (
popen_prepend
diff --git a/tests/jobs/test_local_task_job.py b/tests/jobs/test_local_task_job.py
index f23a94f..d9d677d 100644
--- a/tests/jobs/test_local_task_job.py
+++ b/tests/jobs/test_local_task_job.py
@@ -148,8 +148,9 @@ class TestLocalTaskJob:
with pytest.raises(AirflowException):
job1.heartbeat_callback()
+ @mock.patch('subprocess.check_call')
@mock.patch('airflow.jobs.local_task_job.psutil')
- def test_localtaskjob_heartbeat_with_run_as_user(self, psutil_mock, dag_maker):
+ def test_localtaskjob_heartbeat_with_run_as_user(self, psutil_mock, _, dag_maker):
session = settings.Session()
with dag_maker('test_localtaskjob_heartbeat'):
op1 = DummyOperator(task_id='op1', run_as_user='myuser')
@@ -190,8 +191,9 @@ class TestLocalTaskJob:
job1.heartbeat_callback()
@conf_vars({('core', 'default_impersonation'): 'testuser'})
+ @mock.patch('subprocess.check_call')
@mock.patch('airflow.jobs.local_task_job.psutil')
- def test_localtaskjob_heartbeat_with_default_impersonation(self, psutil_mock, dag_maker):
+ def test_localtaskjob_heartbeat_with_default_impersonation(self, psutil_mock, _, dag_maker):
session = settings.Session()
with dag_maker('test_localtaskjob_heartbeat'):
op1 = DummyOperator(task_id='op1')
diff --git a/tests/task/task_runner/test_base_task_runner.py b/tests/task/task_runner/test_base_task_runner.py
index 499bba7..a0bd0ac 100644
--- a/tests/task/task_runner/test_base_task_runner.py
+++ b/tests/task/task_runner/test_base_task_runner.py
@@ -25,10 +25,11 @@ from airflow.task.task_runner.base_task_runner import BaseTaskRunner
@pytest.mark.parametrize(["impersonation"], (("nobody",), (None,)))
-@mock.patch('subprocess.call')
-@mock.patch('os.chown')
+@mock.patch('subprocess.check_call')
@mock.patch('airflow.task.task_runner.base_task_runner.tmp_configuration_copy')
-def test_config_copy_mode(tmp_configuration_copy, chown, subprocess_call, dag_maker, impersonation):
+def test_config_copy_mode(tmp_configuration_copy, subprocess_call, dag_maker, impersonation):
+ tmp_configuration_copy.return_value = "/tmp/some-string"
+
with dag_maker("test"):
BaseOperator(task_id="task_1", run_as_user=impersonation)
@@ -45,10 +46,8 @@ def test_config_copy_mode(tmp_configuration_copy, chown, subprocess_call, dag_ma
tmp_configuration_copy.assert_called_with(chmod=0o600, include_env=includes, include_cmds=includes)
if impersonation:
- chown.assert_called()
subprocess_call.assert_called_with(
- ['sudo', 'chown', impersonation, tmp_configuration_copy.return_value], close_fds=True
+ ['sudo', 'chown', impersonation, "/tmp/some-string", runner._error_file.name], close_fds=True
)
else:
- chown.assert_not_called()
subprocess_call.not_assert_called()