You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/11/20 19:41:10 UTC
[airflow] branch v1-10-test updated (55cc1a5 -> e348467)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a change to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
discard 55cc1a5 Change log level for User's session to DEBUG (#12414)
discard 4086736 Handle outdated webserver session timeout gracefully. (#12332)
discard 765b696 Unify user session lifetime configuration (#11970)
discard 8245965 Merge invalid_kubernetes_config rule and pod_template_file rule
discard 44868bc TimeSensor should respect DAG timezone (#9882)
discard a20cae9 TimeSensor should respect the default_timezone config (#9699)
discard 21af507 Show Generic Error for Charts & Query View in old UI (#12495)
discard e28f412 Add 1.10.13 Changelog
new e0ade6a Show Generic Error for Charts & Query View in old UI (#12495)
new 652b0c1 TimeSensor should respect the default_timezone config (#9699)
new cad03ba TimeSensor should respect DAG timezone (#9882)
new 37da5a1 Merge invalid_kubernetes_config rule and pod_template_file rule
new 7e0ab06 Unify user session lifetime configuration (#11970)
new fbaddaa Handle outdated webserver session timeout gracefully. (#12332)
new f91966d Change log level for User's session to DEBUG (#12414)
new 560ae75 Add Note about hook, Operators/Senesors deprecation in Updating.md
new e348467 Add 1.10.13 Changelog
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (55cc1a5)
\
N -- N -- N refs/heads/v1-10-test (e348467)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG.txt | 11 +++++++++++
UPDATING.md | 8 ++++++++
tests/test_local_settings/test_local_settings.py | 9 ++++++++-
3 files changed, 27 insertions(+), 1 deletion(-)
[airflow] 07/09: Change log level for User's session to DEBUG
(#12414)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f91966dd4ce6da827f8292c429c15fddf57f2fa6
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Tue Nov 17 20:48:24 2020 +0000
Change log level for User's session to DEBUG (#12414)
This line was logged too often -- too chatty
(cherry picked from commit bf3ead13cc21e69cf4ffcfea4e184e311cecee60)
---
airflow/settings.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/airflow/settings.py b/airflow/settings.py
index f4cca1a..2aedb72 100644
--- a/airflow/settings.py
+++ b/airflow/settings.py
@@ -406,7 +406,7 @@ def get_session_lifetime_config():
session_lifetime_days = 30
session_lifetime_minutes = minutes_per_day * session_lifetime_days
- logging.info('User session lifetime is set to %s minutes.', session_lifetime_minutes)
+ logging.debug('User session lifetime is set to %s minutes.', session_lifetime_minutes)
return int(session_lifetime_minutes)
[airflow] 06/09: Handle outdated webserver session timeout
gracefully. (#12332)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit fbaddaa4fff86251ab9d9fe9bb1663306c5131c4
Author: James Timmins <ja...@astronomer.io>
AuthorDate: Mon Nov 16 11:38:28 2020 -0800
Handle outdated webserver session timeout gracefully. (#12332)
(cherry picked from commit d4e1ff290f59f365163b0d969840f72746364e8e)
---
airflow/settings.py | 33 ++++++++++++++++++-
airflow/www/app.py | 4 +++
airflow/www_rbac/app.py | 19 +----------
tests/test_local_settings/test_local_settings.py | 40 +++++++++++++++++++++++-
tests/www/test_app.py | 33 +++++++++++++++++++
tests/www_rbac/test_app.py | 20 +-----------
6 files changed, 110 insertions(+), 39 deletions(-)
diff --git a/airflow/settings.py b/airflow/settings.py
index 0f35dc5..f4cca1a 100644
--- a/airflow/settings.py
+++ b/airflow/settings.py
@@ -26,10 +26,11 @@ import atexit
import json
import logging
import os
-import pendulum
import sys
+import warnings
from typing import Any
+import pendulum
from sqlalchemy import create_engine, exc
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
@@ -380,6 +381,36 @@ def prepare_syspath():
sys.path.append(PLUGINS_FOLDER)
+def get_session_lifetime_config():
+ """Gets session timeout configs and handles outdated configs gracefully."""
+ session_lifetime_minutes = conf.get('webserver', 'session_lifetime_minutes', fallback=None)
+ session_lifetime_days = conf.get('webserver', 'session_lifetime_days', fallback=None)
+ uses_deprecated_lifetime_configs = session_lifetime_days or conf.get(
+ 'webserver', 'force_logout_after', fallback=None
+ )
+
+ minutes_per_day = 24 * 60
+ default_lifetime_minutes = '43200'
+ if uses_deprecated_lifetime_configs and session_lifetime_minutes == default_lifetime_minutes:
+ warnings.warn(
+ '`session_lifetime_days` option from `[webserver]` section has been '
+ 'renamed to `session_lifetime_minutes`. The new option allows to configure '
+ 'session lifetime in minutes. The `force_logout_after` option has been removed '
+ 'from `[webserver]` section. Please update your configuration.',
+ category=DeprecationWarning,
+ )
+ if session_lifetime_days:
+ session_lifetime_minutes = minutes_per_day * int(session_lifetime_days)
+
+ if not session_lifetime_minutes:
+ session_lifetime_days = 30
+ session_lifetime_minutes = minutes_per_day * session_lifetime_days
+
+ logging.info('User session lifetime is set to %s minutes.', session_lifetime_minutes)
+
+ return int(session_lifetime_minutes)
+
+
def import_local_settings():
try:
import airflow_local_settings
diff --git a/airflow/www/app.py b/airflow/www/app.py
index 7d0dae7..58e82b9 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -60,6 +60,7 @@ def create_app(config=None, testing=False):
x_prefix=conf.getint("webserver", "PROXY_FIX_X_PREFIX", fallback=1)
)
app.secret_key = conf.get('webserver', 'SECRET_KEY')
+ app.config['PERMANENT_SESSION_LIFETIME'] = datetime.timedelta(minutes=settings.get_session_lifetime_config())
app.config['LOGIN_DISABLED'] = not conf.getboolean(
'webserver', 'AUTHENTICATE')
@@ -70,6 +71,9 @@ def create_app(config=None, testing=False):
if config:
app.config.from_mapping(config)
+ if 'SQLALCHEMY_ENGINE_OPTIONS' not in app.config:
+ app.config['SQLALCHEMY_ENGINE_OPTIONS'] = settings.prepare_engine_args()
+
csrf.init_app(app)
app.config['TESTING'] = testing
diff --git a/airflow/www_rbac/app.py b/airflow/www_rbac/app.py
index 0b1a8c7..a2ebf7b 100644
--- a/airflow/www_rbac/app.py
+++ b/airflow/www_rbac/app.py
@@ -18,7 +18,6 @@
# under the License.
#
import logging
-import sys
import socket
from datetime import timedelta
from typing import Any
@@ -62,23 +61,7 @@ def create_app(config=None, session=None, testing=False, app_name="Airflow"):
x_prefix=conf.getint("webserver", "PROXY_FIX_X_PREFIX", fallback=1)
)
app.secret_key = conf.get('webserver', 'SECRET_KEY')
-
- if conf.has_option('webserver', 'SESSION_LIFETIME_DAYS') or conf.has_option(
- 'webserver', 'FORCE_LOG_OUT_AFTER'
- ):
- logging.error(
- '`SESSION_LIFETIME_DAYS` option from `webserver` section has been '
- 'renamed to `SESSION_LIFETIME_MINUTES`. New option allows to configure '
- 'session lifetime in minutes. FORCE_LOG_OUT_AFTER option has been removed '
- 'from `webserver` section. Please update your configuration.'
- )
- # Stop gunicorn server https://github.com/benoitc/gunicorn/blob/20.0.4/gunicorn/arbiter.py#L526
- # sys.exit(4)
- else:
- session_lifetime_minutes = conf.getint('webserver', 'SESSION_LIFETIME_MINUTES', fallback=43200)
- logging.info('User session lifetime is set to %s minutes.', session_lifetime_minutes)
-
- app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(minutes=session_lifetime_minutes)
+ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(minutes=settings.get_session_lifetime_config())
app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
diff --git a/tests/test_local_settings/test_local_settings.py b/tests/test_local_settings/test_local_settings.py
index e69bd05..00ffe94 100644
--- a/tests/test_local_settings/test_local_settings.py
+++ b/tests/test_local_settings/test_local_settings.py
@@ -20,11 +20,19 @@
import os
import sys
import tempfile
-import unittest
+import six
+
from airflow.kubernetes import pod_generator
from kubernetes.client import ApiClient
import kubernetes.client.models as k8s
from tests.compat import MagicMock, Mock, mock, call, patch
+from tests.test_utils.config import conf_vars
+
+if six.PY2:
+ # Need `assertWarns` back-ported from unittest2
+ import unittest2 as unittest
+else:
+ import unittest
api_client = ApiClient()
@@ -442,6 +450,36 @@ class LocalSettingsTest(unittest.TestCase):
)
+class TestUpdatedConfigNames(unittest.TestCase):
+ @conf_vars(
+ {("webserver", "session_lifetime_days"): '5', ("webserver", "session_lifetime_minutes"): '43200'}
+ )
+ def test_updates_deprecated_session_timeout_config_val_when_new_config_val_is_default(self):
+ from airflow import settings
+
+ with self.assertWarns(DeprecationWarning):
+ session_lifetime_config = settings.get_session_lifetime_config()
+ minutes_in_five_days = 5 * 24 * 60
+ self.assertEqual(session_lifetime_config, minutes_in_five_days)
+
+ @conf_vars(
+ {("webserver", "session_lifetime_days"): '5', ("webserver", "session_lifetime_minutes"): '43201'}
+ )
+ def test_uses_updated_session_timeout_config_when_val_is_not_default(self):
+ from airflow import settings
+
+ session_lifetime_config = settings.get_session_lifetime_config()
+ self.assertEqual(session_lifetime_config, 43201)
+
+ @conf_vars({("webserver", "session_lifetime_days"): ''})
+ def test_uses_updated_session_timeout_config_by_default(self):
+ from airflow import settings
+
+ session_lifetime_config = settings.get_session_lifetime_config()
+ default_timeout_minutes = 30 * 24 * 60
+ self.assertEqual(session_lifetime_config, default_timeout_minutes)
+
+
class TestStatsWithAllowList(unittest.TestCase):
def setUp(self):
diff --git a/tests/www/test_app.py b/tests/www/test_app.py
index 56ec213..83df598 100644
--- a/tests/www/test_app.py
+++ b/tests/www/test_app.py
@@ -17,10 +17,14 @@
# specific language governing permissions and limitations
# under the License.
import unittest
+from datetime import timedelta
+import pytest
from werkzeug.middleware.proxy_fix import ProxyFix
+from airflow.settings import Session
from airflow.www_rbac import app as application
+from tests.compat import mock
from tests.test_utils.config import conf_vars
@@ -54,3 +58,32 @@ class TestApp(unittest.TestCase):
self.assertEqual(app.wsgi_app.x_host, 5)
self.assertEqual(app.wsgi_app.x_port, 6)
self.assertEqual(app.wsgi_app.x_prefix, 7)
+
+ @conf_vars({
+ ('core', 'sql_alchemy_pool_enabled'): 'True',
+ ('core', 'sql_alchemy_pool_size'): '3',
+ ('core', 'sql_alchemy_max_overflow'): '5',
+ ('core', 'sql_alchemy_pool_recycle'): '120',
+ ('core', 'sql_alchemy_pool_pre_ping'): 'True',
+ })
+ @mock.patch("airflow.www.app.app", None)
+ @pytest.mark.backend("mysql", "postgres")
+ def test_should_set_sqlalchemy_engine_options(self):
+ app, _ = application.create_app(session=Session, testing=True)
+ engine_params = {
+ 'pool_size': 3,
+ 'pool_recycle': 120,
+ 'pool_pre_ping': True,
+ 'max_overflow': 5
+ }
+ self.assertEqual(app.config['SQLALCHEMY_ENGINE_OPTIONS'], engine_params)
+
+ @conf_vars(
+ {
+ ('webserver', 'session_lifetime_minutes'): '3600',
+ }
+ )
+ @mock.patch("airflow.www.app.app", None)
+ def test_should_set_permanent_session_timeout(self):
+ app, _ = application.create_app(session=Session, testing=True)
+ self.assertEqual(app.config['PERMANENT_SESSION_LIFETIME'], timedelta(minutes=3600))
diff --git a/tests/www_rbac/test_app.py b/tests/www_rbac/test_app.py
index a0607cb..a1ac2d3 100644
--- a/tests/www_rbac/test_app.py
+++ b/tests/www_rbac/test_app.py
@@ -16,15 +16,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import json
import unittest
from datetime import timedelta
import pytest
-import six
from werkzeug.middleware.proxy_fix import ProxyFix
-from airflow.configuration import conf
from airflow.settings import Session
from airflow.www_rbac import app as application
from tests.compat import mock
@@ -79,8 +76,6 @@ class TestApp(unittest.TestCase):
'pool_pre_ping': True,
'max_overflow': 5
}
- if six.PY2:
- engine_params = json.dumps(engine_params)
self.assertEqual(app.config['SQLALCHEMY_ENGINE_OPTIONS'], engine_params)
@conf_vars(
@@ -90,18 +85,5 @@ class TestApp(unittest.TestCase):
)
@mock.patch("airflow.www_rbac.app.app", None)
def test_should_set_permanent_session_timeout(self):
- app, _ = application.cached_appbuilder(testing=True)
+ app, _ = application.create_app(session=Session, testing=True)
self.assertEqual(app.config['PERMANENT_SESSION_LIFETIME'], timedelta(minutes=3600))
-
- @conf_vars(
- {
- ('webserver', 'session_lifetime_days'): '30',
- ('webserver', 'force_log_out_after'): '30',
- }
- )
- @mock.patch("airflow.www_rbac.app.app", None)
- def test_should_stop_app_when_removed_options_are_provided(self):
- with self.assertRaises(SystemExit) as e:
- conf.remove_option('webserver', 'session_lifetime_minutes')
- application.cached_appbuilder(testing=True)
- self.assertEqual(e.exception.code, 4)
[airflow] 09/09: Add 1.10.13 Changelog
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e348467c14c54b474a841c5192c03ebaa0e3a8be
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Thu Nov 19 21:28:11 2020 +0000
Add 1.10.13 Changelog
---
CHANGELOG.txt | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 112 insertions(+)
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
index 4fb12de..40bbd7e 100644
--- a/CHANGELOG.txt
+++ b/CHANGELOG.txt
@@ -1,3 +1,114 @@
+Airflow 1.10.13, 2020-11-24
+----------------------------
+
+New Features
+""""""""""""
+
+- Add "already checked" to failed pods in K8sPodOperator (#11368)
+- Pass SQLAlchemy engine options to FAB based UI (#11395)
+- [AIRFLOW-4438] Add Gzip compression to S3_hook (#8571)
+- Add permission "extra_links" for Viewer role and above (#10719)
+- Add generate_yaml command to easily test KubernetesExecutor before deploying pods (#10677)
+- Add Secrets backend for Microsoft Azure Key Vault (#10898)
+
+Bug Fixes
+"""""""""
+
+- SkipMixin: Handle empty branches (#11120)
+- [AIRFLOW-5274] dag loading duration metric name too long (#5890)
+- Handle no Dagrun in DagrunIdDep (#8389) (#11343)
+- Fix Kubernetes Executor logs for long dag names (#10942)
+- Add on_kill support for the KubernetesPodOperator (#10666)
+- KubernetesPodOperator template fix (#10963)
+- Fix displaying of add serialized_dag table migration
+- Fix Start Date tooltip on DAGs page (#10637)
+- URL encode execution date in the Last Run link (#10595)
+- Fixes issue with affinity backcompat in Airflow 1.10
+- Fix KubernetesExecutor import in views.py
+- Fix issues with Gantt View (#12419)
+- Fix Entrypoint and _CMD config variables (#12411)
+- Fix operator field update for SerializedBaseOperator (#10924)
+- Limited cryptography to < 3.2 for python 2.7
+- Install cattr on Python 3.7 - Fix docs build on RTD (#12045)
+- Limit version of marshmallow-sqlalchemy
+- Pin `kubernetes` to a max version of 11.0.0 (#11974)
+- Use snakebite-py3 for HDFS dependency for Python3 (#12340)
+- Removes snakebite kerberos dependency (#10865)
+- Fix failing dependencies for FAB and Celery (#10828)
+- Fix pod_mutation_hook for 1.10.13 (#10850)
+- Fix formatting of Host information
+- Fix Logout Google Auth issue in Non-RBAC UI (#11890)
+- Add missing imports to app.py (#10650)
+- Show Generic Error for Charts & Query View in old UI (#12495)
+- TimeSensor should respect the default_timezone config (#9699)
+- TimeSensor should respect DAG timezone (#9882)
+- Unify user session lifetime configuration (#11970)
+- Handle outdated webserver session timeout gracefully. (#12332)
+
+
+Improvements
+""""""""""""
+
+- Add XCom.deserialize_value to Airflow 1.10.13 (#12328)
+- Mount airflow.cfg to pod_template_file (#12311)
+- All k8s object must comply with JSON Schema (#12003)
+- Validate airflow chart values.yaml & values.schema.json (#11990)
+- Pod template file uses custom custom env variable (#11480)
+- Bump attrs and cattrs dependencies (#11969)
+- Bump attrs to > 20.0 (#11799)
+- [AIRFLOW-3607] Only query DB once per DAG run for TriggerRuleDep (#4751)
+- Rename task with duplicate task_id
+- Manage Flask AppBuilder Tables using Alembic Migrations (#12352)
+- ``airflow test`` only works for tasks in 1.10, not whole dags (#11191)
+- Improve warning messaging for duplicate task_ids in a DAG (#11126)
+- Pins moto to 1.3.14 (#10986)
+- DbApiHook: Support kwargs in get_pandas_df (#9730)
+- Make grace_period_seconds option on K8sPodOperator (#10727)
+- Fix syntax error in Dockerfile 'maintainer' Label (#10899)
+- The entrypoints in Docker Image should be owned by Airflow (#10853)
+- Make dockerfiles Google Shell Guide Compliant (#10734)
+- clean-logs script for Dockerfile: trim logs before sleep (#10685)
+- When sending tasks to celery from a sub-process, reset signal handlers (#11278)
+- SkipMixin: Add missing session.commit() and test (#10421)
+- Webserver: Further Sanitize values passed to origin param (#12459)
+- Security upgrade lodash from 4.17.19 to 4.17.20 (#11095)
+- Log instead of raise an Error for unregistered OperatorLinks (#11959)
+- Mask Password in Log table when using the CLI (#11468)
+- [AIRFLOW-3607] Optimize dep checking when depends on past set and concurrency limit
+- Execute job cancel HTTPRequest in Dataproc Hook (#10361)
+- Use rst lexer to format airflow upgrade check output (#11259)
+- Remove deprecation warning from contrib/kubernetes/pod.py
+- adding body as templated field for CloudSqlImportOperator (#10510)
+- Change log level for User's session to DEBUG (#12414)
+
+Deprecations
+""""""""""""
+
+- Deprecate importing Hooks from plugin-created module (#12133)
+- Deprecate adding Operators and Sensors via plugins (#12069)
+
+Doc only changes
+""""""""""""""""
+
+- [Doc] Correct description for macro task_instance_key_str (#11062)
+- Checks if all the libraries in setup.py are listed in installation.rst file (#12023)
+- Revise "Project Focus" copy (#12011)
+- Move Project focus and Principles higher in the README (#11973)
+- Remove archived link from README.md (#11945)
+- Update download url for Airflow Version (#11800)
+- Add Project URLs for PyPI page (#11801)
+- Move Backport Providers docs to our docsite (#11136)
+- Refactor rebase copy (#11030)
+- Add missing images for kubernetes executor docs (#11083)
+- Fix identation in executor_config example (#10467)
+- Enhanced the Kubernetes Executor doc (#10433)
+- Refactor content to a markdown table (#10863)
+- Rename "Beyond the Horizon" section and refactor content (#10802)
+- Refactor official source section to use bullets (#10801)
+- Add section for official source code (#10678)
+- Add redbubble link to Airflow merchandise (#10359)
+- README Doc: Link to Airflow directory in ASF Directory (#11137)
+
Airflow 1.10.12, 2020-08-25
----------------------------
@@ -36,6 +147,7 @@ Bug Fixes
- BugFix: K8s Executor Multinamespace mode is evaluated to true by default (#10410)
- Make KubernetesExecutor recognize kubernetes_labels (#10412)
- Fix broken Kubernetes PodRuntimeInfoEnv (#10478)
+- Sync FAB Permissions for all base views (#12162)
Improvements
""""""""""""
[airflow] 03/09: TimeSensor should respect DAG timezone (#9882)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cad03ba247c3b87178b7e4af8a8d32ad7f4f39af
Author: zikun <33...@users.noreply.github.com>
AuthorDate: Tue Jul 21 00:19:08 2020 +0800
TimeSensor should respect DAG timezone (#9882)
(cherry picked from commit 9c518fe937f8dc5e35908be96bd075c4ff666755)
---
UPDATING.md | 6 ++---
airflow/sensors/time_sensor.py | 2 +-
tests/sensors/test_time_sensor.py | 52 +++++++++++++++++++++++++++++++++++++++
3 files changed, 56 insertions(+), 4 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index faa75fd..e2285df 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -63,12 +63,12 @@ https://developers.google.com/style/inclusive-documentation
-->
## Airflow 1.10.13
-### TimeSensor will consider default_timezone setting.
+### TimeSensor is now timezone aware
Previously `TimeSensor` always compared the `target_time` with the current time in UTC.
-Now it will compare `target_time` with the current time in the timezone set by `default_timezone` under the `core`
-section of the config.
+Now it will compare `target_time` with the current time in the timezone of the DAG,
+defaulting to the `default_timezone` in the global config.
### Removed Kerberos support for HDFS hook
diff --git a/airflow/sensors/time_sensor.py b/airflow/sensors/time_sensor.py
index d26c32d..5c41c2c 100644
--- a/airflow/sensors/time_sensor.py
+++ b/airflow/sensors/time_sensor.py
@@ -37,4 +37,4 @@ class TimeSensor(BaseSensorOperator):
def poke(self, context):
self.log.info('Checking if the time (%s) has come', self.target_time)
- return timezone.make_naive(timezone.utcnow()).time() > self.target_time
+ return timezone.make_naive(timezone.utcnow(), self.dag.timezone).time() > self.target_time
diff --git a/tests/sensors/test_time_sensor.py b/tests/sensors/test_time_sensor.py
new file mode 100644
index 0000000..c08bdd1
--- /dev/null
+++ b/tests/sensors/test_time_sensor.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime, time
+
+import pendulum
+from parameterized import parameterized
+
+from airflow.models.dag import DAG
+from airflow.sensors.time_sensor import TimeSensor
+from airflow.utils import timezone
+from tests.compat import patch
+
+DEFAULT_TIMEZONE = "Asia/Singapore" # UTC+08:00
+DEFAULT_DATE_WO_TZ = datetime(2015, 1, 1)
+DEFAULT_DATE_WITH_TZ = datetime(
+ 2015, 1, 1, tzinfo=pendulum.tz.timezone(DEFAULT_TIMEZONE)
+)
+
+
+@patch(
+ "airflow.sensors.time_sensor.timezone.utcnow",
+ return_value=timezone.datetime(2020, 1, 1, 23, 0).replace(tzinfo=timezone.utc),
+)
+class TestTimeSensor:
+ @parameterized.expand(
+ [
+ ("UTC", DEFAULT_DATE_WO_TZ, True),
+ ("UTC", DEFAULT_DATE_WITH_TZ, False),
+ (DEFAULT_TIMEZONE, DEFAULT_DATE_WO_TZ, False),
+ ]
+ )
+ def test_timezone(self, mock_utcnow, default_timezone, start_date, expected):
+ with patch("airflow.settings.TIMEZONE", pendulum.timezone(default_timezone)):
+ dag = DAG("test", default_args={"start_date": start_date})
+ op = TimeSensor(task_id="test", target_time=time(10, 0), dag=dag)
+ assert op.poke(None) == expected
[airflow] 04/09: Merge invalid_kubernetes_config rule and
pod_template_file rule
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 37da5a1dd5d0b5bffb69f1c923086bf6ad30c1e8
Author: Daniel Imberman <da...@gmail.com>
AuthorDate: Fri Nov 20 07:19:10 2020 -0800
Merge invalid_kubernetes_config rule and pod_template_file rule
---
.../upgrade/rules/invalid_kubernetes_configs.py | 69 ----------------------
airflow/upgrade/rules/pod_template_file_rule.py | 42 ++++++++++++-
2 files changed, 40 insertions(+), 71 deletions(-)
diff --git a/airflow/upgrade/rules/invalid_kubernetes_configs.py b/airflow/upgrade/rules/invalid_kubernetes_configs.py
deleted file mode 100644
index 0850501..0000000
--- a/airflow/upgrade/rules/invalid_kubernetes_configs.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import absolute_import
-
-from airflow.upgrade.rules.base_rule import BaseRule
-from airflow.configuration import conf
-
-invalid_keys = {"airflow_configmap",
- "airflow_local_settings_configmap",
- "dags_in_image",
- "dags_volume_subpath",
- "dags_volume_mount_point", "dags_volume_claim",
- "logs_volume_subpath", "logs_volume_claim",
- "dags_volume_host", "logs_volume_host",
- "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
- "git_branch", "git_sync_depth", "git_subpath",
- "git_sync_rev", "git_user", "git_password",
- "git_sync_root", "git_sync_dest",
- "git_dags_folder_mount_point", "git_ssh_key_secret_name",
- "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
- "git_sync_container_repository",
- "git_sync_container_tag", "git_sync_init_container_name",
- "git_sync_run_as_user",
- "worker_service_account_name", "image_pull_secrets",
- "gcp_service_account_keys", "affinity",
- "tolerations", "run_as_user", "fs_group"}
-
-
-class InvalidKubernetesConfigRule(BaseRule):
-
- title = "Users must delete deprecated configs for KubernetesExecutor"
-
- description = """\
-In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
-needed once this pod_template_file has been generated.
-"""
-
- def check(self):
- conf_dict = conf.as_dict(display_sensitive=True)
- kube_conf = conf_dict['kubernetes']
- keys = kube_conf.keys()
- resp = [k for k in keys if k in invalid_keys]
- if conf_dict['kubernetes_labels']:
- resp.append("kubernetes_labels")
- if conf_dict['kubernetes_secrets']:
- resp.append("kubernetes_secrets")
-
- if resp:
- resp_string = "\n".join(resp)
- return "The following invalid keys were found in your airflow.cfg: \
- \n\n{resp_string}\n\n \
- Please generate a pod_template_file by running `airflow generate_pod_template` \
- and delete these keys.".format(resp_string=resp_string)
diff --git a/airflow/upgrade/rules/pod_template_file_rule.py b/airflow/upgrade/rules/pod_template_file_rule.py
index 21363d3..8cbbf96 100644
--- a/airflow/upgrade/rules/pod_template_file_rule.py
+++ b/airflow/upgrade/rules/pod_template_file_rule.py
@@ -20,14 +20,36 @@ from __future__ import absolute_import
from airflow.upgrade.rules.base_rule import BaseRule
from airflow.configuration import conf
+invalid_config_keys = {
+ "airflow_configmap",
+ "airflow_local_settings_configmap",
+ "dags_in_image",
+ "dags_volume_subpath",
+ "dags_volume_mount_point", "dags_volume_claim",
+ "logs_volume_subpath", "logs_volume_claim",
+ "dags_volume_host", "logs_volume_host",
+ "env_from_configmap_ref", "env_from_secret_ref", "git_repo",
+ "git_branch", "git_sync_depth", "git_subpath",
+ "git_sync_rev", "git_user", "git_password",
+ "git_sync_root", "git_sync_dest",
+ "git_dags_folder_mount_point", "git_ssh_key_secret_name",
+ "git_ssh_known_hosts_configmap_name", "git_sync_credentials_secret",
+ "git_sync_container_repository",
+ "git_sync_container_tag", "git_sync_init_container_name",
+ "git_sync_run_as_user",
+ "worker_service_account_name", "image_pull_secrets",
+ "gcp_service_account_keys", "affinity",
+ "tolerations", "run_as_user", "fs_group"
+}
-class PodTemplateFileRule(BaseRule):
+class PodTemplateFileRule(BaseRule):
title = "Users must set a kubernetes.pod_template_file value"
description = """\
In Airflow 2.0, KubernetesExecutor Users need to set a pod_template_file as a base
-value for all pods launched by the KubernetesExecutor
+value for all pods launched by the KubernetesExecutor. Many Kubernetes configs are no longer
+needed once this pod_template_file has been generated.
"""
def check(self):
@@ -37,3 +59,19 @@ value for all pods launched by the KubernetesExecutor
"Please create a pod_template_file by running `airflow generate_pod_template`.\n"
"This will generate a pod using your aiflow.cfg settings"
)
+
+ conf_dict = conf.as_dict(display_sensitive=True)
+ kube_conf = conf_dict['kubernetes']
+ keys = kube_conf.keys()
+ resp = [k for k in keys if k in invalid_config_keys]
+ if conf_dict['kubernetes_labels']:
+ resp.append("kubernetes_labels")
+ if conf_dict['kubernetes_secrets']:
+ resp.append("kubernetes_secrets")
+
+ if resp:
+ resp_string = "\n".join(resp)
+ return "The following invalid keys were found in your airflow.cfg: \
+ \n\n{resp_string}\n\n \
+ Now that you have a pod_template_file, these keys no longer do anything.\n\
+ Please delete these keys.".format(resp_string=resp_string)
[airflow] 01/09: Show Generic Error for Charts & Query View in old
UI (#12495)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e0ade6aa968fe2f4548e0ce13bf9bcef8630275c
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Fri Nov 20 13:31:10 2020 +0000
Show Generic Error for Charts & Query View in old UI (#12495)
---
airflow/www/views.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/airflow/www/views.py b/airflow/www/views.py
index fdb674d..f11fb84 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -471,8 +471,9 @@ class Airflow(AirflowViewMixin, BaseView):
df = hook.get_pandas_df(
wwwutils.limit_sql(sql, CHART_LIMIT, conn_type=db.conn_type))
df = df.fillna(0)
- except Exception as e:
- payload['error'] += "SQL execution failed. Details: " + str(e)
+ except Exception:
+ log.exception("Chart SQL execution failed")
+ payload['error'] += "SQL execution failed. Contact your System Administrator for more details"
if csv:
return Response(
@@ -2398,8 +2399,9 @@ class QueryView(wwwutils.DataProfilingMixin, AirflowViewMixin, BaseView):
index=False,
na_rep='',
) if has_data else ''
- except Exception as e:
- flash(str(e), 'error')
+ except Exception:
+ log.exception("Query SQL execution failed")
+ flash("SQL execution failed. Contact your System Administrator for more details", "error")
error = True
if has_data and len(df) == QUERY_LIMIT:
[airflow] 08/09: Add Note about hook,
Operators/Senesors deprecation in Updating.md
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 560ae753625190c6899f8ec77c8332dfea102510
Author: Kaxil Naik <ka...@gmail.com>
AuthorDate: Fri Nov 20 19:31:25 2020 +0000
Add Note about hook, Operators/Senesors deprecation in Updating.md
---
UPDATING.md | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/UPDATING.md b/UPDATING.md
index 6dbe8cc..c29c8a3 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -101,6 +101,14 @@ After
session_lifetime_minutes = 43200
```
+### Adding Operators, Hooks and Sensors via Airflow Plugins is deprecated
+
+The ability to import Operators, Hooks and Senors via the plugin mechanism has been deprecated and will raise warnings
+in Airflow 1.10.13 and will be removed completely in Airflow 2.0.
+
+Check http://airflow.apache.org/docs/1.10.13/howto/custom-operator.html to see how you can create and import
+Custom Hooks, Operators and Sensors.
+
## Airflow 1.10.12
### Clearing tasks skipped by SkipMixin will skip them
[airflow] 02/09: TimeSensor should respect the default_timezone
config (#9699)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 652b0c179eb5acd653669d06b76758938cb1307c
Author: zikun <33...@users.noreply.github.com>
AuthorDate: Sun Jul 19 01:36:28 2020 +0800
TimeSensor should respect the default_timezone config (#9699)
(cherry picked from commit b34ba874452809495354f3012e0b1dcbf4209e09)
---
UPDATING.md | 7 +++++++
airflow/sensors/time_sensor.py | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/UPDATING.md b/UPDATING.md
index 2057491..faa75fd 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -63,6 +63,13 @@ https://developers.google.com/style/inclusive-documentation
-->
## Airflow 1.10.13
+### TimeSensor will consider default_timezone setting.
+
+Previously `TimeSensor` always compared the `target_time` with the current time in UTC.
+
+Now it will compare `target_time` with the current time in the timezone set by `default_timezone` under the `core`
+section of the config.
+
### Removed Kerberos support for HDFS hook
The HDFS hook's Kerberos support has been removed due to removed python-krbV dependency from PyPI
diff --git a/airflow/sensors/time_sensor.py b/airflow/sensors/time_sensor.py
index 0c39235..d26c32d 100644
--- a/airflow/sensors/time_sensor.py
+++ b/airflow/sensors/time_sensor.py
@@ -37,4 +37,4 @@ class TimeSensor(BaseSensorOperator):
def poke(self, context):
self.log.info('Checking if the time (%s) has come', self.target_time)
- return timezone.utcnow().time() > self.target_time
+ return timezone.make_naive(timezone.utcnow()).time() > self.target_time
[airflow] 05/09: Unify user session lifetime configuration (#11970)
Posted by ka...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 7e0ab065ec8ffb359b21e56dab12ec8bdc7d7bd3
Author: Michał Misiewicz <mi...@gmail.com>
AuthorDate: Wed Nov 11 13:28:58 2020 +0100
Unify user session lifetime configuration (#11970)
* Unify user session lifetime configuration
* align with new linting rules
* exit app when removed args are provided in conf
* add one more test
* extract stopping gunicorn to method
* add docstring to stop_webserver method
* use lazy formatting
* exit webserver when removed options are provided
* align with markdown lint
* Move unify user session lifetime configuration section to master
* add new line
* remove quotes
---
UPDATING.md | 23 +++++++++++++++++++++++
airflow/config_templates/config.yml | 19 ++++++-------------
airflow/config_templates/default_airflow.cfg | 9 +++------
airflow/www_rbac/app.py | 28 +++++++++++++++++-----------
tests/www_rbac/test_app.py | 25 +++++++++++++++++++++++++
5 files changed, 74 insertions(+), 30 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index e2285df..6dbe8cc 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -78,6 +78,29 @@ support for SSL connection to HDFS).
SSL support still works for WebHDFS hook.
+### Unify user session lifetime configuration
+
+In previous version of Airflow user session lifetime could be configured by
+`session_lifetime_days` and `force_log_out_after` options. In practise only `session_lifetime_days`
+had impact on session lifetime, but it was limited to values in day.
+We have removed mentioned options and introduced new `session_lifetime_minutes`
+option which simplify session lifetime configuration.
+
+Before
+
+ ```ini
+[webserver]
+force_log_out_after = 0
+session_lifetime_days = 30
+ ```
+
+After
+
+ ```ini
+[webserver]
+session_lifetime_minutes = 43200
+ ```
+
## Airflow 1.10.12
### Clearing tasks skipped by SkipMixin will skip them
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
index d2621eb..e89df22 100644
--- a/airflow/config_templates/config.yml
+++ b/airflow/config_templates/config.yml
@@ -1001,21 +1001,14 @@
type: string
example: ~
default: "True"
- - name: force_log_out_after
+ - name: session_lifetime_minutes
description: |
- Minutes of non-activity before logged out from UI
- 0 means never get forcibly logged out
- version_added: 1.10.8
- type: string
- example: ~
- default: "0"
- - name: session_lifetime_days
- description: |
- The UI cookie lifetime in days
- version_added: 1.10.8
- type: string
+ The UI cookie lifetime in minutes. User will be logged out from UI after
+ ``session_lifetime_minutes`` of non-activity
+ version_added: 1.10.13
+ type: int
example: ~
- default: "30"
+ default: "43200"
- name: email
description: ~
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 71a5172..ea64d8f 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -484,12 +484,9 @@ x_frame_enabled = True
# on webserver startup
update_fab_perms = True
-# Minutes of non-activity before logged out from UI
-# 0 means never get forcibly logged out
-force_log_out_after = 0
-
-# The UI cookie lifetime in days
-session_lifetime_days = 30
+# The UI cookie lifetime in minutes. User will be logged out from UI after
+# ``session_lifetime_minutes`` of non-activity
+session_lifetime_minutes = 43200
[email]
email_backend = airflow.utils.email.send_email_smtp
diff --git a/airflow/www_rbac/app.py b/airflow/www_rbac/app.py
index 29a364b..0b1a8c7 100644
--- a/airflow/www_rbac/app.py
+++ b/airflow/www_rbac/app.py
@@ -18,6 +18,7 @@
# under the License.
#
import logging
+import sys
import socket
from datetime import timedelta
from typing import Any
@@ -62,8 +63,22 @@ def create_app(config=None, session=None, testing=False, app_name="Airflow"):
)
app.secret_key = conf.get('webserver', 'SECRET_KEY')
- session_lifetime_days = conf.getint('webserver', 'SESSION_LIFETIME_DAYS', fallback=30)
- app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=session_lifetime_days)
+ if conf.has_option('webserver', 'SESSION_LIFETIME_DAYS') or conf.has_option(
+ 'webserver', 'FORCE_LOG_OUT_AFTER'
+ ):
+ logging.error(
+ '`SESSION_LIFETIME_DAYS` option from `webserver` section has been '
+ 'renamed to `SESSION_LIFETIME_MINUTES`. New option allows to configure '
+ 'session lifetime in minutes. FORCE_LOG_OUT_AFTER option has been removed '
+ 'from `webserver` section. Please update your configuration.'
+ )
+ # Stop gunicorn server https://github.com/benoitc/gunicorn/blob/20.0.4/gunicorn/arbiter.py#L526
+ # sys.exit(4)
+ else:
+ session_lifetime_minutes = conf.getint('webserver', 'SESSION_LIFETIME_MINUTES', fallback=43200)
+ logging.info('User session lifetime is set to %s minutes.', session_lifetime_minutes)
+
+ app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(minutes=session_lifetime_minutes)
app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
@@ -268,15 +283,6 @@ def create_app(config=None, session=None, testing=False, app_name="Airflow"):
def shutdown_session(exception=None):
settings.Session.remove()
- @app.before_request
- def before_request():
- _force_log_out_after = conf.getint('webserver', 'FORCE_LOG_OUT_AFTER', fallback=0)
- if _force_log_out_after > 0:
- flask.session.permanent = True
- app.permanent_session_lifetime = timedelta(minutes=_force_log_out_after)
- flask.session.modified = True
- flask.g.user = flask_login.current_user
-
@app.after_request
def apply_caching(response):
_x_frame_enabled = conf.getboolean('webserver', 'X_FRAME_ENABLED', fallback=True)
diff --git a/tests/www_rbac/test_app.py b/tests/www_rbac/test_app.py
index 0fe8508..a0607cb 100644
--- a/tests/www_rbac/test_app.py
+++ b/tests/www_rbac/test_app.py
@@ -18,11 +18,13 @@
# under the License.
import json
import unittest
+from datetime import timedelta
import pytest
import six
from werkzeug.middleware.proxy_fix import ProxyFix
+from airflow.configuration import conf
from airflow.settings import Session
from airflow.www_rbac import app as application
from tests.compat import mock
@@ -80,3 +82,26 @@ class TestApp(unittest.TestCase):
if six.PY2:
engine_params = json.dumps(engine_params)
self.assertEqual(app.config['SQLALCHEMY_ENGINE_OPTIONS'], engine_params)
+
+ @conf_vars(
+ {
+ ('webserver', 'session_lifetime_minutes'): '3600',
+ }
+ )
+ @mock.patch("airflow.www_rbac.app.app", None)
+ def test_should_set_permanent_session_timeout(self):
+ app, _ = application.cached_appbuilder(testing=True)
+ self.assertEqual(app.config['PERMANENT_SESSION_LIFETIME'], timedelta(minutes=3600))
+
+ @conf_vars(
+ {
+ ('webserver', 'session_lifetime_days'): '30',
+ ('webserver', 'force_log_out_after'): '30',
+ }
+ )
+ @mock.patch("airflow.www_rbac.app.app", None)
+ def test_should_stop_app_when_removed_options_are_provided(self):
+ with self.assertRaises(SystemExit) as e:
+ conf.remove_option('webserver', 'session_lifetime_minutes')
+ application.cached_appbuilder(testing=True)
+ self.assertEqual(e.exception.code, 4)