You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/06/07 07:02:44 UTC
[1/3] incubator-airflow git commit: [AIRFLOW-1276] Forbid event
creation with end_data earlier than start_date
Repository: incubator-airflow
Updated Branches:
refs/heads/master e3e6aa719 -> ae5c53b6c
[AIRFLOW-1276] Forbid event creation with end_data earlier than start_date
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d5d02ff7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d5d02ff7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d5d02ff7
Branch: refs/heads/master
Commit: d5d02ff7257621af40728ed4089d9fe812b98daf
Parents: e3e6aa7
Author: Stanislav Kudriashev <st...@gmail.com>
Authored: Mon Jun 5 12:56:25 2017 +0300
Committer: Stanislav Kudriashev <st...@gmail.com>
Committed: Tue Jun 6 13:03:44 2017 +0300
----------------------------------------------------------------------
airflow/www/validators.py | 54 +++++++++++++++++++++++
airflow/www/views.py | 18 +++++++-
tests/www/test_validators.py | 92 +++++++++++++++++++++++++++++++++++++++
tests/www/test_views.py | 59 ++++++++++++++++++++++++-
4 files changed, 220 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/airflow/www/validators.py
----------------------------------------------------------------------
diff --git a/airflow/www/validators.py b/airflow/www/validators.py
new file mode 100644
index 0000000..4a72983
--- /dev/null
+++ b/airflow/www/validators.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from wtforms.validators import EqualTo
+from wtforms.validators import ValidationError
+
+
+class GreaterEqualThan(EqualTo):
+ """Compares the values of two fields.
+
+ :param fieldname:
+ The name of the other field to compare to.
+ :param message:
+ Error message to raise in case of a validation error. Can be
+ interpolated with `%(other_label)s` and `%(other_name)s` to provide a
+ more helpful error.
+ """
+
+ def __call__(self, form, field):
+ try:
+ other = form[self.fieldname]
+ except KeyError:
+ raise ValidationError(
+ field.gettext("Invalid field name '%s'." % self.fieldname)
+ )
+
+ if field.data is None or other.data is None:
+ return
+
+ if field.data < other.data:
+ d = {
+ 'other_label': hasattr(other, 'label') and other.label.text
+ or self.fieldname,
+ 'other_name': self.fieldname,
+ }
+ message = self.message
+ if message is None:
+ message = field.gettext('Field must be greater than or equal '
+ 'to %(other_label)s.' % d)
+ else:
+ message = message % d
+
+ raise ValidationError(message)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 4fd52fe..e16d201 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -75,6 +75,7 @@ from airflow.utils import logging as log_utils
from airflow.utils.dates import infer_time_unit, scale_time_units
from airflow.www import utils as wwwutils
from airflow.www.forms import DateTimeForm, DateTimeWithNumRunsForm
+from airflow.www.validators import GreaterEqualThan
from airflow.configuration import AirflowConfigException
QUERY_LIMIT = 100000
@@ -2148,9 +2149,22 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
'start_date',
'end_date',
'reported_by',
- 'description')
+ 'description',
+ )
+ form_args = {
+ 'end_date': {
+ 'validators': {
+ GreaterEqualThan(fieldname='start_date'),
+ }
+ }
+ }
column_list = (
- 'label', 'event_type', 'start_date', 'end_date', 'reported_by')
+ 'label',
+ 'event_type',
+ 'start_date',
+ 'end_date',
+ 'reported_by',
+ )
column_default_sort = ("start_date", True)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/tests/www/test_validators.py
----------------------------------------------------------------------
diff --git a/tests/www/test_validators.py b/tests/www/test_validators.py
new file mode 100644
index 0000000..14e964e
--- /dev/null
+++ b/tests/www/test_validators.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+from airflow.www import validators
+
+
+class TestGreaterEqualThan(unittest.TestCase):
+
+ def setUp(self):
+ super(TestGreaterEqualThan, self).setUp()
+ self.form_field_mock = mock.MagicMock(data='2017-05-06')
+ self.form_field_mock.gettext.side_effect = lambda msg: msg
+ self.other_field_mock = mock.MagicMock(data='2017-05-05')
+ self.other_field_mock.gettext.side_effect = lambda msg: msg
+ self.other_field_mock.label.text = 'other field'
+ self.form_stub = {'other_field': self.other_field_mock}
+ self.form_mock = mock.MagicMock(spec_set=dict)
+ self.form_mock.__getitem__.side_effect = self.form_stub.__getitem__
+
+ def _validate(self, fieldname=None, message=None):
+ if fieldname is None:
+ fieldname = 'other_field'
+
+ validator = validators.GreaterEqualThan(fieldname=fieldname,
+ message=message)
+
+ return validator(self.form_mock, self.form_field_mock)
+
+ def test_field_not_found(self):
+ self.assertRaisesRegexp(
+ validators.ValidationError,
+ "^Invalid field name 'some'.$",
+ self._validate,
+ fieldname='some',
+ )
+
+ def test_form_field_is_none(self):
+ self.form_field_mock.data = None
+
+ self.assertIsNone(self._validate())
+
+ def test_other_field_is_none(self):
+ self.other_field_mock.data = None
+
+ self.assertIsNone(self._validate())
+
+ def test_both_fields_are_none(self):
+ self.form_field_mock.data = None
+ self.other_field_mock.data = None
+
+ self.assertIsNone(self._validate())
+
+ def test_validation_pass(self):
+ self.assertIsNone(self._validate())
+
+ def test_validation_raises(self):
+ self.form_field_mock.data = '2017-05-04'
+
+ self.assertRaisesRegexp(
+ validators.ValidationError,
+ "^Field must be greater than or equal to other field.$",
+ self._validate,
+ )
+
+ def test_validation_raises_custom_message(self):
+ self.form_field_mock.data = '2017-05-04'
+
+ self.assertRaisesRegexp(
+ validators.ValidationError,
+ "^This field must be greater than or equal to MyField.$",
+ self._validate,
+ message="This field must be greater than or equal to MyField.",
+ )
+
+
+
+if __name__ == '__main__':
+ unittest.main()
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/tests/www/test_views.py
----------------------------------------------------------------------
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index a8823e6..3e2fe61 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -15,11 +15,68 @@
import unittest
from airflow import configuration
-from airflow.models import Pool
+from airflow.models import KnownEvent, Pool
from airflow.settings import Session
from airflow.www import app as application
+class TestKnownEventView(unittest.TestCase):
+
+ CREATE_ENDPOINT = '/admin/knownevent/new/?url=/admin/knownevent/'
+
+ @classmethod
+ def setUpClass(cls):
+ super(TestKnownEventView, cls).setUpClass()
+ session = Session()
+ session.query(KnownEvent).delete()
+ session.commit()
+ session.close()
+
+ def setUp(self):
+ super(TestKnownEventView, self).setUp()
+ configuration.load_test_config()
+ app = application.create_app(testing=True)
+ app.config['WTF_CSRF_METHODS'] = []
+ self.app = app.test_client()
+ self.session = Session()
+ self.known_event = {
+ 'label': 'event-label',
+ 'event_type': 1,
+ 'start_date': '2017-06-05 12:00:00',
+ 'end_date': '2017-06-05 13:00:00',
+ 'reported_by': 'airflow',
+ 'description': '',
+ }
+
+ def tearDown(self):
+ self.session.query(KnownEvent).delete()
+ self.session.commit()
+ self.session.close()
+ super(TestKnownEventView, self).tearDown()
+
+ def test_create_known_event(self):
+ response = self.app.post(
+ self.CREATE_ENDPOINT,
+ data=self.known_event,
+ follow_redirects=True,
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(self.session.query(KnownEvent).count(), 1)
+
+ def test_create_known_event_with_end_data_earlier_than_start_date(self):
+ self.known_event['end_date'] = '2017-06-05 11:00:00'
+ response = self.app.post(
+ self.CREATE_ENDPOINT,
+ data=self.known_event,
+ follow_redirects=True,
+ )
+ self.assertIn(
+ 'Field must be greater than or equal to Start Date.',
+ response.data.decode('utf-8'),
+ )
+ self.assertEqual(self.session.query(KnownEvent).count(), 0)
+
+
class TestPoolModelView(unittest.TestCase):
CREATE_ENDPOINT = '/admin/pool/new/?url=/admin/pool/'
[3/3] incubator-airflow git commit: Merge pull request #2349 from
skudriashev/airflow-1277
Posted by bo...@apache.org.
Merge pull request #2349 from skudriashev/airflow-1277
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/ae5c53b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/ae5c53b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/ae5c53b6
Branch: refs/heads/master
Commit: ae5c53b6cbacc6f4b49cc1fe85a96323ee1ddb6e
Parents: e3e6aa7 65184a1
Author: Bolke de Bruin <bo...@xs4all.nl>
Authored: Wed Jun 7 09:02:36 2017 +0200
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Wed Jun 7 09:02:36 2017 +0200
----------------------------------------------------------------------
airflow/www/app.py | 2 +-
airflow/www/validators.py | 54 +++++++++++++++++++++++
airflow/www/views.py | 43 ++++++++++++++++--
tests/www/test_validators.py | 92 +++++++++++++++++++++++++++++++++++++++
tests/www/test_views.py | 82 +++++++++++++++++++++++++++++++---
5 files changed, 262 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-airflow git commit: [AIRFLOW-1277] Forbid KE creation
with empty fields
Posted by bo...@apache.org.
[AIRFLOW-1277] Forbid KE creation with empty fields
KE = KnownEvent
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/65184a10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/65184a10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/65184a10
Branch: refs/heads/master
Commit: 65184a109e0179b50e4dae534a38c6d66bf080d8
Parents: d5d02ff
Author: Stanislav Kudriashev <st...@gmail.com>
Authored: Tue Jun 6 16:21:49 2017 +0300
Committer: Stanislav Kudriashev <st...@gmail.com>
Committed: Tue Jun 6 19:29:12 2017 +0300
----------------------------------------------------------------------
airflow/www/app.py | 2 +-
airflow/www/views.py | 29 +++++++++++++++++++++++++----
tests/www/test_views.py | 37 +++++++++++++++++++++++++------------
3 files changed, 51 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/airflow/www/app.py
----------------------------------------------------------------------
diff --git a/airflow/www/app.py b/airflow/www/app.py
index 80e9b87..1ae2731 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -72,7 +72,7 @@ def create_app(config=None, testing=False):
av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling"))
av(vs.ChartModelView(
models.Chart, Session, name="Charts", category="Data Profiling"))
- av(vs.KnowEventView(
+ av(vs.KnownEventView(
models.KnownEvent,
Session, name="Known Events", category="Data Profiling"))
av(vs.SlaMissModelView(
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index e16d201..aadd200 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2140,7 +2140,7 @@ chart_mapping = (
chart_mapping = dict(chart_mapping)
-class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
+class KnownEventView(wwwutils.DataProfilingMixin, AirflowModelView):
verbose_name = "known event"
verbose_name_plural = "known events"
form_columns = (
@@ -2152,10 +2152,31 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
'description',
)
form_args = {
+ 'label': {
+ 'validators': [
+ validators.DataRequired(),
+ ],
+ },
+ 'event_type': {
+ 'validators': [
+ validators.DataRequired(),
+ ],
+ },
+ 'start_date': {
+ 'validators': [
+ validators.DataRequired(),
+ ],
+ },
'end_date': {
- 'validators': {
+ 'validators': [
+ validators.DataRequired(),
GreaterEqualThan(fieldname='start_date'),
- }
+ ],
+ },
+ 'reported_by': {
+ 'validators': [
+ validators.DataRequired(),
+ ],
}
}
column_list = (
@@ -2168,7 +2189,7 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
column_default_sort = ("start_date", True)
-class KnowEventTypeView(wwwutils.DataProfilingMixin, AirflowModelView):
+class KnownEventTypeView(wwwutils.DataProfilingMixin, AirflowModelView):
pass
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/tests/www/test_views.py
----------------------------------------------------------------------
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index 3e2fe61..5a20cbd 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -15,7 +15,7 @@
import unittest
from airflow import configuration
-from airflow.models import KnownEvent, Pool
+from airflow import models
from airflow.settings import Session
from airflow.www import app as application
@@ -28,8 +28,13 @@ class TestKnownEventView(unittest.TestCase):
def setUpClass(cls):
super(TestKnownEventView, cls).setUpClass()
session = Session()
- session.query(KnownEvent).delete()
+ session.query(models.KnownEvent).delete()
+ session.query(models.User).delete()
session.commit()
+ user = models.User(username='airflow')
+ session.add(user)
+ session.commit()
+ cls.user_id = user.id
session.close()
def setUp(self):
@@ -41,19 +46,27 @@ class TestKnownEventView(unittest.TestCase):
self.session = Session()
self.known_event = {
'label': 'event-label',
- 'event_type': 1,
+ 'event_type': '1',
'start_date': '2017-06-05 12:00:00',
'end_date': '2017-06-05 13:00:00',
- 'reported_by': 'airflow',
+ 'reported_by': self.user_id,
'description': '',
}
def tearDown(self):
- self.session.query(KnownEvent).delete()
+ self.session.query(models.KnownEvent).delete()
self.session.commit()
self.session.close()
super(TestKnownEventView, self).tearDown()
+ @classmethod
+ def tearDownClass(cls):
+ session = Session()
+ session.query(models.User).delete()
+ session.commit()
+ session.close()
+ super(TestKnownEventView, cls).tearDownClass()
+
def test_create_known_event(self):
response = self.app.post(
self.CREATE_ENDPOINT,
@@ -61,7 +74,7 @@ class TestKnownEventView(unittest.TestCase):
follow_redirects=True,
)
self.assertEqual(response.status_code, 200)
- self.assertEqual(self.session.query(KnownEvent).count(), 1)
+ self.assertEqual(self.session.query(models.KnownEvent).count(), 1)
def test_create_known_event_with_end_data_earlier_than_start_date(self):
self.known_event['end_date'] = '2017-06-05 11:00:00'
@@ -74,7 +87,7 @@ class TestKnownEventView(unittest.TestCase):
'Field must be greater than or equal to Start Date.',
response.data.decode('utf-8'),
)
- self.assertEqual(self.session.query(KnownEvent).count(), 0)
+ self.assertEqual(self.session.query(models.KnownEvent).count(), 0)
class TestPoolModelView(unittest.TestCase):
@@ -85,7 +98,7 @@ class TestPoolModelView(unittest.TestCase):
def setUpClass(cls):
super(TestPoolModelView, cls).setUpClass()
session = Session()
- session.query(Pool).delete()
+ session.query(models.Pool).delete()
session.commit()
session.close()
@@ -103,7 +116,7 @@ class TestPoolModelView(unittest.TestCase):
}
def tearDown(self):
- self.session.query(Pool).delete()
+ self.session.query(models.Pool).delete()
self.session.commit()
self.session.close()
super(TestPoolModelView, self).tearDown()
@@ -115,7 +128,7 @@ class TestPoolModelView(unittest.TestCase):
follow_redirects=True,
)
self.assertEqual(response.status_code, 200)
- self.assertEqual(self.session.query(Pool).count(), 1)
+ self.assertEqual(self.session.query(models.Pool).count(), 1)
def test_create_pool_with_same_name(self):
# create test pool
@@ -131,7 +144,7 @@ class TestPoolModelView(unittest.TestCase):
follow_redirects=True,
)
self.assertIn('Already exists.', response.data.decode('utf-8'))
- self.assertEqual(self.session.query(Pool).count(), 1)
+ self.assertEqual(self.session.query(models.Pool).count(), 1)
def test_create_pool_with_empty_name(self):
self.pool['pool'] = ''
@@ -141,7 +154,7 @@ class TestPoolModelView(unittest.TestCase):
follow_redirects=True,
)
self.assertIn('This field is required.', response.data.decode('utf-8'))
- self.assertEqual(self.session.query(Pool).count(), 0)
+ self.assertEqual(self.session.query(models.Pool).count(), 0)
if __name__ == '__main__':