You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by bo...@apache.org on 2017/06/07 07:02:44 UTC

[1/3] incubator-airflow git commit: [AIRFLOW-1276] Forbid event creation with end_data earlier than start_date

Repository: incubator-airflow
Updated Branches:
  refs/heads/master e3e6aa719 -> ae5c53b6c


[AIRFLOW-1276] Forbid event creation with end_data earlier than start_date


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/d5d02ff7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/d5d02ff7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/d5d02ff7

Branch: refs/heads/master
Commit: d5d02ff7257621af40728ed4089d9fe812b98daf
Parents: e3e6aa7
Author: Stanislav Kudriashev <st...@gmail.com>
Authored: Mon Jun 5 12:56:25 2017 +0300
Committer: Stanislav Kudriashev <st...@gmail.com>
Committed: Tue Jun 6 13:03:44 2017 +0300

----------------------------------------------------------------------
 airflow/www/validators.py    | 54 +++++++++++++++++++++++
 airflow/www/views.py         | 18 +++++++-
 tests/www/test_validators.py | 92 +++++++++++++++++++++++++++++++++++++++
 tests/www/test_views.py      | 59 ++++++++++++++++++++++++-
 4 files changed, 220 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/airflow/www/validators.py
----------------------------------------------------------------------
diff --git a/airflow/www/validators.py b/airflow/www/validators.py
new file mode 100644
index 0000000..4a72983
--- /dev/null
+++ b/airflow/www/validators.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from wtforms.validators import EqualTo
+from wtforms.validators import ValidationError
+
+
+class GreaterEqualThan(EqualTo):
+    """Compares the values of two fields.
+
+    :param fieldname:
+        The name of the other field to compare to.
+    :param message:
+        Error message to raise in case of a validation error. Can be
+        interpolated with `%(other_label)s` and `%(other_name)s` to provide a
+        more helpful error.
+    """
+
+    def __call__(self, form, field):
+        try:
+            other = form[self.fieldname]
+        except KeyError:
+            raise ValidationError(
+                field.gettext("Invalid field name '%s'." % self.fieldname)
+            )
+
+        if field.data is None or other.data is None:
+            return
+
+        if field.data < other.data:
+            d = {
+                'other_label': hasattr(other, 'label') and other.label.text
+                               or self.fieldname,
+                'other_name': self.fieldname,
+            }
+            message = self.message
+            if message is None:
+                message = field.gettext('Field must be greater than or equal '
+                                        'to %(other_label)s.' % d)
+            else:
+                message = message % d
+
+            raise ValidationError(message)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 4fd52fe..e16d201 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -75,6 +75,7 @@ from airflow.utils import logging as log_utils
 from airflow.utils.dates import infer_time_unit, scale_time_units
 from airflow.www import utils as wwwutils
 from airflow.www.forms import DateTimeForm, DateTimeWithNumRunsForm
+from airflow.www.validators import GreaterEqualThan
 from airflow.configuration import AirflowConfigException
 
 QUERY_LIMIT = 100000
@@ -2148,9 +2149,22 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
         'start_date',
         'end_date',
         'reported_by',
-        'description')
+        'description',
+    )
+    form_args = {
+        'end_date': {
+            'validators': {
+                GreaterEqualThan(fieldname='start_date'),
+            }
+        }
+    }
     column_list = (
-        'label', 'event_type', 'start_date', 'end_date', 'reported_by')
+        'label',
+        'event_type',
+        'start_date',
+        'end_date',
+        'reported_by',
+    )
     column_default_sort = ("start_date", True)
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/tests/www/test_validators.py
----------------------------------------------------------------------
diff --git a/tests/www/test_validators.py b/tests/www/test_validators.py
new file mode 100644
index 0000000..14e964e
--- /dev/null
+++ b/tests/www/test_validators.py
@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+from airflow.www import validators
+
+
+class TestGreaterEqualThan(unittest.TestCase):
+
+    def setUp(self):
+        super(TestGreaterEqualThan, self).setUp()
+        self.form_field_mock = mock.MagicMock(data='2017-05-06')
+        self.form_field_mock.gettext.side_effect = lambda msg: msg
+        self.other_field_mock = mock.MagicMock(data='2017-05-05')
+        self.other_field_mock.gettext.side_effect = lambda msg: msg
+        self.other_field_mock.label.text = 'other field'
+        self.form_stub = {'other_field': self.other_field_mock}
+        self.form_mock = mock.MagicMock(spec_set=dict)
+        self.form_mock.__getitem__.side_effect = self.form_stub.__getitem__
+
+    def _validate(self, fieldname=None, message=None):
+        if fieldname is None:
+            fieldname = 'other_field'
+
+        validator = validators.GreaterEqualThan(fieldname=fieldname,
+                                                message=message)
+
+        return validator(self.form_mock, self.form_field_mock)
+
+    def test_field_not_found(self):
+        self.assertRaisesRegexp(
+            validators.ValidationError,
+            "^Invalid field name 'some'.$",
+            self._validate,
+            fieldname='some',
+        )
+
+    def test_form_field_is_none(self):
+        self.form_field_mock.data = None
+
+        self.assertIsNone(self._validate())
+
+    def test_other_field_is_none(self):
+        self.other_field_mock.data = None
+
+        self.assertIsNone(self._validate())
+
+    def test_both_fields_are_none(self):
+        self.form_field_mock.data = None
+        self.other_field_mock.data = None
+
+        self.assertIsNone(self._validate())
+
+    def test_validation_pass(self):
+        self.assertIsNone(self._validate())
+
+    def test_validation_raises(self):
+        self.form_field_mock.data = '2017-05-04'
+
+        self.assertRaisesRegexp(
+            validators.ValidationError,
+            "^Field must be greater than or equal to other field.$",
+            self._validate,
+        )
+
+    def test_validation_raises_custom_message(self):
+        self.form_field_mock.data = '2017-05-04'
+
+        self.assertRaisesRegexp(
+            validators.ValidationError,
+            "^This field must be greater than or equal to MyField.$",
+            self._validate,
+            message="This field must be greater than or equal to MyField.",
+        )
+
+
+
+if __name__ == '__main__':
+    unittest.main()

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/d5d02ff7/tests/www/test_views.py
----------------------------------------------------------------------
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index a8823e6..3e2fe61 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -15,11 +15,68 @@
 import unittest
 
 from airflow import configuration
-from airflow.models import Pool
+from airflow.models import KnownEvent, Pool
 from airflow.settings import Session
 from airflow.www import app as application
 
 
+class TestKnownEventView(unittest.TestCase):
+
+    CREATE_ENDPOINT = '/admin/knownevent/new/?url=/admin/knownevent/'
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestKnownEventView, cls).setUpClass()
+        session = Session()
+        session.query(KnownEvent).delete()
+        session.commit()
+        session.close()
+
+    def setUp(self):
+        super(TestKnownEventView, self).setUp()
+        configuration.load_test_config()
+        app = application.create_app(testing=True)
+        app.config['WTF_CSRF_METHODS'] = []
+        self.app = app.test_client()
+        self.session = Session()
+        self.known_event = {
+            'label': 'event-label',
+            'event_type': 1,
+            'start_date': '2017-06-05 12:00:00',
+            'end_date': '2017-06-05 13:00:00',
+            'reported_by': 'airflow',
+            'description': '',
+        }
+
+    def tearDown(self):
+        self.session.query(KnownEvent).delete()
+        self.session.commit()
+        self.session.close()
+        super(TestKnownEventView, self).tearDown()
+
+    def test_create_known_event(self):
+        response = self.app.post(
+            self.CREATE_ENDPOINT,
+            data=self.known_event,
+            follow_redirects=True,
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(self.session.query(KnownEvent).count(), 1)
+
+    def test_create_known_event_with_end_data_earlier_than_start_date(self):
+        self.known_event['end_date'] = '2017-06-05 11:00:00'
+        response = self.app.post(
+            self.CREATE_ENDPOINT,
+            data=self.known_event,
+            follow_redirects=True,
+        )
+        self.assertIn(
+            'Field must be greater than or equal to Start Date.',
+            response.data.decode('utf-8'),
+        )
+        self.assertEqual(self.session.query(KnownEvent).count(), 0)
+
+
 class TestPoolModelView(unittest.TestCase):
 
     CREATE_ENDPOINT = '/admin/pool/new/?url=/admin/pool/'


[3/3] incubator-airflow git commit: Merge pull request #2349 from skudriashev/airflow-1277

Posted by bo...@apache.org.
Merge pull request #2349 from skudriashev/airflow-1277


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/ae5c53b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/ae5c53b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/ae5c53b6

Branch: refs/heads/master
Commit: ae5c53b6cbacc6f4b49cc1fe85a96323ee1ddb6e
Parents: e3e6aa7 65184a1
Author: Bolke de Bruin <bo...@xs4all.nl>
Authored: Wed Jun 7 09:02:36 2017 +0200
Committer: Bolke de Bruin <bo...@xs4all.nl>
Committed: Wed Jun 7 09:02:36 2017 +0200

----------------------------------------------------------------------
 airflow/www/app.py           |  2 +-
 airflow/www/validators.py    | 54 +++++++++++++++++++++++
 airflow/www/views.py         | 43 ++++++++++++++++--
 tests/www/test_validators.py | 92 +++++++++++++++++++++++++++++++++++++++
 tests/www/test_views.py      | 82 +++++++++++++++++++++++++++++++---
 5 files changed, 262 insertions(+), 11 deletions(-)
----------------------------------------------------------------------



[2/3] incubator-airflow git commit: [AIRFLOW-1277] Forbid KE creation with empty fields

Posted by bo...@apache.org.
[AIRFLOW-1277] Forbid KE creation with empty fields

KE = KnownEvent


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/65184a10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/65184a10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/65184a10

Branch: refs/heads/master
Commit: 65184a109e0179b50e4dae534a38c6d66bf080d8
Parents: d5d02ff
Author: Stanislav Kudriashev <st...@gmail.com>
Authored: Tue Jun 6 16:21:49 2017 +0300
Committer: Stanislav Kudriashev <st...@gmail.com>
Committed: Tue Jun 6 19:29:12 2017 +0300

----------------------------------------------------------------------
 airflow/www/app.py      |  2 +-
 airflow/www/views.py    | 29 +++++++++++++++++++++++++----
 tests/www/test_views.py | 37 +++++++++++++++++++++++++------------
 3 files changed, 51 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/airflow/www/app.py
----------------------------------------------------------------------
diff --git a/airflow/www/app.py b/airflow/www/app.py
index 80e9b87..1ae2731 100644
--- a/airflow/www/app.py
+++ b/airflow/www/app.py
@@ -72,7 +72,7 @@ def create_app(config=None, testing=False):
         av(vs.QueryView(name='Ad Hoc Query', category="Data Profiling"))
         av(vs.ChartModelView(
             models.Chart, Session, name="Charts", category="Data Profiling"))
-        av(vs.KnowEventView(
+        av(vs.KnownEventView(
             models.KnownEvent,
             Session, name="Known Events", category="Data Profiling"))
         av(vs.SlaMissModelView(

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index e16d201..aadd200 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2140,7 +2140,7 @@ chart_mapping = (
 chart_mapping = dict(chart_mapping)
 
 
-class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
+class KnownEventView(wwwutils.DataProfilingMixin, AirflowModelView):
     verbose_name = "known event"
     verbose_name_plural = "known events"
     form_columns = (
@@ -2152,10 +2152,31 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
         'description',
     )
     form_args = {
+        'label': {
+            'validators': [
+                validators.DataRequired(),
+            ],
+        },
+        'event_type': {
+            'validators': [
+                validators.DataRequired(),
+            ],
+        },
+        'start_date': {
+            'validators': [
+                validators.DataRequired(),
+            ],
+        },
         'end_date': {
-            'validators': {
+            'validators': [
+                validators.DataRequired(),
                 GreaterEqualThan(fieldname='start_date'),
-            }
+            ],
+        },
+        'reported_by': {
+            'validators': [
+                validators.DataRequired(),
+            ],
         }
     }
     column_list = (
@@ -2168,7 +2189,7 @@ class KnowEventView(wwwutils.DataProfilingMixin, AirflowModelView):
     column_default_sort = ("start_date", True)
 
 
-class KnowEventTypeView(wwwutils.DataProfilingMixin, AirflowModelView):
+class KnownEventTypeView(wwwutils.DataProfilingMixin, AirflowModelView):
     pass
 
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/65184a10/tests/www/test_views.py
----------------------------------------------------------------------
diff --git a/tests/www/test_views.py b/tests/www/test_views.py
index 3e2fe61..5a20cbd 100644
--- a/tests/www/test_views.py
+++ b/tests/www/test_views.py
@@ -15,7 +15,7 @@
 import unittest
 
 from airflow import configuration
-from airflow.models import KnownEvent, Pool
+from airflow import models
 from airflow.settings import Session
 from airflow.www import app as application
 
@@ -28,8 +28,13 @@ class TestKnownEventView(unittest.TestCase):
     def setUpClass(cls):
         super(TestKnownEventView, cls).setUpClass()
         session = Session()
-        session.query(KnownEvent).delete()
+        session.query(models.KnownEvent).delete()
+        session.query(models.User).delete()
         session.commit()
+        user = models.User(username='airflow')
+        session.add(user)
+        session.commit()
+        cls.user_id = user.id
         session.close()
 
     def setUp(self):
@@ -41,19 +46,27 @@ class TestKnownEventView(unittest.TestCase):
         self.session = Session()
         self.known_event = {
             'label': 'event-label',
-            'event_type': 1,
+            'event_type': '1',
             'start_date': '2017-06-05 12:00:00',
             'end_date': '2017-06-05 13:00:00',
-            'reported_by': 'airflow',
+            'reported_by': self.user_id,
             'description': '',
         }
 
     def tearDown(self):
-        self.session.query(KnownEvent).delete()
+        self.session.query(models.KnownEvent).delete()
         self.session.commit()
         self.session.close()
         super(TestKnownEventView, self).tearDown()
 
+    @classmethod
+    def tearDownClass(cls):
+        session = Session()
+        session.query(models.User).delete()
+        session.commit()
+        session.close()
+        super(TestKnownEventView, cls).tearDownClass()
+
     def test_create_known_event(self):
         response = self.app.post(
             self.CREATE_ENDPOINT,
@@ -61,7 +74,7 @@ class TestKnownEventView(unittest.TestCase):
             follow_redirects=True,
         )
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(self.session.query(KnownEvent).count(), 1)
+        self.assertEqual(self.session.query(models.KnownEvent).count(), 1)
 
     def test_create_known_event_with_end_data_earlier_than_start_date(self):
         self.known_event['end_date'] = '2017-06-05 11:00:00'
@@ -74,7 +87,7 @@ class TestKnownEventView(unittest.TestCase):
             'Field must be greater than or equal to Start Date.',
             response.data.decode('utf-8'),
         )
-        self.assertEqual(self.session.query(KnownEvent).count(), 0)
+        self.assertEqual(self.session.query(models.KnownEvent).count(), 0)
 
 
 class TestPoolModelView(unittest.TestCase):
@@ -85,7 +98,7 @@ class TestPoolModelView(unittest.TestCase):
     def setUpClass(cls):
         super(TestPoolModelView, cls).setUpClass()
         session = Session()
-        session.query(Pool).delete()
+        session.query(models.Pool).delete()
         session.commit()
         session.close()
 
@@ -103,7 +116,7 @@ class TestPoolModelView(unittest.TestCase):
         }
 
     def tearDown(self):
-        self.session.query(Pool).delete()
+        self.session.query(models.Pool).delete()
         self.session.commit()
         self.session.close()
         super(TestPoolModelView, self).tearDown()
@@ -115,7 +128,7 @@ class TestPoolModelView(unittest.TestCase):
             follow_redirects=True,
         )
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(self.session.query(Pool).count(), 1)
+        self.assertEqual(self.session.query(models.Pool).count(), 1)
 
     def test_create_pool_with_same_name(self):
         # create test pool
@@ -131,7 +144,7 @@ class TestPoolModelView(unittest.TestCase):
             follow_redirects=True,
         )
         self.assertIn('Already exists.', response.data.decode('utf-8'))
-        self.assertEqual(self.session.query(Pool).count(), 1)
+        self.assertEqual(self.session.query(models.Pool).count(), 1)
 
     def test_create_pool_with_empty_name(self):
         self.pool['pool'] = ''
@@ -141,7 +154,7 @@ class TestPoolModelView(unittest.TestCase):
             follow_redirects=True,
         )
         self.assertIn('This field is required.', response.data.decode('utf-8'))
-        self.assertEqual(self.session.query(Pool).count(), 0)
+        self.assertEqual(self.session.query(models.Pool).count(), 0)
 
 
 if __name__ == '__main__':