You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/02/10 16:07:25 UTC

incubator-airflow git commit: [AIRFLOW-1002] Add ability to clean all dependencies of removed DAG

Repository: incubator-airflow
Updated Branches:
  refs/heads/master 556c9ec5b -> 7488f2938


[AIRFLOW-1002] Add ability to clean all dependencies of removed DAG

After removing a dag file, there is no way to
clean database
except for removing corresponding records directly
for now.
This PR enables user to do this via command line.

Closes #2199 from sekikn/AIRFLOW-1002


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/7488f293
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/7488f293
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/7488f293

Branch: refs/heads/master
Commit: 7488f2938da4e08645060531aa363204db7f50a5
Parents: 556c9ec
Author: Kengo Seki <se...@apache.org>
Authored: Sat Feb 10 17:07:19 2018 +0100
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Sat Feb 10 17:07:19 2018 +0100

----------------------------------------------------------------------
 airflow/api/client/api_client.py              |  7 +++
 airflow/api/client/json_client.py             |  6 +++
 airflow/api/client/local_client.py            |  5 ++
 airflow/api/common/experimental/delete_dag.py | 55 ++++++++++++++++++++++
 airflow/bin/cli.py                            | 35 +++++++++++---
 airflow/www/api/experimental/endpoints.py     | 18 +++++++
 tests/api/client/test_local_client.py         | 13 +++++
 tests/core.py                                 | 18 +++++++
 tests/www/api/experimental/test_endpoints.py  | 22 ++++++++-
 9 files changed, 172 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/api_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/api_client.py b/airflow/api/client/api_client.py
index f24d809..719270b 100644
--- a/airflow/api/client/api_client.py
+++ b/airflow/api/client/api_client.py
@@ -32,6 +32,13 @@ class Client(object):
         """
         raise NotImplementedError()
 
+    def delete_dag(self, dag_id):
+        """Delete all DB records related to the specified dag.
+
+        :param dag_id:
+        """
+        raise NotImplementedError()
+
     def get_pool(self, name):
         """Get pool.
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/json_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/json_client.py b/airflow/api/client/json_client.py
index 37e24d3..54a5401 100644
--- a/airflow/api/client/json_client.py
+++ b/airflow/api/client/json_client.py
@@ -50,6 +50,12 @@ class Client(api_client.Client):
                              })
         return data['message']
 
+    def delete_dag(self, dag_id):
+        endpoint = '/api/experimental/dags/{}/delete_dag'.format(dag_id)
+        url = urljoin(self._api_base_url, endpoint)
+        data = self._request(url, method='DELETE')
+        return data['message']
+
     def get_pool(self, name):
         endpoint = '/api/experimental/pools/{}'.format(name)
         url = urljoin(self._api_base_url, endpoint)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/local_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py
index 5bc7f76..8cdecd0 100644
--- a/airflow/api/client/local_client.py
+++ b/airflow/api/client/local_client.py
@@ -15,6 +15,7 @@
 from airflow.api.client import api_client
 from airflow.api.common.experimental import pool
 from airflow.api.common.experimental import trigger_dag
+from airflow.api.common.experimental import delete_dag
 
 
 class Client(api_client.Client):
@@ -27,6 +28,10 @@ class Client(api_client.Client):
                                      execution_date=execution_date)
         return "Created {}".format(dr)
 
+    def delete_dag(self, dag_id):
+        count = delete_dag.delete_dag(dag_id)
+        return "Removed {} record(s)".format(count)
+
     def get_pool(self, name):
         p = pool.get_pool(name=name)
         return p.pool, p.slots, p.description

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/common/experimental/delete_dag.py
----------------------------------------------------------------------
diff --git a/airflow/api/common/experimental/delete_dag.py b/airflow/api/common/experimental/delete_dag.py
new file mode 100644
index 0000000..cb414c5
--- /dev/null
+++ b/airflow/api/common/experimental/delete_dag.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from airflow import models, settings
+from airflow.exceptions import AirflowException
+from sqlalchemy import or_
+
+
+class DagFileExists(AirflowException):
+    status = 400
+
+
+class DagNotFound(AirflowException):
+    status = 404
+
+
+def delete_dag(dag_id):
+    session = settings.Session()
+
+    DM = models.DagModel
+    dag = session.query(DM).filter(DM.dag_id == dag_id).first()
+    if dag is None:
+        raise DagNotFound("Dag id {} not found".format(dag_id))
+
+    dagbag = models.DagBag()
+    if dag_id in dagbag.dags:
+        raise DagFileExists("Dag id {} is still in DagBag. "
+                            "Remove the DAG file first.".format(dag_id))
+
+    count = 0
+
+    for m in models.Base._decl_class_registry.values():
+        if hasattr(m, "dag_id"):
+            cond = or_(m.dag_id == dag_id, m.dag_id.like(dag_id + ".%"))
+            count += session.query(m).filter(cond).delete(synchronize_session='fetch')
+
+    if dag.is_subdag:
+        p, c = dag_id.rsplit(".", 1)
+        for m in models.DagRun, models.TaskFail, models.TaskInstance:
+            count += session.query(m).filter(m.dag_id == p, m.task_id == c).delete()
+
+    session.commit()
+
+    return count

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 424fcda..bf21a54 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -203,6 +203,26 @@ def trigger_dag(args):
     log.info(message)
 
 
+def delete_dag(args):
+    """
+    Deletes all DB records related to the specified dag
+    :param args:
+    :return:
+    """
+    log = LoggingMixin().log
+    if args.yes or input(
+            "This will drop all existing records related to the specified DAG. "
+            "Proceed? (y/n)").upper() == "Y":
+        try:
+            message = api_client.delete_dag(dag_id=args.dag_id)
+        except IOError as err:
+            log.error(err)
+            raise AirflowException(err)
+        log.info(message)
+    else:
+        print("Bail.")
+
+
 def pool(args):
     log = LoggingMixin().log
 
@@ -1157,6 +1177,11 @@ class CLIFactory(object):
             ("--stdout",), "Redirect stdout to this file"),
         'log_file': Arg(
             ("-l", "--log-file"), "Location of the log file"),
+        'yes': Arg(
+            ("-y", "--yes"),
+            "Do not prompt to confirm reset. Use with care!",
+            "store_true",
+            default=False),
 
         # backfill
         'mark_success': Arg(
@@ -1374,12 +1399,6 @@ class CLIFactory(object):
             default=conf.get('webserver', 'ERROR_LOGFILE'),
             help="The logfile to store the webserver error log. Use '-' to print to "
                  "stderr."),
-        # resetdb
-        'yes': Arg(
-            ("-y", "--yes"),
-            "Do not prompt to confirm reset. Use with care!",
-            "store_true",
-            default=False),
         # scheduler
         'dag_id_opt': Arg(("-d", "--dag_id"), help="The id of the dag to run"),
         'run_duration': Arg(
@@ -1516,6 +1535,10 @@ class CLIFactory(object):
             'help': "Trigger a DAG run",
             'args': ('dag_id', 'subdir', 'run_id', 'conf', 'exec_date'),
         }, {
+            'func': delete_dag,
+            'help': "Delete all DB records related to the specified DAG",
+            'args': ('dag_id', 'yes',),
+        }, {
             'func': pool,
             'help': "CRUD operations on pools",
             "args": ('pool_set', 'pool_get', 'pool_delete'),

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/www/api/experimental/endpoints.py
----------------------------------------------------------------------
diff --git a/airflow/www/api/experimental/endpoints.py b/airflow/www/api/experimental/endpoints.py
index d1e2d19..ec1ac5b 100644
--- a/airflow/www/api/experimental/endpoints.py
+++ b/airflow/www/api/experimental/endpoints.py
@@ -15,6 +15,7 @@ import airflow.api
 
 from airflow.api.common.experimental import pool as pool_api
 from airflow.api.common.experimental import trigger_dag as trigger
+from airflow.api.common.experimental import delete_dag as delete
 from airflow.api.common.experimental.get_task import get_task
 from airflow.api.common.experimental.get_task_instance import get_task_instance
 from airflow.exceptions import AirflowException
@@ -85,6 +86,23 @@ def trigger_dag(dag_id):
     return response
 
 
+@csrf.exempt
+@api_experimental.route('/dags/<string:dag_id>', methods=['DELETE'])
+@requires_authentication
+def delete_dag(dag_id):
+    """
+    Delete all DB records related to the specified Dag.
+    """
+    try:
+        count = delete.delete_dag(dag_id)
+    except AirflowException as e:
+        _log.error(e)
+        response = jsonify(error="{}".format(e))
+        response.status_code = getattr(e, 'status', 500)
+        return response
+    return jsonify(message="Removed {} record(s)".format(count), count=count)
+
+
 @api_experimental.route('/test', methods=['GET'])
 @requires_authentication
 def test():

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/api/client/test_local_client.py
----------------------------------------------------------------------
diff --git a/tests/api/client/test_local_client.py b/tests/api/client/test_local_client.py
index 31a1712..51883c9 100644
--- a/tests/api/client/test_local_client.py
+++ b/tests/api/client/test_local_client.py
@@ -98,6 +98,19 @@ class TestLocalClient(unittest.TestCase):
                                          external_trigger=True)
             mock.reset_mock()
 
+    def test_delete_dag(self):
+        key = "my_dag_id"
+        session = settings.Session()
+        DM = models.DagModel
+        self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
+
+        session.add(DM(dag_id=key))
+        session.commit()
+        self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 1)
+
+        self.client.delete_dag(dag_id=key)
+        self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
+
     def test_get_pool(self):
         self.client.create_pool(name='foo', slots=1, description='')
         pool = self.client.get_pool(name='foo')

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index b79c1ff..f76f65e 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1334,6 +1334,24 @@ class CliTests(unittest.TestCase):
                 '-c', 'NOT JSON'])
         )
 
+    def test_delete_dag(self):
+        DM = models.DagModel
+        key = "my_dag_id"
+        session = settings.Session()
+        session.add(DM(dag_id=key))
+        session.commit()
+        cli.delete_dag(self.parser.parse_args([
+            'delete_dag', key, '--yes']))
+        self.assertEqual(session.query(DM).filter_by(dag_id=key).count(), 0)
+        self.assertRaises(
+            AirflowException,
+            cli.delete_dag,
+            self.parser.parse_args([
+                'delete_dag',
+                'does_not_exist_dag',
+                '--yes'])
+        )
+
     def test_pool_create(self):
         cli.pool(self.parser.parse_args(['pool', '-s', 'foo', '1', 'test']))
         self.assertEqual(self.session.query(models.Pool).count(), 1)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/www/api/experimental/test_endpoints.py
----------------------------------------------------------------------
diff --git a/tests/www/api/experimental/test_endpoints.py b/tests/www/api/experimental/test_endpoints.py
index 2c510a6..a73bc9f 100644
--- a/tests/www/api/experimental/test_endpoints.py
+++ b/tests/www/api/experimental/test_endpoints.py
@@ -19,7 +19,7 @@ from urllib.parse import quote_plus
 
 from airflow import configuration
 from airflow.api.common.experimental.trigger_dag import trigger_dag
-from airflow.models import DagBag, DagRun, Pool, TaskInstance
+from airflow.models import DagBag, DagModel, DagRun, Pool, TaskInstance
 from airflow.settings import Session
 from airflow.utils.timezone import datetime, utcnow
 from airflow.www import app as application
@@ -89,6 +89,26 @@ class TestApiExperimental(unittest.TestCase):
         )
         self.assertEqual(404, response.status_code)
 
+    def test_delete_dag(self):
+        url_template = '/api/experimental/dags/{}'
+
+        from airflow import settings
+        session = settings.Session()
+        key = "my_dag_id"
+        session.add(DagModel(dag_id=key))
+        session.commit()
+        response = self.app.delete(
+            url_template.format(key),
+            content_type="application/json"
+        )
+        self.assertEqual(200, response.status_code)
+
+        response = self.app.delete(
+            url_template.format('does_not_exist_dag'),
+            content_type="application/json"
+        )
+        self.assertEqual(404, response.status_code)
+
     def test_trigger_dag_for_date(self):
         url_template = '/api/experimental/dags/{}/dag_runs'
         dag_id = 'example_bash_operator'