You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/02/10 16:07:25 UTC
incubator-airflow git commit: [AIRFLOW-1002] Add ability to clean all
dependencies of removed DAG
Repository: incubator-airflow
Updated Branches:
refs/heads/master 556c9ec5b -> 7488f2938
[AIRFLOW-1002] Add ability to clean all dependencies of removed DAG
After removing a dag file, there is no way to
clean database
except for removing corresponding records directly
for now.
This PR enables user to do this via command line.
Closes #2199 from sekikn/AIRFLOW-1002
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/7488f293
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/7488f293
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/7488f293
Branch: refs/heads/master
Commit: 7488f2938da4e08645060531aa363204db7f50a5
Parents: 556c9ec
Author: Kengo Seki <se...@apache.org>
Authored: Sat Feb 10 17:07:19 2018 +0100
Committer: Fokko Driesprong <fo...@godatadriven.com>
Committed: Sat Feb 10 17:07:19 2018 +0100
----------------------------------------------------------------------
airflow/api/client/api_client.py | 7 +++
airflow/api/client/json_client.py | 6 +++
airflow/api/client/local_client.py | 5 ++
airflow/api/common/experimental/delete_dag.py | 55 ++++++++++++++++++++++
airflow/bin/cli.py | 35 +++++++++++---
airflow/www/api/experimental/endpoints.py | 18 +++++++
tests/api/client/test_local_client.py | 13 +++++
tests/core.py | 18 +++++++
tests/www/api/experimental/test_endpoints.py | 22 ++++++++-
9 files changed, 172 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/api_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/api_client.py b/airflow/api/client/api_client.py
index f24d809..719270b 100644
--- a/airflow/api/client/api_client.py
+++ b/airflow/api/client/api_client.py
@@ -32,6 +32,13 @@ class Client(object):
"""
raise NotImplementedError()
+ def delete_dag(self, dag_id):
+ """Delete all DB records related to the specified dag.
+
+ :param dag_id:
+ """
+ raise NotImplementedError()
+
def get_pool(self, name):
"""Get pool.
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/json_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/json_client.py b/airflow/api/client/json_client.py
index 37e24d3..54a5401 100644
--- a/airflow/api/client/json_client.py
+++ b/airflow/api/client/json_client.py
@@ -50,6 +50,12 @@ class Client(api_client.Client):
})
return data['message']
+ def delete_dag(self, dag_id):
+ endpoint = '/api/experimental/dags/{}/delete_dag'.format(dag_id)
+ url = urljoin(self._api_base_url, endpoint)
+ data = self._request(url, method='DELETE')
+ return data['message']
+
def get_pool(self, name):
endpoint = '/api/experimental/pools/{}'.format(name)
url = urljoin(self._api_base_url, endpoint)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/client/local_client.py
----------------------------------------------------------------------
diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py
index 5bc7f76..8cdecd0 100644
--- a/airflow/api/client/local_client.py
+++ b/airflow/api/client/local_client.py
@@ -15,6 +15,7 @@
from airflow.api.client import api_client
from airflow.api.common.experimental import pool
from airflow.api.common.experimental import trigger_dag
+from airflow.api.common.experimental import delete_dag
class Client(api_client.Client):
@@ -27,6 +28,10 @@ class Client(api_client.Client):
execution_date=execution_date)
return "Created {}".format(dr)
+ def delete_dag(self, dag_id):
+ count = delete_dag.delete_dag(dag_id)
+ return "Removed {} record(s)".format(count)
+
def get_pool(self, name):
p = pool.get_pool(name=name)
return p.pool, p.slots, p.description
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/api/common/experimental/delete_dag.py
----------------------------------------------------------------------
diff --git a/airflow/api/common/experimental/delete_dag.py b/airflow/api/common/experimental/delete_dag.py
new file mode 100644
index 0000000..cb414c5
--- /dev/null
+++ b/airflow/api/common/experimental/delete_dag.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from airflow import models, settings
+from airflow.exceptions import AirflowException
+from sqlalchemy import or_
+
+
+class DagFileExists(AirflowException):
+ status = 400
+
+
+class DagNotFound(AirflowException):
+ status = 404
+
+
+def delete_dag(dag_id):
+ session = settings.Session()
+
+ DM = models.DagModel
+ dag = session.query(DM).filter(DM.dag_id == dag_id).first()
+ if dag is None:
+ raise DagNotFound("Dag id {} not found".format(dag_id))
+
+ dagbag = models.DagBag()
+ if dag_id in dagbag.dags:
+ raise DagFileExists("Dag id {} is still in DagBag. "
+ "Remove the DAG file first.".format(dag_id))
+
+ count = 0
+
+ for m in models.Base._decl_class_registry.values():
+ if hasattr(m, "dag_id"):
+ cond = or_(m.dag_id == dag_id, m.dag_id.like(dag_id + ".%"))
+ count += session.query(m).filter(cond).delete(synchronize_session='fetch')
+
+ if dag.is_subdag:
+ p, c = dag_id.rsplit(".", 1)
+ for m in models.DagRun, models.TaskFail, models.TaskInstance:
+ count += session.query(m).filter(m.dag_id == p, m.task_id == c).delete()
+
+ session.commit()
+
+ return count
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 424fcda..bf21a54 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -203,6 +203,26 @@ def trigger_dag(args):
log.info(message)
+def delete_dag(args):
+ """
+ Deletes all DB records related to the specified dag
+ :param args:
+ :return:
+ """
+ log = LoggingMixin().log
+ if args.yes or input(
+ "This will drop all existing records related to the specified DAG. "
+ "Proceed? (y/n)").upper() == "Y":
+ try:
+ message = api_client.delete_dag(dag_id=args.dag_id)
+ except IOError as err:
+ log.error(err)
+ raise AirflowException(err)
+ log.info(message)
+ else:
+ print("Bail.")
+
+
def pool(args):
log = LoggingMixin().log
@@ -1157,6 +1177,11 @@ class CLIFactory(object):
("--stdout",), "Redirect stdout to this file"),
'log_file': Arg(
("-l", "--log-file"), "Location of the log file"),
+ 'yes': Arg(
+ ("-y", "--yes"),
+ "Do not prompt to confirm reset. Use with care!",
+ "store_true",
+ default=False),
# backfill
'mark_success': Arg(
@@ -1374,12 +1399,6 @@ class CLIFactory(object):
default=conf.get('webserver', 'ERROR_LOGFILE'),
help="The logfile to store the webserver error log. Use '-' to print to "
"stderr."),
- # resetdb
- 'yes': Arg(
- ("-y", "--yes"),
- "Do not prompt to confirm reset. Use with care!",
- "store_true",
- default=False),
# scheduler
'dag_id_opt': Arg(("-d", "--dag_id"), help="The id of the dag to run"),
'run_duration': Arg(
@@ -1516,6 +1535,10 @@ class CLIFactory(object):
'help': "Trigger a DAG run",
'args': ('dag_id', 'subdir', 'run_id', 'conf', 'exec_date'),
}, {
+ 'func': delete_dag,
+ 'help': "Delete all DB records related to the specified DAG",
+ 'args': ('dag_id', 'yes',),
+ }, {
'func': pool,
'help': "CRUD operations on pools",
"args": ('pool_set', 'pool_get', 'pool_delete'),
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/airflow/www/api/experimental/endpoints.py
----------------------------------------------------------------------
diff --git a/airflow/www/api/experimental/endpoints.py b/airflow/www/api/experimental/endpoints.py
index d1e2d19..ec1ac5b 100644
--- a/airflow/www/api/experimental/endpoints.py
+++ b/airflow/www/api/experimental/endpoints.py
@@ -15,6 +15,7 @@ import airflow.api
from airflow.api.common.experimental import pool as pool_api
from airflow.api.common.experimental import trigger_dag as trigger
+from airflow.api.common.experimental import delete_dag as delete
from airflow.api.common.experimental.get_task import get_task
from airflow.api.common.experimental.get_task_instance import get_task_instance
from airflow.exceptions import AirflowException
@@ -85,6 +86,23 @@ def trigger_dag(dag_id):
return response
+@csrf.exempt
+@api_experimental.route('/dags/<string:dag_id>', methods=['DELETE'])
+@requires_authentication
+def delete_dag(dag_id):
+ """
+ Delete all DB records related to the specified Dag.
+ """
+ try:
+ count = delete.delete_dag(dag_id)
+ except AirflowException as e:
+ _log.error(e)
+ response = jsonify(error="{}".format(e))
+ response.status_code = getattr(e, 'status', 500)
+ return response
+ return jsonify(message="Removed {} record(s)".format(count), count=count)
+
+
@api_experimental.route('/test', methods=['GET'])
@requires_authentication
def test():
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/api/client/test_local_client.py
----------------------------------------------------------------------
diff --git a/tests/api/client/test_local_client.py b/tests/api/client/test_local_client.py
index 31a1712..51883c9 100644
--- a/tests/api/client/test_local_client.py
+++ b/tests/api/client/test_local_client.py
@@ -98,6 +98,19 @@ class TestLocalClient(unittest.TestCase):
external_trigger=True)
mock.reset_mock()
+ def test_delete_dag(self):
+ key = "my_dag_id"
+ session = settings.Session()
+ DM = models.DagModel
+ self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
+
+ session.add(DM(dag_id=key))
+ session.commit()
+ self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 1)
+
+ self.client.delete_dag(dag_id=key)
+ self.assertEqual(session.query(DM).filter(DM.dag_id == key).count(), 0)
+
def test_get_pool(self):
self.client.create_pool(name='foo', slots=1, description='')
pool = self.client.get_pool(name='foo')
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index b79c1ff..f76f65e 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -1334,6 +1334,24 @@ class CliTests(unittest.TestCase):
'-c', 'NOT JSON'])
)
+ def test_delete_dag(self):
+ DM = models.DagModel
+ key = "my_dag_id"
+ session = settings.Session()
+ session.add(DM(dag_id=key))
+ session.commit()
+ cli.delete_dag(self.parser.parse_args([
+ 'delete_dag', key, '--yes']))
+ self.assertEqual(session.query(DM).filter_by(dag_id=key).count(), 0)
+ self.assertRaises(
+ AirflowException,
+ cli.delete_dag,
+ self.parser.parse_args([
+ 'delete_dag',
+ 'does_not_exist_dag',
+ '--yes'])
+ )
+
def test_pool_create(self):
cli.pool(self.parser.parse_args(['pool', '-s', 'foo', '1', 'test']))
self.assertEqual(self.session.query(models.Pool).count(), 1)
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/7488f293/tests/www/api/experimental/test_endpoints.py
----------------------------------------------------------------------
diff --git a/tests/www/api/experimental/test_endpoints.py b/tests/www/api/experimental/test_endpoints.py
index 2c510a6..a73bc9f 100644
--- a/tests/www/api/experimental/test_endpoints.py
+++ b/tests/www/api/experimental/test_endpoints.py
@@ -19,7 +19,7 @@ from urllib.parse import quote_plus
from airflow import configuration
from airflow.api.common.experimental.trigger_dag import trigger_dag
-from airflow.models import DagBag, DagRun, Pool, TaskInstance
+from airflow.models import DagBag, DagModel, DagRun, Pool, TaskInstance
from airflow.settings import Session
from airflow.utils.timezone import datetime, utcnow
from airflow.www import app as application
@@ -89,6 +89,26 @@ class TestApiExperimental(unittest.TestCase):
)
self.assertEqual(404, response.status_code)
+ def test_delete_dag(self):
+ url_template = '/api/experimental/dags/{}'
+
+ from airflow import settings
+ session = settings.Session()
+ key = "my_dag_id"
+ session.add(DagModel(dag_id=key))
+ session.commit()
+ response = self.app.delete(
+ url_template.format(key),
+ content_type="application/json"
+ )
+ self.assertEqual(200, response.status_code)
+
+ response = self.app.delete(
+ url_template.format('does_not_exist_dag'),
+ content_type="application/json"
+ )
+ self.assertEqual(404, response.status_code)
+
def test_trigger_dag_for_date(self):
url_template = '/api/experimental/dags/{}/dag_runs'
dag_id = 'example_bash_operator'