You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by jl...@apache.org on 2017/02/13 19:32:33 UTC
incubator-airflow git commit: [AIRFLOW-842] do not query the DB with
an empty IN clause
Repository: incubator-airflow
Updated Branches:
refs/heads/master 2ce7556a5 -> 485280a9f
[AIRFLOW-842] do not query the DB with an empty IN clause
This is done to silence warnings coming from
sqlachemy, e.g.:
sqlalchemy/sql/default_comparator.py:161:
SAWarning: The IN-predicate on
"dag_run.dag_id" was invoked with an empty
sequence. This results in a
contradiction, which nonetheless can be expensive
to evaluate. Consider
alternative strategies for improved performance.
Closes #2072 from imbaczek/bug842
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/485280a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/485280a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/485280a9
Branch: refs/heads/master
Commit: 485280a9f180bd6f85ec031bd7fa100a2ba89a7b
Parents: 2ce7556
Author: Marek Baczynski <ma...@intel.com>
Authored: Mon Feb 13 14:32:13 2017 -0500
Committer: Jeremiah Lowin <jl...@apache.org>
Committed: Mon Feb 13 14:32:13 2017 -0500
----------------------------------------------------------------------
airflow/models.py | 8 ++++++++
tests/core.py | 16 +++++++++++++---
2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/485280a9/airflow/models.py
----------------------------------------------------------------------
diff --git a/airflow/models.py b/airflow/models.py
index 6cf7ad9..503f146 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -3734,6 +3734,10 @@ class DagStat(Base):
:param full_query: whether to check dag_runs for new drs not in dag_stats
:type full_query: bool
"""
+ # avoid querying with an empty IN clause
+ if not dag_ids:
+ return
+
dag_ids = set(dag_ids)
qry = (
@@ -3745,6 +3749,10 @@ class DagStat(Base):
qry.delete(synchronize_session='fetch')
session.commit()
+ # avoid querying with an empty IN clause
+ if not dirty_ids:
+ return
+
qry = (
session.query(DagRun.dag_id, DagRun.state, func.count('*'))
.filter(DagRun.dag_id.in_(dirty_ids))
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/485280a9/tests/core.py
----------------------------------------------------------------------
diff --git a/tests/core.py b/tests/core.py
index fba05f7..fcf69ad 100644
--- a/tests/core.py
+++ b/tests/core.py
@@ -978,12 +978,18 @@ class CoreTest(unittest.TestCase):
session.query(models.DagStat).delete()
session.commit()
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ models.DagStat.clean_dirty([], session=session)
+ self.assertEqual([], caught_warnings)
+
run1 = self.dag_bash.create_dagrun(
run_id="run1",
execution_date=DEFAULT_DATE,
state=State.RUNNING)
- models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ self.assertEqual([], caught_warnings)
qry = session.query(models.DagStat).all()
@@ -998,7 +1004,9 @@ class CoreTest(unittest.TestCase):
execution_date=DEFAULT_DATE+timedelta(days=1),
state=State.RUNNING)
- models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ self.assertEqual([], caught_warnings)
qry = session.query(models.DagStat).all()
@@ -1011,7 +1019,9 @@ class CoreTest(unittest.TestCase):
session.query(models.DagRun).first().state = State.SUCCESS
session.commit()
- models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ with warnings.catch_warnings(record=True) as caught_warnings:
+ models.DagStat.clean_dirty([self.dag_bash.dag_id], session=session)
+ self.assertEqual([], caught_warnings)
qry = session.query(models.DagStat).filter(models.DagStat.state == State.SUCCESS).all()
self.assertEqual(1, len(qry))