You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2019/11/27 19:10:21 UTC
[airflow-site] 08/30: v1.8.0
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
commit 69cff49228f32d88fcdf3e92808ad51ab3438d8d
Author: Maxime Beauchemin <ma...@apache.org>
AuthorDate: Mon Mar 20 17:04:37 2017 -0700
v1.8.0
---
.../airflow/contrib/executors/mesos_executor.html | 2 +-
_modules/airflow/models.html | 134 +++++++++++++--------
_modules/airflow/operators/sensors.html | 2 +-
_modules/mysql_hook.html | 15 +--
_modules/mysql_operator.html | 8 +-
_modules/sensors.html | 2 +-
_sources/concepts.rst.txt | 3 +-
_sources/configuration.rst.txt | 35 ------
code.html | 27 +++--
concepts.html | 3 +-
configuration.html | 28 -----
genindex.html | 2 +
index.html | 1 -
objects.inv | Bin 2147 -> 2159 bytes
searchindex.js | 2 +-
15 files changed, 123 insertions(+), 141 deletions(-)
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index dbca37d..311d1a8 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -331,7 +331,7 @@
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
<span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span>
<span class="c1"># Discard these tasks.</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span>
<span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index a94686b..0b043ea 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -632,7 +632,7 @@
<span class="k">def</span> <span class="nf">paused_dags</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="n">session</span> <span class="o">=</span> <span class="n">settings</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
<span class="n">dag_ids</span> <span class="o">=</span> <span class="p">[</span><span class="n">dp</span><span class="o">.</span><span class="n">dag_id</span> <span class="k">for</span> <span class="n">dp</span> <span class="ow">in</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagModel</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span>
- <span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="kc">True</span><span class="p">))]</span>
+ <span class="n">DagModel</span><span class="o">.</span><span class="n">is_paused</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">True</span><span class="p">))]</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="n">session</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
<span class="k">return</span> <span class="n">dag_ids</span></div>
@@ -1161,6 +1161,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">end_date</span>
<span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">try_number</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hostname</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">hostname</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pid</span> <span class="o">=</span> <span class="n">ti</span><span class="o">.</span><span class="n">pid</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="kc">None</span></div>
@@ -1452,19 +1453,20 @@
<span class="n">verbose</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">runnable</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">mark_success</span><span class="p">:</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">!=</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span><span class="p">:</span>
- <span class="c1"># If a task's dependencies are met but it can't be run yet then queue it</span>
- <span class="c1"># instead</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">QUEUED</span>
- <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Queuing attempt </span><span class="si">{attempt}</span><span class="s2"> of </span><span class="si">{total}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
- <span class="n">attempt</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">%</span> <span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
- <span class="n">total</span><span class="o">=</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">hr</span> <span class="o">+</span> <span class="n">msg</span> <span class="o">+</span> <span class="n">hr</span><span class="p">)</span>
-
- <span class="bp">self</span><span class="o">.</span><span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
- <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Queuing into pool </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
- <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+ <span class="c1"># FIXME: we might have hit concurrency limits, which means we probably</span>
+ <span class="c1"># have been running prematurely. This should be handled in the</span>
+ <span class="c1"># scheduling mechanism.</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">NONE</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"FIXME: Rescheduling due to concurrency limits reached at task "</span>
+ <span class="s2">"runtime. Attempt </span><span class="si">{attempt}</span><span class="s2"> of </span><span class="si">{total}</span><span class="s2">. State set to NONE."</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="n">attempt</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">try_number</span> <span class="o">%</span> <span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
+ <span class="n">total</span><span class="o">=</span><span class="n">task</span><span class="o">.</span><span class="n">retries</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">hr</span> <span class="o">+</span> <span class="n">msg</span> <span class="o">+</span> <span class="n">hr</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">queued_dttm</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">now</span><span class="p">()</span>
+ <span class="n">msg</span> <span class="o">=</span> <span class="s2">"Queuing into pool </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">pool</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
+ <span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span>
@@ -1472,7 +1474,7 @@
<span class="c1"># the current worker process was blocked on refresh_from_db</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
<span class="n">msg</span> <span class="o">=</span> <span class="s2">"Task Instance already running </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
<span class="k">return</span>
@@ -1483,6 +1485,7 @@
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">Log</span><span class="p">(</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> <span class="bp">self</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">pid</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">getpid</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">end_date</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">test_mode</span><span class="p">:</span>
<span class="n">session</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
@@ -1538,22 +1541,7 @@
<span class="k">if</span> <span class="n">result</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">XCOM_RETURN_KEY</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
- <span class="c1"># TODO remove deprecated behavior in Airflow 2.0</span>
- <span class="k">try</span><span class="p">:</span>
- <span class="n">task_copy</span><span class="o">.</span><span class="n">post_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="n">result</span><span class="p">)</span>
- <span class="k">except</span> <span class="ne">TypeError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
- <span class="k">if</span> <span class="s1">'unexpected keyword argument'</span> <span class="ow">in</span> <span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">):</span>
- <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
- <span class="s1">'BaseOperator.post_execute() now takes two '</span>
- <span class="s1">'arguments, `context` and `result`, but "</span><span class="si">{}</span><span class="s1">" only '</span>
- <span class="s1">'expected one. This behavior is deprecated and '</span>
- <span class="s1">'will be removed in a future version of '</span>
- <span class="s1">'Airflow.'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">),</span>
- <span class="n">category</span><span class="o">=</span><span class="ne">DeprecationWarning</span><span class="p">)</span>
- <span class="n">task_copy</span><span class="o">.</span><span class="n">post_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
- <span class="k">else</span><span class="p">:</span>
- <span class="k">raise</span>
-
+ <span class="n">task_copy</span><span class="o">.</span><span class="n">post_execute</span><span class="p">(</span><span class="n">context</span><span class="o">=</span><span class="n">context</span><span class="p">)</span>
<span class="n">Stats</span><span class="o">.</span><span class="n">incr</span><span class="p">(</span><span class="s1">'operator_successes_</span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
@@ -1931,7 +1919,7 @@
<span class="sd"> dag crawling behavior. To derive this class, you are expected to override</span>
<span class="sd"> the constructor as well as the 'execute' method.</span>
-<span class="sd"> Operators derived from this class should perform or trigger certain tasks</span>
+<span class="sd"> Operators derived from this task should perform or trigger certain tasks</span>
<span class="sd"> synchronously (wait for completion). Example of operators could be an</span>
<span class="sd"> operator the runs a Pig job (PigOperator), a sensor operator that</span>
<span class="sd"> waits for a partition to land in Hive (HiveSensorOperator), or one that</span>
@@ -2333,7 +2321,8 @@
<div class="viewcode-block" id="BaseOperator.pre_execute"><a class="viewcode-back" href="../../code.html#airflow.models.BaseOperator.pre_execute">[docs]</a> <span class="k">def</span> <span class="nf">pre_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> This hook is triggered right before self.execute() is called.</span>
+<span class="sd"> This is triggered right before self.execute, it's mostly a hook</span>
+<span class="sd"> for people deriving operators.</span>
<span class="sd"> """</span>
<span class="k">pass</span></div>
@@ -2346,11 +2335,10 @@
<span class="sd"> """</span>
<span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">()</span></div>
-<div class="viewcode-block" id="BaseOperator.post_execute"><a class="viewcode-back" href="../../code.html#airflow.models.BaseOperator.post_execute">[docs]</a> <span class="k">def</span> <span class="nf">post_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">,</span> <span class="n">result</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+<div class="viewcode-block" id="BaseOperator.post_execute"><a class="viewcode-back" href="../../code.html#airflow.models.BaseOperator.post_execute">[docs]</a> <span class="k">def</span> <span class="nf">post_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="sd">"""</span>
-<span class="sd"> This hook is triggered right after self.execute() is called.</span>
-<span class="sd"> It is passed the execution context and any results returned by the</span>
-<span class="sd"> operator.</span>
+<span class="sd"> This is triggered right after self.execute, it's mostly a hook</span>
+<span class="sd"> for people deriving operators.</span>
<span class="sd"> """</span>
<span class="k">pass</span></div>
@@ -2859,6 +2847,8 @@
<span class="bp">self</span><span class="o">.</span><span class="n">orientation</span> <span class="o">=</span> <span class="n">orientation</span>
<span class="bp">self</span><span class="o">.</span><span class="n">catchup</span> <span class="o">=</span> <span class="n">catchup</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">False</span>
+
<span class="bp">self</span><span class="o">.</span><span class="n">_comps</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">'dag_id'</span><span class="p">,</span>
<span class="s1">'task_ids'</span><span class="p">,</span>
@@ -2964,7 +2954,7 @@
<span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">include_externally_triggered</span><span class="p">:</span>
- <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">external_trigger</span><span class="o">.</span><span class="n">__eq__</span><span class="p">(</span><span class="kc">False</span><span class="p">))</span>
+ <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">external_trigger</span><span class="o">.</span><span class="n">is_</span><span class="p">(</span><span class="kc">False</span><span class="p">))</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">order_by</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">execution_date</span><span class="o">.</span><span class="n">desc</span><span class="p">())</span>
@@ -3196,6 +3186,56 @@
<span class="k">def</span> <span class="nf">roots</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">[</span><span class="n">t</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">t</span><span class="o">.</span><span class="n">downstream_list</span><span class="p">]</span>
+<div class="viewcode-block" id="DAG.topological_sort"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.topological_sort">[docs]</a> <span class="k">def</span> <span class="nf">topological_sort</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Sorts tasks in topographical order, such that a task comes after any of its</span>
+<span class="sd"> upstream dependencies.</span>
+
+<span class="sd"> Heavily inspired by:</span>
+<span class="sd"> http://blog.jupo.org/2012/04/06/topological-sorting-acyclic-directed-graphs/</span>
+<span class="sd"> :returns: list of tasks in topological order</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># copy the the tasks so we leave it unmodified</span>
+ <span class="n">graph_unsorted</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">[:]</span>
+
+ <span class="n">graph_sorted</span> <span class="o">=</span> <span class="p">[]</span>
+
+ <span class="c1"># special case</span>
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">graph_sorted</span><span class="p">)</span>
+
+ <span class="c1"># Run until the unsorted graph is empty.</span>
+ <span class="k">while</span> <span class="n">graph_unsorted</span><span class="p">:</span>
+ <span class="c1"># Go through each of the node/edges pairs in the unsorted</span>
+ <span class="c1"># graph. If a set of edges doesn't contain any nodes that</span>
+ <span class="c1"># haven't been resolved, that is, that are still in the</span>
+ <span class="c1"># unsorted graph, remove the pair from the unsorted graph,</span>
+ <span class="c1"># and append it to the sorted graph. Note here that by using</span>
+ <span class="c1"># using the items() method for iterating, a copy of the</span>
+ <span class="c1"># unsorted graph is used, allowing us to modify the unsorted</span>
+ <span class="c1"># graph as we move through it. We also keep a flag for</span>
+ <span class="c1"># checking that that graph is acyclic, which is true if any</span>
+ <span class="c1"># nodes are resolved during each pass through the graph. If</span>
+ <span class="c1"># not, we need to bail out as the graph therefore can't be</span>
+ <span class="c1"># sorted.</span>
+ <span class="n">acyclic</span> <span class="o">=</span> <span class="kc">False</span>
+ <span class="k">for</span> <span class="n">node</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">graph_unsorted</span><span class="p">):</span>
+ <span class="k">for</span> <span class="n">edge</span> <span class="ow">in</span> <span class="n">node</span><span class="o">.</span><span class="n">upstream_list</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">edge</span> <span class="ow">in</span> <span class="n">graph_unsorted</span><span class="p">:</span>
+ <span class="k">break</span>
+ <span class="c1"># no edges in upstream tasks</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">acyclic</span> <span class="o">=</span> <span class="kc">True</span>
+ <span class="n">graph_unsorted</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="n">node</span><span class="p">)</span>
+ <span class="n">graph_sorted</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">node</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">acyclic</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"A cyclic dependency occurred in dag: </span><span class="si">{}</span><span class="s2">"</span>
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag_id</span><span class="p">))</span>
+
+ <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">graph_sorted</span><span class="p">)</span></div>
+
<span class="nd">@provide_session</span>
<span class="k">def</span> <span class="nf">set_dag_runs_state</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
@@ -3314,6 +3354,10 @@
<span class="n">tid</span> <span class="k">for</span> <span class="n">tid</span> <span class="ow">in</span> <span class="n">t</span><span class="o">.</span><span class="n">_upstream_task_ids</span> <span class="k">if</span> <span class="n">tid</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">]</span>
<span class="n">t</span><span class="o">.</span><span class="n">_downstream_task_ids</span> <span class="o">=</span> <span class="p">[</span>
<span class="n">tid</span> <span class="k">for</span> <span class="n">tid</span> <span class="ow">in</span> <span class="n">t</span><span class="o">.</span><span class="n">_downstream_task_ids</span> <span class="k">if</span> <span class="n">tid</span> <span class="ow">in</span> <span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">]</span>
+
+ <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">dag</span><span class="o">.</span><span class="n">tasks</span><span class="p">)</span> <span class="o"><</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tasks</span><span class="p">):</span>
+ <span class="n">dag</span><span class="o">.</span><span class="n">partial</span> <span class="o">=</span> <span class="kc">True</span>
+
<span class="k">return</span> <span class="n">dag</span></div>
<span class="k">def</span> <span class="nf">has_task</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">task_id</span><span class="p">):</span>
@@ -3912,10 +3956,6 @@
<span class="sd"> :param full_query: whether to check dag_runs for new drs not in dag_stats</span>
<span class="sd"> :type full_query: bool</span>
<span class="sd"> """</span>
- <span class="c1"># avoid querying with an empty IN clause</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">dag_ids</span><span class="p">:</span>
- <span class="k">return</span>
-
<span class="n">dag_ids</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">dag_ids</span><span class="p">)</span>
<span class="n">qry</span> <span class="o">=</span> <span class="p">(</span>
@@ -3927,10 +3967,6 @@
<span class="n">qry</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">synchronize_session</span><span class="o">=</span><span class="s1">'fetch'</span><span class="p">)</span>
<span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
- <span class="c1"># avoid querying with an empty IN clause</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">dirty_ids</span><span class="p">:</span>
- <span class="k">return</span>
-
<span class="n">qry</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">DagRun</span><span class="o">.</span><span class="n">state</span><span class="p">,</span> <span class="n">func</span><span class="o">.</span><span class="n">count</span><span class="p">(</span><span class="s1">'*'</span><span class="p">))</span>
<span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DagRun</span><span class="o">.</span><span class="n">dag_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">dirty_ids</span><span class="p">))</span>
@@ -4082,6 +4118,9 @@
<span class="k">else</span><span class="p">:</span>
<span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="n">state</span><span class="p">))</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">partial</span><span class="p">:</span>
+ <span class="n">tis</span> <span class="o">=</span> <span class="n">tis</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">TI</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">in_</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">task_ids</span><span class="p">))</span>
+
<span class="k">return</span> <span class="n">tis</span><span class="o">.</span><span class="n">all</span><span class="p">()</span>
<span class="nd">@provide_session</span>
@@ -4142,6 +4181,7 @@
<span class="sd"> """</span>
<span class="n">dag</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_dag</span><span class="p">()</span>
+
<span class="n">tis</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_task_instances</span><span class="p">(</span><span class="n">session</span><span class="o">=</span><span class="n">session</span><span class="p">)</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Updating state for </span><span class="si">{}</span><span class="s2"> considering </span><span class="si">{}</span><span class="s2"> task(s)"</span>
@@ -4226,7 +4266,7 @@
<span class="k">try</span><span class="p">:</span>
<span class="n">dag</span><span class="o">.</span><span class="n">get_task</span><span class="p">(</span><span class="n">ti</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span>
<span class="k">except</span> <span class="n">AirflowException</span><span class="p">:</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">state</span> <span class="ow">is</span> <span class="ow">not</span> <span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dag</span><span class="o">.</span><span class="n">partial</span><span class="p">:</span>
<span class="n">ti</span><span class="o">.</span><span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">REMOVED</span>
<span class="c1"># check for missing tasks</span>
diff --git a/_modules/airflow/operators/sensors.html b/_modules/airflow/operators/sensors.html
index 6d51369..03643fe 100644
--- a/_modules/airflow/operators/sensors.html
+++ b/_modules/airflow/operators/sensors.html
@@ -806,7 +806,7 @@
<span class="sd"> depends on the option that's being modified.</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'params'</span><span class="p">)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,)</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
diff --git a/_modules/mysql_hook.html b/_modules/mysql_hook.html
index 8c0c5c9..a8f6aea 100644
--- a/_modules/mysql_hook.html
+++ b/_modules/mysql_hook.html
@@ -183,22 +183,18 @@
<div class="viewcode-block" id="MySqlHook"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook">[docs]</a><span class="k">class</span> <span class="nc">MySqlHook</span><span class="p">(</span><span class="n">DbApiHook</span><span class="p">):</span>
- <span class="sd">"""</span>
+ <span class="sd">'''</span>
<span class="sd"> Interact with MySQL.</span>
<span class="sd"> You can specify charset in the extra field of your connection</span>
<span class="sd"> as ``{"charset": "utf8"}``. Also you can choose cursor as</span>
<span class="sd"> ``{"cursor": "SSCursor"}``. Refer to the MySQLdb.cursors for more details.</span>
-<span class="sd"> """</span>
+<span class="sd"> '''</span>
<span class="n">conn_name_attr</span> <span class="o">=</span> <span class="s1">'mysql_conn_id'</span>
<span class="n">default_conn_name</span> <span class="o">=</span> <span class="s1">'mysql_default'</span>
<span class="n">supports_autocommit</span> <span class="o">=</span> <span class="kc">True</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">MySqlHook</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s2">"schema"</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
-
<div class="viewcode-block" id="MySqlHook.get_conn"><a class="viewcode-back" href="../code.html#airflow.hooks.MySqlHook.get_conn">[docs]</a> <span class="k">def</span> <span class="nf">get_conn</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a mysql connection object</span>
@@ -206,16 +202,17 @@
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
<span class="n">conn_config</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">"user"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">login</span><span class="p">,</span>
- <span class="s2">"passwd"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span><span class="p">,</span>
- <span class="s2">"host"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span><span class="p">,</span>
- <span class="s2">"db"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span>
+ <span class="s2">"passwd"</span><span class="p">:</span> <span class="n">conn</span><span class="o">.</span><span class="n">password</span> <span class="ow">or</span> <span class="s1">''</span>
<span class="p">}</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"host"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span> <span class="ow">or</span> <span class="s1">'localhost'</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
<span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="mi">3306</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">conn_config</span><span class="p">[</span><span class="s2">"port"</span><span class="p">]</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span>
+ <span class="n">conn_config</span><span class="p">[</span><span class="s2">"db"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="ow">or</span> <span class="s1">''</span>
+
<span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'charset'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
<span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">extra_dejson</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">]</span>
<span class="k">if</span> <span class="p">(</span><span class="n">conn_config</span><span class="p">[</span><span class="s2">"charset"</span><span class="p">])</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">'utf8'</span> <span class="ow">or</span>\
diff --git a/_modules/mysql_operator.html b/_modules/mysql_operator.html
index 09a5aab..a60fab2 100644
--- a/_modules/mysql_operator.html
+++ b/_modules/mysql_operator.html
@@ -193,8 +193,6 @@
<span class="sd"> :type sql: Can receive a str representing a sql statement,</span>
<span class="sd"> a list of str (sql statements), or reference to a template file.</span>
<span class="sd"> Template reference are recognized by str ending in '.sql'</span>
-<span class="sd"> :param database: name of database which overwrite defined one in connection</span>
-<span class="sd"> :type database: string</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,)</span>
@@ -204,18 +202,16 @@
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span> <span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span> <span class="n">parameters</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
- <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">database</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="n">autocommit</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">MySqlOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span> <span class="o">=</span> <span class="n">autocommit</span>
<span class="bp">self</span><span class="o">.</span><span class="n">parameters</span> <span class="o">=</span> <span class="n">parameters</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">database</span> <span class="o">=</span> <span class="n">database</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: '</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">))</span>
- <span class="n">hook</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">,</span>
- <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">database</span><span class="p">)</span>
+ <span class="n">hook</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
<span class="n">hook</span><span class="o">.</span><span class="n">run</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span>
<span class="n">autocommit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">autocommit</span><span class="p">,</span>
diff --git a/_modules/sensors.html b/_modules/sensors.html
index f2b1cfe..604c19e 100644
--- a/_modules/sensors.html
+++ b/_modules/sensors.html
@@ -806,7 +806,7 @@
<span class="sd"> depends on the option that's being modified.</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'params'</span><span class="p">)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,)</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
diff --git a/_sources/concepts.rst.txt b/_sources/concepts.rst.txt
index 48c15a4..9f65256 100644
--- a/_sources/concepts.rst.txt
+++ b/_sources/concepts.rst.txt
@@ -207,7 +207,8 @@ We can put this all together to build a simple pipeline:
with DAG('my_dag', start_date=datetime(2016, 1, 1)) as dag:
(
- DummyOperator(task_id='dummy_1')
+ dag
+ >> DummyOperator(task_id='dummy_1')
>> BashOperator(
task_id='bash_1',
bash_command='echo "HELLO!"')
diff --git a/_sources/configuration.rst.txt b/_sources/configuration.rst.txt
index 5ff4284..c4a3442 100644
--- a/_sources/configuration.rst.txt
+++ b/_sources/configuration.rst.txt
@@ -131,41 +131,6 @@ to monitor your workers. You can use the shortcut command ``airflow flower``
to start a Flower web server.
-Scaling Out with Dask
-'''''''''''''''''''''
-
-``DaskExecutor`` allows you to run Airflow tasks in a Dask Distributed cluster.
-
-Dask clusters can be run on a single machine or on remote networks. For complete
-details, consult the `Distributed documentation <https://distributed.readthedocs.io/>`_.
-
-To create a cluster, first start a Scheduler:
-
-.. code-block:: bash
-
- # default settings for a local cluster
- DASK_HOST=127.0.0.1
- DASK_PORT=8786
-
- dask-scheduler --host $DASK_HOST --port $DASK_PORT
-
-Next start at least one Worker on any machine that can connect to the host:
-
-.. code-block:: bash
-
- dask-worker $DASK_HOST:$DASK_PORT
-
-Edit your ``airflow.cfg`` to set your executor to ``DaskExecutor`` and provide
-the Dask Scheduler address in the ``[dask]`` section.
-
-Please note:
-
-- Each Dask worker must be able to import Airflow and any dependencies you
- require.
-- Dask does not support queues. If an Airflow task was created with a queue, a
- warning will be raised but the task will be submitted to the cluster.
-
-
Logs
''''
Users can specify a logs folder in ``airflow.cfg``. By default, it is in
diff --git a/code.html b/code.html
index 732db2e..e37b111 100644
--- a/code.html
+++ b/code.html
@@ -217,7 +217,7 @@ DAGs.</p>
become node in the dag, BaseOperator contains many recursive methods for
dag crawling behavior. To derive this class, you are expected to override
the constructor as well as the ‘execute’ method.</p>
-<p>Operators derived from this class should perform or trigger certain tasks
+<p>Operators derived from this task should perform or trigger certain tasks
synchronously (wait for completion). Example of operators could be an
operator the runs a Pig job (PigOperator), a sensor operator that
waits for a partition to land in Hive (HiveSensorOperator), or one that
@@ -674,7 +674,7 @@ Examples: <code class="docutils literal"><span class="pre">ds=2016-01-01</span><
<dl class="class">
<dt id="airflow.operators.MySqlOperator">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">MySqlOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>mysql_conn_id='mysql_default'</em>, <em>parameters=None</em>, <em>autocommit=False</em>, <em>database=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mysql_operator.html#MySqlOperator"><span class="viewcode-link">[source]</span></a><a class= [...]
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">MySqlOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>mysql_conn_id='mysql_default'</em>, <em>parameters=None</em>, <em>autocommit=False</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/mysql_operator.html#MySqlOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airf [...]
<dd><p>Bases: <a class="reference internal" href="#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
<p>Executes sql code in a specific MySQL database</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -686,7 +686,6 @@ Examples: <code class="docutils literal"><span class="pre">ds=2016-01-01</span><
<li><strong>sql</strong> (<em>Can receive a str representing a sql statement</em><em>,
</em><em>a list of str</em><em> (</em><em>sql statements</em><em>)</em><em></em><em>, or </em><em>reference to a template file.
Template reference are recognized by str ending in '.sql'</em>) – the sql code to be executed</li>
-<li><strong>database</strong> (<em>string</em>) – name of database which overwrite defined one in connection</li>
</ul>
</td>
</tr>
@@ -1557,6 +1556,16 @@ SubDagOperator.</p>
</dd></dl>
<dl class="method">
+<dt id="airflow.models.DAG.topological_sort">
+<code class="descname">topological_sort</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.topological_sort"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.topological_sort" title="Permalink to this definition">¶</a></dt>
+<dd><p>Sorts tasks in topographical order, such that a task comes after any of its
+upstream dependencies.</p>
+<p>Heavily inspired by:
+<a class="reference external" href="http://blog.jupo.org/2012/04/06/topological-sorting-acyclic-directed-graphs/">http://blog.jupo.org/2012/04/06/topological-sorting-acyclic-directed-graphs/</a>
+:returns: list of tasks in topological order</p>
+</dd></dl>
+
+<dl class="method">
<dt id="airflow.models.DAG.tree_view">
<code class="descname">tree_view</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.tree_view"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.tree_view" title="Permalink to this definition">¶</a></dt>
<dd><p>Shows an ascii tree representation of the DAG</p>
@@ -1572,7 +1581,7 @@ SubDagOperator.</p>
become node in the dag, BaseOperator contains many recursive methods for
dag crawling behavior. To derive this class, you are expected to override
the constructor as well as the ‘execute’ method.</p>
-<p>Operators derived from this class should perform or trigger certain tasks
+<p>Operators derived from this task should perform or trigger certain tasks
synchronously (wait for completion). Example of operators could be an
operator the runs a Pig job (PigOperator), a sensor operator that
waits for a partition to land in Hive (HiveSensorOperator), or one that
@@ -1754,16 +1763,16 @@ ghost processes behind.</p>
<dl class="method">
<dt id="airflow.models.BaseOperator.post_execute">
-<code class="descname">post_execute</code><span class="sig-paren">(</span><em>context</em>, <em>result=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#BaseOperator.post_execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.BaseOperator.post_execute" title="Permalink to this definition">¶</a></dt>
-<dd><p>This hook is triggered right after self.execute() is called.
-It is passed the execution context and any results returned by the
-operator.</p>
+<code class="descname">post_execute</code><span class="sig-paren">(</span><em>context</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#BaseOperator.post_execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.BaseOperator.post_execute" title="Permalink to this definition">¶</a></dt>
+<dd><p>This is triggered right after self.execute, it’s mostly a hook
+for people deriving operators.</p>
</dd></dl>
<dl class="method">
<dt id="airflow.models.BaseOperator.pre_execute">
<code class="descname">pre_execute</code><span class="sig-paren">(</span><em>context</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#BaseOperator.pre_execute"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.BaseOperator.pre_execute" title="Permalink to this definition">¶</a></dt>
-<dd><p>This hook is triggered right before self.execute() is called.</p>
+<dd><p>This is triggered right before self.execute, it’s mostly a hook
+for people deriving operators.</p>
</dd></dl>
<dl class="method">
diff --git a/concepts.html b/concepts.html
index 1ed51a5..a329ae8 100644
--- a/concepts.html
+++ b/concepts.html
@@ -367,7 +367,8 @@ object is always returned. For example:</p>
<p>We can put this all together to build a simple pipeline:</p>
<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="k">with</span> <span class="n">DAG</span><span class="p">(</span><span class="s1">'my_dag'</span><span class="p">,</span> <span class="n">start_date</span><span class="o">=</span><span class="n">datetime</span><span class="p">(</span><span class="mi">2016</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span> [...]
<span class="p">(</span>
- <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'dummy_1'</span><span class="p">)</span>
+ <span class="n">dag</span>
+ <span class="o">>></span> <span class="n">DummyOperator</span><span class="p">(</span><span class="n">task_id</span><span class="o">=</span><span class="s1">'dummy_1'</span><span class="p">)</span>
<span class="o">>></span> <span class="n">BashOperator</span><span class="p">(</span>
<span class="n">task_id</span><span class="o">=</span><span class="s1">'bash_1'</span><span class="p">,</span>
<span class="n">bash_command</span><span class="o">=</span><span class="s1">'echo "HELLO!"'</span><span class="p">)</span>
diff --git a/configuration.html b/configuration.html
index dc4577c..ea1d2e9 100644
--- a/configuration.html
+++ b/configuration.html
@@ -92,7 +92,6 @@
<li class="toctree-l2"><a class="reference internal" href="#setting-up-a-backend">Setting up a Backend</a></li>
<li class="toctree-l2"><a class="reference internal" href="#connections">Connections</a></li>
<li class="toctree-l2"><a class="reference internal" href="#scaling-out-with-celery">Scaling Out with Celery</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#scaling-out-with-dask">Scaling Out with Dask</a></li>
<li class="toctree-l2"><a class="reference internal" href="#logs">Logs</a></li>
<li class="toctree-l2"><a class="reference internal" href="#scaling-out-on-mesos-community-contributed">Scaling Out on Mesos (community contributed)</a></li>
<li class="toctree-l2"><a class="reference internal" href="#integration-with-systemd">Integration with systemd</a></li>
@@ -285,33 +284,6 @@ its direction.</p>
to monitor your workers. You can use the shortcut command <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">flower</span></code>
to start a Flower web server.</p>
</div>
-<div class="section" id="scaling-out-with-dask">
-<h2>Scaling Out with Dask<a class="headerlink" href="#scaling-out-with-dask" title="Permalink to this headline">¶</a></h2>
-<p><code class="docutils literal"><span class="pre">DaskExecutor</span></code> allows you to run Airflow tasks in a Dask Distributed cluster.</p>
-<p>Dask clusters can be run on a single machine or on remote networks. For complete
-details, consult the <a class="reference external" href="https://distributed.readthedocs.io/">Distributed documentation</a>.</p>
-<p>To create a cluster, first start a Scheduler:</p>
-<div class="highlight-bash"><div class="highlight"><pre><span></span><span class="c1"># default settings for a local cluster</span>
-<span class="nv">DASK_HOST</span><span class="o">=</span>127.0.0.1
-<span class="nv">DASK_PORT</span><span class="o">=</span>8786
-
-dask-scheduler --host <span class="nv">$DASK_HOST</span> --port <span class="nv">$DASK_PORT</span>
-</pre></div>
-</div>
-<p>Next start at least one Worker on any machine that can connect to the host:</p>
-<div class="highlight-bash"><div class="highlight"><pre><span></span>dask-worker <span class="nv">$DASK_HOST</span>:<span class="nv">$DASK_PORT</span>
-</pre></div>
-</div>
-<p>Edit your <code class="docutils literal"><span class="pre">airflow.cfg</span></code> to set your executor to <code class="docutils literal"><span class="pre">DaskExecutor</span></code> and provide
-the Dask Scheduler address in the <code class="docutils literal"><span class="pre">[dask]</span></code> section.</p>
-<p>Please note:</p>
-<ul class="simple">
-<li>Each Dask worker must be able to import Airflow and any dependencies you
-require.</li>
-<li>Dask does not support queues. If an Airflow task was created with a queue, a
-warning will be raised but the task will be submitted to the cluster.</li>
-</ul>
-</div>
<div class="section" id="logs">
<h2>Logs<a class="headerlink" href="#logs" title="Permalink to this headline">¶</a></h2>
<p>Users can specify a logs folder in <code class="docutils literal"><span class="pre">airflow.cfg</span></code>. By default, it is in
diff --git a/genindex.html b/genindex.html
index 76acd11..bb1eacf 100644
--- a/genindex.html
+++ b/genindex.html
@@ -655,6 +655,8 @@
</li>
<li><a href="code.html#airflow.operators.TimeSensor">TimeSensor (class in airflow.operators)</a>
</li>
+ <li><a href="code.html#airflow.models.DAG.topological_sort">topological_sort() (airflow.models.DAG method)</a>
+</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="code.html#airflow.models.DAG.tree_view">tree_view() (airflow.models.DAG method)</a>
diff --git a/index.html b/index.html
index d3db9c4..eec44c1 100644
--- a/index.html
+++ b/index.html
@@ -258,7 +258,6 @@ unit of work and continuity.</p>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#setting-up-a-backend">Setting up a Backend</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#connections">Connections</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#scaling-out-with-celery">Scaling Out with Celery</a></li>
-<li class="toctree-l2"><a class="reference internal" href="configuration.html#scaling-out-with-dask">Scaling Out with Dask</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#logs">Logs</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#scaling-out-on-mesos-community-contributed">Scaling Out on Mesos (community contributed)</a></li>
<li class="toctree-l2"><a class="reference internal" href="configuration.html#integration-with-systemd">Integration with systemd</a></li>
diff --git a/objects.inv b/objects.inv
index 98f139d..6723914 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/searchindex.js b/searchindex.js
index 625cb26..121f5a8 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["api","cli","code","concepts","configuration","faq","index","installation","integration","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],envversion:50,filenames:["api.rst","cli.rst","code.rst","concepts.rst","configuration.rst","faq.rst","index.rst","installation.rst","integration.rst","license.rst","plugins.rst","profiling.rst","project.rst","scheduler.rst","security.rst","start.rst","tutorial.rst","ui.rst"],objects:{" [...]
\ No newline at end of file
+Search.setIndex({docnames:["api","cli","code","concepts","configuration","faq","index","installation","integration","license","plugins","profiling","project","scheduler","security","start","tutorial","ui"],envversion:50,filenames:["api.rst","cli.rst","code.rst","concepts.rst","configuration.rst","faq.rst","index.rst","installation.rst","integration.rst","license.rst","plugins.rst","profiling.rst","project.rst","scheduler.rst","security.rst","start.rst","tutorial.rst","ui.rst"],objects:{" [...]
\ No newline at end of file