You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2017/03/06 17:10:08 UTC
[03/22] incubator-airflow-site git commit: Latest docs version as of
1.8.x
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/plugins.html
----------------------------------------------------------------------
diff --git a/plugins.html b/plugins.html
index 68c615d..be1cfbb 100644
--- a/plugins.html
+++ b/plugins.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
<link rel="next" title="Security" href="security.html"/>
<link rel="prev" title="Scheduling & Triggers" href="scheduler.html"/>
@@ -41,6 +44,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -97,6 +101,8 @@
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -111,8 +117,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -121,23 +129,40 @@
<div class="rst-content">
-
+
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li>Plugins</li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li>Plugins</li>
+
+
<li class="wy-breadcrumbs-aside">
-
- <a href="_sources/plugins.txt" rel="nofollow"> View page source</a>
+
+ <a href="_sources/plugins.rst.txt" rel="nofollow"> View page source</a>
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -226,18 +251,22 @@ definitions in Airflow.</p>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
-<span class="c1"># Will show up under airflow.hooks.PluginHook</span>
+<span class="c1"># Will show up under airflow.hooks.test_plugin.PluginHook</span>
<span class="k">class</span> <span class="nc">PluginHook</span><span class="p">(</span><span class="n">BaseHook</span><span class="p">):</span>
<span class="k">pass</span>
-<span class="c1"># Will show up under airflow.operators.PluginOperator</span>
+<span class="c1"># Will show up under airflow.operators.test_plugin.PluginOperator</span>
<span class="k">class</span> <span class="nc">PluginOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="k">pass</span>
-<span class="c1"># Will show up under airflow.executors.PluginExecutor</span>
+<span class="c1"># Will show up under airflow.executors.test_plugin.PluginExecutor</span>
<span class="k">class</span> <span class="nc">PluginExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
<span class="k">pass</span>
+<span class="c1"># Will show up under airflow.macros.test_plugin.plugin_macro</span>
+<span class="k">def</span> <span class="nf">plugin_macro</span><span class="p">():</span>
+ <span class="k">pass</span>
+
<span class="c1"># Creating a flask admin BaseView</span>
<span class="k">class</span> <span class="nc">TestView</span><span class="p">(</span><span class="n">BaseView</span><span class="p">):</span>
<span class="nd">@expose</span><span class="p">(</span><span class="s1">'/'</span><span class="p">)</span>
@@ -262,10 +291,11 @@ definitions in Airflow.</p>
<span class="k">class</span> <span class="nc">AirflowTestPlugin</span><span class="p">(</span><span class="n">AirflowPlugin</span><span class="p">):</span>
<span class="n">name</span> <span class="o">=</span> <span class="s2">"test_plugin"</span>
<span class="n">operators</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginOperator</span><span class="p">]</span>
- <span class="n">flask_blueprints</span> <span class="o">=</span> <span class="p">[</span><span class="n">bp</span><span class="p">]</span>
<span class="n">hooks</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginHook</span><span class="p">]</span>
<span class="n">executors</span> <span class="o">=</span> <span class="p">[</span><span class="n">PluginExecutor</span><span class="p">]</span>
+ <span class="n">macros</span> <span class="o">=</span> <span class="p">[</span><span class="n">plugin_macro</span><span class="p">]</span>
<span class="n">admin_views</span> <span class="o">=</span> <span class="p">[</span><span class="n">v</span><span class="p">]</span>
+ <span class="n">flask_blueprints</span> <span class="o">=</span> <span class="p">[</span><span class="n">bp</span><span class="p">]</span>
<span class="n">menu_links</span> <span class="o">=</span> <span class="p">[</span><span class="n">ml</span><span class="p">]</span>
</pre></div>
</div>
@@ -274,15 +304,18 @@ definitions in Airflow.</p>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="security.html" class="btn btn-neutral float-right" title="Security" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+ <a href="security.html" class="btn btn-neutral float-right" title="Security" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
- <a href="scheduler.html" class="btn btn-neutral" title="Scheduling & Triggers" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+ <a href="scheduler.html" class="btn btn-neutral" title="Scheduling & Triggers" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
@@ -315,7 +348,8 @@ definitions in Airflow.</p>
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/profiling.html
----------------------------------------------------------------------
diff --git a/profiling.html b/profiling.html
index 795ec81..350c015 100644
--- a/profiling.html
+++ b/profiling.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
<link rel="next" title="Command Line Interface" href="cli.html"/>
<link rel="prev" title="Concepts" href="concepts.html"/>
@@ -41,6 +44,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -99,6 +103,8 @@
<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -113,8 +119,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -123,23 +131,40 @@
<div class="rst-content">
-
+
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li>Data Profiling</li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li>Data Profiling</li>
+
+
<li class="wy-breadcrumbs-aside">
-
- <a href="_sources/profiling.txt" rel="nofollow"> View page source</a>
+
+ <a href="_sources/profiling.rst.txt" rel="nofollow"> View page source</a>
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -181,15 +206,18 @@ directly in the URL.</p>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="cli.html" class="btn btn-neutral float-right" title="Command Line Interface" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+ <a href="cli.html" class="btn btn-neutral float-right" title="Command Line Interface" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
- <a href="concepts.html" class="btn btn-neutral" title="Concepts" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+ <a href="concepts.html" class="btn btn-neutral" title="Concepts" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
@@ -222,7 +250,8 @@ directly in the URL.</p>
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/project.html
----------------------------------------------------------------------
diff --git a/project.html b/project.html
index d2eb7f2..1bc09d0 100644
--- a/project.html
+++ b/project.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
<link rel="next" title="License" href="license.html"/>
<link rel="prev" title="Apache Airflow (incubating) Documentation" href="index.html"/>
@@ -41,6 +44,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -97,6 +101,8 @@
<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -111,8 +117,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -121,23 +129,40 @@
<div class="rst-content">
-
+
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li>Project</li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li>Project</li>
+
+
<li class="wy-breadcrumbs-aside">
-
- <a href="_sources/project.txt" rel="nofollow"> View page source</a>
+
+ <a href="_sources/project.rst.txt" rel="nofollow"> View page source</a>
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -147,11 +172,10 @@
<h1>Project<a class="headerlink" href="#project" title="Permalink to this headline">�</a></h1>
<div class="section" id="history">
<h2>History<a class="headerlink" href="#history" title="Permalink to this headline">�</a></h2>
-<p>Airflow was started in the fall of 2014 by Maxime Beauchemin at Airbnb.
+<p>Airflow was started in October 2014 by Maxime Beauchemin at Airbnb.
It was open source from the very first commit and officially brought under
-the Airbnb Github and announced in the spring of 2015.</p>
-<p>The project joined the Apache Software Foundation’s incubation program in the
-winter of 2016.</p>
+the Airbnb Github and announced in June 2015.</p>
+<p>The project joined the Apache Software Foundation’s incubation program in March 2016.</p>
</div>
<div class="section" id="committers">
<h2>Committers<a class="headerlink" href="#committers" title="Permalink to this headline">�</a></h2>
@@ -172,24 +196,14 @@ Contributor page:</a></p>
<div class="section" id="resources-links">
<h2>Resources & links<a class="headerlink" href="#resources-links" title="Permalink to this headline">�</a></h2>
<ul class="simple">
+<li><a class="reference external" href="http://airflow.apache.org/">Airflow’s official documentation</a></li>
<li>Mailing list (send emails to
-<code class="docutils literal"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and
+<code class="docutils literal"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and/or
<code class="docutils literal"><span class="pre">commits-subscribe@airflow.incubator.apache.org</span></code>
to subscribe to each)</li>
-<li><a class="reference external" href="https://issues.apache.org/jira/browse/AIRFLOW">Issues</a></li>
-<li><a class="reference external" href="http://nerds.airbnb.com/airflow/">Airbnb Blog Post about Airflow</a></li>
-<li><a class="reference external" href="https://cwiki.apache.org/confluence/display/AIRFLOW/Common+Pitfalls">Airflow Common Pitfalls</a></li>
-<li><a class="reference external" href="https://www.youtube.com/watch?v=oYp49mBwH60">Hadoop Summit Airflow Video</a></li>
-<li><a class="reference external" href="http://agari.com/blog/airflow-agari">Airflow at Agari Blog Post</a></li>
-<li><a class="reference external" href="https://youtu.be/dgaoqOZlvEA">Talk: Best practices with Airflow (nov 2015)</a></li>
-<li><a class="reference external" href="https://www.linkedin.com/pulse/airflow-lesson-1-triggerdagrunoperator-siddharth-anand?published=t">Airflow Lesson 1: TriggerDagRunOperator</a></li>
-<li><a class="reference external" href="https://github.com/puckel/docker-airflow">Docker Airflow (externally maintained)</a></li>
-<li><a class="reference external" href="https://medium.com/handy-tech/airflow-tips-tricks-and-pitfalls-9ba53fba14eb#.o2snqeoz7">Airflow: Tips, Tricks, and Pitfalls @ Handy</a></li>
-<li><a class="reference external" href="https://github.com/bahchis/airflow-cookbook">Airflow Chef recipe (community contributed)</a> ,
-<a class="reference external" href="https://supermarket.chef.io/cookbooks/airflow">another here</a></li>
-<li>Airflow Puppet Module (community contributed) <a class="reference external" href="https://github.com/similarweb/puppet-airflow">puppet-airflow</a>,
-<a class="reference external" href="https://forge.puppetlabs.com/similarweb/airflow">airflow</a></li>
+<li><a class="reference external" href="https://issues.apache.org/jira/browse/AIRFLOW">Issues on Apache’s Jira</a></li>
<li><a class="reference external" href="https://gitter.im/airbnb/airflow">Gitter (chat) Channel</a></li>
+<li><a class="reference external" href="https://cwiki.apache.org/confluence/display/AIRFLOW/Airflow+Links">More resources and links to Airflow related content on the Wiki</a></li>
</ul>
</div>
<div class="section" id="roadmap">
@@ -200,15 +214,18 @@ to subscribe to each)</li>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="license.html" class="btn btn-neutral float-right" title="License" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+ <a href="license.html" class="btn btn-neutral float-right" title="License" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
- <a href="index.html" class="btn btn-neutral" title="Apache Airflow (incubating) Documentation" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+ <a href="index.html" class="btn btn-neutral" title="Apache Airflow (incubating) Documentation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
@@ -241,7 +258,8 @@ to subscribe to each)</li>
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/py-modindex.html
----------------------------------------------------------------------
diff --git a/py-modindex.html b/py-modindex.html
index d377e08..3f84d0f 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
@@ -42,6 +45,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -92,6 +96,8 @@
<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -106,8 +112,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -120,17 +128,34 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li></li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li></li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -143,8 +168,8 @@
<a href="#cap-a"><strong>a</strong></a>
</div>
- <table class="indextable modindextable" cellspacing="0" cellpadding="2">
- <tr class="pcap"><td></td><td> </td><td></td></tr>
+ <table class="indextable modindextable">
+ <tr class="pcap"><td></td><td> </td><td></td></tr>
<tr class="cap" id="cap-a"><td></td><td>
<strong>a</strong></td><td></td></tr>
<tr>
@@ -155,48 +180,51 @@
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.contrib.hooks"><code class="xref">airflow.contrib.hooks</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.contrib.operators"><code class="xref">airflow.contrib.operators</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.executors"><code class="xref">airflow.executors</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.hooks"><code class="xref">airflow.hooks</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.macros"><code class="xref">airflow.macros</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.macros.hive"><code class="xref">airflow.macros.hive</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.models"><code class="xref">airflow.models</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
<td></td>
- <td>
+ <td>   
<a href="code.html#module-airflow.operators"><code class="xref">airflow.operators</code></a></td><td>
<em></em></td></tr>
</table>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -229,7 +257,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/scheduler.html
----------------------------------------------------------------------
diff --git a/scheduler.html b/scheduler.html
index 7f0bb2c..3f3aa9a 100644
--- a/scheduler.html
+++ b/scheduler.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="search.html"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
<link rel="next" title="Plugins" href="plugins.html"/>
<link rel="prev" title="Command Line Interface" href="cli.html"/>
@@ -41,6 +44,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,12 +94,15 @@
<li class="toctree-l1"><a class="reference internal" href="cli.html">Command Line Interface</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Scheduling & Triggers</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#dag-runs">DAG Runs</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#backfill-and-catchup">Backfill and Catchup</a></li>
<li class="toctree-l2"><a class="reference internal" href="#external-triggers">External Triggers</a></li>
<li class="toctree-l2"><a class="reference internal" href="#to-keep-in-mind">To Keep in Mind</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -110,8 +117,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -120,23 +129,40 @@
<div class="rst-content">
-
+
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li>Scheduling & Triggers</li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li>Scheduling & Triggers</li>
+
+
<li class="wy-breadcrumbs-aside">
-
- <a href="_sources/scheduler.txt" rel="nofollow"> View page source</a>
+
+ <a href="_sources/scheduler.rst.txt" rel="nofollow"> View page source</a>
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -157,6 +183,8 @@ execute <code class="docutils literal"><span class="pre">airflow</span> <span cl
the run stamped <code class="docutils literal"><span class="pre">2016-01-01</span></code> will be trigger soon after <code class="docutils literal"><span class="pre">2016-01-01T23:59</span></code>.
In other words, the job instance is started once the period it covers
has ended.</p>
+<p><strong>Let’s Repeat That</strong> The scheduler runs your job one <code class="docutils literal"><span class="pre">schedule_interval</span></code> AFTER the
+start date, at the END of the period.</p>
<p>The scheduler starts an instance of the executor specified in the your
<code class="docutils literal"><span class="pre">airflow.cfg</span></code>. If it happens to be the <code class="docutils literal"><span class="pre">LocalExecutor</span></code>, tasks will be
executed as subprocesses; in the case of <code class="docutils literal"><span class="pre">CeleryExecutor</span></code> and
@@ -190,11 +218,11 @@ use one of these cron “preset”:</p>
<tr class="row-even"><td><code class="docutils literal"><span class="pre">None</span></code></td>
<td>Don’t schedule, use for exclusively “externally triggered”
DAGs</td>
-<td> </td>
+<td> </td>
</tr>
<tr class="row-odd"><td><code class="docutils literal"><span class="pre">@once</span></code></td>
<td>Schedule once and only once</td>
-<td> </td>
+<td> </td>
</tr>
<tr class="row-even"><td><code class="docutils literal"><span class="pre">@hourly</span></code></td>
<td>Run once an hour at the beginning of the hour</td>
@@ -228,6 +256,27 @@ should be triggered and come to a crawl. It might also create undesired
processing when changing the shape of your DAG, by say adding in new
tasks.</p>
</div>
+<div class="section" id="backfill-and-catchup">
+<h2>Backfill and Catchup<a class="headerlink" href="#backfill-and-catchup" title="Permalink to this headline">�</a></h2>
+<p>An Airflow DAG with a <code class="docutils literal"><span class="pre">start_date</span></code>, possibly an <code class="docutils literal"><span class="pre">end_date</span></code>, and a <code class="docutils literal"><span class="pre">schedule_interval</span></code> defines a
+series of intervals which the scheduler turn into individual Dag Runs and execute. A key capability of
+Airflow is that these DAG Runs are atomic, idempotent items, and the scheduler, by default, will examine
+the lifetime of the DAG (from start to end/now, one interval at a time) and kick off a DAG Run for any
+interval that has not been run (or has been cleared). This concept is called Catchup.</p>
+<p>If your DAG is written to handle it’s own catchup (IE not limited to the interval, but instead to “Now”
+for instance.), then you will want to turn catchup off (Either on the DAG itself with <code class="docutils literal"><span class="pre">dag.catchup</span> <span class="pre">=</span>
+<span class="pre">False</span></code>) or by default at the configuration file level with <code class="docutils literal"><span class="pre">catchup_by_default</span> <span class="pre">=</span> <span class="pre">False</span></code>. What this
+will do, is to instruct the scheduler to only create a DAG Run for the most current instance of the DAG
+interval series.</p>
+<p>In the example above, if the DAG is picked up by the scheduler daemon on 2016-01-02 at 6 AM, (or from the
+command line), a single DAG Run will be created, with an <code class="docutils literal"><span class="pre">execution_date</span></code> of 2016-01-01, and the next
+one will be created just after midnight on the morning of 2016-01-03 with an execution date of 2016-01-02.</p>
+<p>If the <code class="docutils literal"><span class="pre">dag.catchup</span></code> value had been True instead, the scheduler would have created a DAG Run for each
+completed interval between 2015-12-01 and 2016-01-02 (but not yet one for 2016-01-02, as that interval
+hasn’t completed) and the scheduler will execute them sequentially. This behavior is great for atomic
+datasets that can easily be split into periods. Turning catchup off is great if your DAG Runs perform
+backfill internally.</p>
+</div>
<div class="section" id="external-triggers">
<h2>External Triggers<a class="headerlink" href="#external-triggers" title="Permalink to this headline">�</a></h2>
<p>Note that <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> can also be created manually through the CLI while
@@ -259,15 +308,18 @@ whether the scheduler should look into triggering tasks for that run.</li>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
- <a href="plugins.html" class="btn btn-neutral float-right" title="Plugins" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+ <a href="plugins.html" class="btn btn-neutral float-right" title="Plugins" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
- <a href="cli.html" class="btn btn-neutral" title="Command Line Interface" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+ <a href="cli.html" class="btn btn-neutral" title="Command Line Interface" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
</div>
@@ -300,7 +352,8 @@ whether the scheduler should look into triggering tasks for that run.</li>
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/search.html
----------------------------------------------------------------------
diff --git a/search.html b/search.html
index 9935561..eed50cd 100644
--- a/search.html
+++ b/search.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="genindex.html"/>
+ <link rel="search" title="Search" href="#"/>
<link rel="top" title="Airflow Documentation" href="index.html"/>
@@ -39,6 +42,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -89,6 +93,8 @@
<li class="toctree-l1"><a class="reference internal" href="scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="code.html">API Reference</a></li>
</ul>
@@ -103,8 +109,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="index.html">Airflow</a>
+
</nav>
@@ -117,15 +125,32 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="index.html">Docs</a> »</li>
-
- <li></li>
+
+ <li><a href="index.html">Docs</a> »</li>
+
+ <li></li>
+
+
<li class="wy-breadcrumbs-aside">
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -146,6 +171,9 @@
</div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -178,7 +206,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="_static/jquery.js"></script>