You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/08/29 20:24:31 UTC
[03/48] incubator-airflow-site git commit: 1.10.0 with Updated Api
Reference
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/kubernetes.html
----------------------------------------------------------------------
diff --git a/kubernetes.html b/kubernetes.html
index 62110ac..9d0ca02 100644
--- a/kubernetes.html
+++ b/kubernetes.html
@@ -24,25 +24,17 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -115,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -123,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -176,7 +169,7 @@
</div>
<div class="section" id="kubernetes-operator">
<h1>Kubernetes Operator<a class="headerlink" href="#kubernetes-operator" title="Permalink to this headline">¶</a></h1>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow.contrib.operators</span> <span class="k">import</span> <span class="n">KubernetesOperator</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow.contrib.operators</span> <span class="k">import</span> <span class="n">KubernetesOperator</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.operators.kubernetes_pod_operator</span> <span class="k">import</span> <span class="n">KubernetesPodOperator</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.kubernetes.secret</span> <span class="k">import</span> <span class="n">Secret</span>
@@ -260,8 +253,8 @@
</div>
<dl class="class">
<dt id="airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator">
-<em class="property">class </em><code class="descclassname">airflow.contrib.operators.kubernetes_pod_operator.</code><code class="descname">KubernetesPodOperator</code><span class="sig-paren">(</span><em>namespace</em>, <em>image</em>, <em>name</em>, <em>cmds=None</em>, <em>arguments=None</em>, <em>env_vars=None</em>, <em>secrets=None</em>, <em>in_cluster=False</em>, <em>labels=None</em>, <em>startup_timeout_seconds=120</em>, <em>get_logs=True</em>, <em>image_pull_policy='IfNotPresent'</em>, <em>annotations=None</em>, <em>resources=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/kubernetes_pod_operator.html#KubernetesPodOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.kubernetes_pod_operator.KubernetesPodOperator" title="Permalink to this definition">¶</a></dt>
-<dd><p>Bases: <a class="reference internal" href="code.html#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.kubernetes_pod_operator.</code><code class="descname">KubernetesPodOperator</code><span class="sig-paren">(</span><em>namespace</em>, <em>image</em>, <em>name</em>, <em>cmds=None</em>, <em>arguments=None</em>, <em>volume_mounts=None</em>, <em>volumes=None</em>, <em>env_vars=None</em>, <em>secrets=None</em>, <em>in_cluster=False</em>, <em>cluster_context=None</em>, <em>labels=None</em>, <em>startup_timeout_seconds=120</em>, <em>get_logs=True</em>, <em>image_pull_policy='IfNotPresent'</em>, <em>annotations=None</em>, <em>resources=None</em>, <em>affinity=None</em>, <em>config_file=None</em>, <em>xcom_push=False</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/kubernetes_pod_operator.html#KubernetesPodOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operator
s.kubernetes_pod_operator.KubernetesPodOperator" title="Permalink to this definition">¶</a></dt>
+<dd><p>Bases: <a class="reference internal" href="code.html#airflow.models.BaseOperator" title="airflow.models.BaseOperator"><code class="xref py py-class docutils literal notranslate"><span class="pre">airflow.models.BaseOperator</span></code></a></p>
<p>Execute a task in a Kubernetes Pod</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
@@ -270,19 +263,28 @@
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
<li><strong>image</strong> (<em>str</em>) – Docker image you wish to launch. Defaults to dockerhub.io,
but fully qualified URLS will point to custom repositories</li>
-<li><strong>cmds</strong> (<a class="reference internal" href="integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list" title="airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list"><em>list</em></a>) – entrypoint of the container.
+<li><strong>cmds</strong> (<em>list of str</em>) – entrypoint of the container. (templated)
The docker images’s entrypoint is used if this is not provide.</li>
-<li><strong>arguments</strong> (<a class="reference internal" href="integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list" title="airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list"><em>list</em></a>) – arguments of to the entrypoint.
+<li><strong>arguments</strong> (<em>list of str</em>) – arguments of to the entrypoint. (templated)
The docker image’s CMD is used if this is not provided.</li>
+<li><strong>volume_mounts</strong> (<em>list of VolumeMount</em>) – volumeMounts for launched pod</li>
+<li><strong>volumes</strong> (<em>list of Volume</em>) – volumes for launched pod. Includes ConfigMaps and PersistentVolumes</li>
<li><strong>labels</strong> (<em>dict</em>) – labels to apply to the Pod</li>
<li><strong>startup_timeout_seconds</strong> (<em>int</em>) – timeout in seconds to startup the pod</li>
<li><strong>name</strong> (<em>str</em>) – name of the task you want to run,
will be used to generate a pod id</li>
-<li><strong>env_vars</strong> (<em>dict</em>) – Environment variables initialized in the container</li>
-<li><strong>secrets</strong> (<a class="reference internal" href="integration.html#airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list" title="airflow.contrib.hooks.gcs_hook.GoogleCloudStorageHook.list"><em>list</em></a>) – Kubernetes secrets to inject in the container,
+<li><strong>env_vars</strong> (<em>dict</em>) – Environment variables initialized in the container. (templated)</li>
+<li><strong>secrets</strong> (<em>list of Secret</em>) – Kubernetes secrets to inject in the container,
They can be exposed as environment vars or files in a volume.</li>
<li><strong>in_cluster</strong> (<em>bool</em>) – run kubernetes client with in_cluster configuration</li>
+<li><strong>cluster_context</strong> (<em>string</em>) – context that points to kubernetes cluster.
+Ignored when in_cluster is True. If None, current-context is used.</li>
<li><strong>get_logs</strong> (<em>bool</em>) – get the stdout of the container as logs of the tasks</li>
+<li><strong>affinity</strong> (<em>dict</em>) – A dict containing a group of affinity scheduling rules</li>
+<li><strong>config_file</strong> (<em>str</em>) – The path to the Kubernetes config file</li>
+<li><strong>xcom_push</strong> (<em>bool</em>) – If xcom_push is True, the content of the file
+/airflow/xcom/return.json in the container will also be pushed to an
+XCom when the container completes.</li>
</ul>
</td>
</tr>
@@ -306,9 +308,7 @@ They can be exposed as environment vars or files in a volume.</li>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -320,7 +320,7 @@ They can be exposed as environment vars or files in a volume.</li>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -339,6 +339,7 @@ They can be exposed as environment vars or files in a volume.</li>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -351,19 +352,13 @@ They can be exposed as environment vars or files in a volume.</li>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/license.html
----------------------------------------------------------------------
diff --git a/license.html b/license.html
index 26654aa..65b90e7 100644
--- a/license.html
+++ b/license.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="Quick Start" href="start.html"/>
- <link rel="prev" title="Project" href="project.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="Quick Start" href="start.html" />
+ <link rel="prev" title="Project" href="project.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -117,7 +109,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -125,9 +117,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -172,7 +165,7 @@
<div class="section" id="license">
<h1>License<a class="headerlink" href="#license" title="Permalink to this headline">¶</a></h1>
<a class="reference internal image-reference" href="_images/apache.jpg"><img alt="_images/apache.jpg" src="_images/apache.jpg" style="width: 150px;" /></a>
-<div class="highlight-default"><div class="highlight"><pre><span></span> <span class="n">Apache</span> <span class="n">License</span>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span> <span class="n">Apache</span> <span class="n">License</span>
<span class="n">Version</span> <span class="mf">2.0</span><span class="p">,</span> <span class="n">January</span> <span class="mi">2004</span>
<span class="n">http</span><span class="p">:</span><span class="o">//</span><span class="n">www</span><span class="o">.</span><span class="n">apache</span><span class="o">.</span><span class="n">org</span><span class="o">/</span><span class="n">licenses</span><span class="o">/</span>
@@ -352,9 +345,7 @@
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -375,7 +366,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -394,6 +385,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -406,19 +398,13 @@
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/lineage.html
----------------------------------------------------------------------
diff --git a/lineage.html b/lineage.html
index afa2726..e58ed3d 100644
--- a/lineage.html
+++ b/lineage.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="FAQ" href="faq.html"/>
- <link rel="prev" title="Integration" href="integration.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="FAQ" href="faq.html" />
+ <link rel="prev" title="Integration" href="integration.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -120,7 +112,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -128,9 +120,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -182,7 +175,7 @@
audit trails and data governance, but also debugging of data flows.</p>
<p>Airflow tracks data by means of inlets and outlets of the tasks. Let’s work from an example and see how it
works.</p>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow.operators.bash_operator</span> <span class="k">import</span> <span class="n">BashOperator</span>
<span class="kn">from</span> <span class="nn">airflow.operators.dummy_operator</span> <span class="k">import</span> <span class="n">DummyOperator</span>
<span class="kn">from</span> <span class="nn">airflow.lineage.datasets</span> <span class="k">import</span> <span class="n">File</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
@@ -240,7 +233,7 @@ respectively.</p>
<h2>Apache Atlas<a class="headerlink" href="#apache-atlas" title="Permalink to this headline">¶</a></h2>
<p>Airflow can send its lineage metadata to Apache Atlas. You need to enable the <cite>atlas</cite> backend and configure it
properly, e.g. in your <cite>airflow.cfg</cite>:</p>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">lineage</span><span class="p">]</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">lineage</span><span class="p">]</span>
<span class="n">backend</span> <span class="o">=</span> <span class="n">airflow</span><span class="o">.</span><span class="n">lineage</span><span class="o">.</span><span class="n">backend</span><span class="o">.</span><span class="n">atlas</span>
<span class="p">[</span><span class="n">atlas</span><span class="p">]</span>
@@ -256,9 +249,7 @@ properly, e.g. in your <cite>airflow.cfg</cite>:</p>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -279,7 +270,7 @@ properly, e.g. in your <cite>airflow.cfg</cite>:</p>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -298,6 +289,7 @@ properly, e.g. in your <cite>airflow.cfg</cite>:</p>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -310,19 +302,13 @@ properly, e.g. in your <cite>airflow.cfg</cite>:</p>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/objects.inv
----------------------------------------------------------------------
diff --git a/objects.inv b/objects.inv
index 7038443..1a4c2a2 100644
Binary files a/objects.inv and b/objects.inv differ
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/plugins.html
----------------------------------------------------------------------
diff --git a/plugins.html b/plugins.html
index 60d3835..bad7f71 100644
--- a/plugins.html
+++ b/plugins.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="Security" href="security.html"/>
- <link rel="prev" title="Scheduling & Triggers" href="scheduler.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="Security" href="security.html" />
+ <link rel="prev" title="Scheduling & Triggers" href="scheduler.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -123,7 +115,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -131,9 +123,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -179,8 +172,8 @@
<h1>Plugins<a class="headerlink" href="#plugins" title="Permalink to this headline">¶</a></h1>
<p>Airflow has a simple plugin manager built-in that can integrate external
features to its core by simply dropping files in your
-<code class="docutils literal"><span class="pre">$AIRFLOW_HOME/plugins</span></code> folder.</p>
-<p>The python modules in the <code class="docutils literal"><span class="pre">plugins</span></code> folder get imported,
+<code class="docutils literal notranslate"><span class="pre">$AIRFLOW_HOME/plugins</span></code> folder.</p>
+<p>The python modules in the <code class="docutils literal notranslate"><span class="pre">plugins</span></code> folder get imported,
and <strong>hooks</strong>, <strong>operators</strong>, <strong>sensors</strong>, <strong>macros</strong>, <strong>executors</strong> and web <strong>views</strong>
get integrated to Airflow’s main collections and become available for use.</p>
<div class="section" id="what-for">
@@ -218,10 +211,10 @@ they should land, alert people, and expose visualizations of outages</li>
<div class="section" id="interface">
<h2>Interface<a class="headerlink" href="#interface" title="Permalink to this headline">¶</a></h2>
<p>To create a plugin you will need to derive the
-<code class="docutils literal"><span class="pre">airflow.plugins_manager.AirflowPlugin</span></code> class and reference the objects
+<code class="docutils literal notranslate"><span class="pre">airflow.plugins_manager.AirflowPlugin</span></code> class and reference the objects
you want to plug into Airflow. Here’s what the class you need to derive
looks like:</p>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">AirflowPlugin</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">class</span> <span class="nc">AirflowPlugin</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="c1"># The name of your plugin (str)</span>
<span class="n">name</span> <span class="o">=</span> <span class="kc">None</span>
<span class="c1"># A list of class(es) derived from BaseOperator</span>
@@ -243,12 +236,26 @@ looks like:</p>
<span class="n">menu_links</span> <span class="o">=</span> <span class="p">[]</span>
</pre></div>
</div>
+<p>You can derive it by inheritance (please refer to the example below).
+Please note <code class="docutils literal notranslate"><span class="pre">name</span></code> inside this class must be specified.</p>
+<p>After the plugin is imported into Airflow,
+you can invoke it using statement like</p>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">airflow.</span><span class="p">{</span><span class="nb">type</span><span class="p">,</span> <span class="n">like</span> <span class="s2">"operators"</span><span class="p">,</span> <span class="s2">"sensors"</span><span class="p">}</span><span class="o">.</span><span class="p">{</span><span class="n">name</span> <span class="n">specificed</span> <span class="n">inside</span> <span class="n">the</span> <span class="n">plugin</span> <span class="n">class</span><span class="p">}</span> <span class="kn">import</span> <span class="o">*</span>
+</pre></div>
+</div>
+<p>When you write your own plugins, make sure you understand them well.
+There are some essential properties for each type of plugin.
+For example,</p>
+<ul class="simple">
+<li>For <code class="docutils literal notranslate"><span class="pre">Operator</span></code> plugin, an <code class="docutils literal notranslate"><span class="pre">execute</span></code> method is compulsory.</li>
+<li>For <code class="docutils literal notranslate"><span class="pre">Sensor</span></code> plugin, a <code class="docutils literal notranslate"><span class="pre">poke</span></code> method returning a Boolean value is compulsory.</li>
+</ul>
</div>
<div class="section" id="example">
<h2>Example<a class="headerlink" href="#example" title="Permalink to this headline">¶</a></h2>
<p>The code below defines a plugin that injects a set of dummy object
definitions in Airflow.</p>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1"># This is the class you derive to create a plugin</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># This is the class you derive to create a plugin</span>
<span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="k">import</span> <span class="n">AirflowPlugin</span>
<span class="kn">from</span> <span class="nn">flask</span> <span class="k">import</span> <span class="n">Blueprint</span>
@@ -319,9 +326,7 @@ definitions in Airflow.</p>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -342,7 +347,7 @@ definitions in Airflow.</p>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -361,6 +366,7 @@ definitions in Airflow.</p>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -373,19 +379,13 @@ definitions in Airflow.</p>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/profiling.html
----------------------------------------------------------------------
diff --git a/profiling.html b/profiling.html
index a94060c..0c63b23 100644
--- a/profiling.html
+++ b/profiling.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="Command Line Interface" href="cli.html"/>
- <link rel="prev" title="Concepts" href="concepts.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="Command Line Interface" href="cli.html" />
+ <link rel="prev" title="Concepts" href="concepts.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -125,7 +117,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -133,9 +125,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -213,9 +206,7 @@ directly in the URL.</p>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -236,7 +227,7 @@ directly in the URL.</p>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -255,6 +246,7 @@ directly in the URL.</p>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -267,19 +259,13 @@ directly in the URL.</p>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/project.html
----------------------------------------------------------------------
diff --git a/project.html b/project.html
index 591638a..c0da142 100644
--- a/project.html
+++ b/project.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="License" href="license.html"/>
- <link rel="prev" title="Apache Airflow (incubating) Documentation" href="index.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="License" href="license.html" />
+ <link rel="prev" title="Apache Airflow (incubating) Documentation" href="index.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -123,7 +115,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -131,9 +123,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -203,6 +196,7 @@ the Airbnb Github and announced in June 2015.</p>
<li>@fokko (Fokko Driesprong)</li>
<li>@ash (Ash Berlin-Taylor)</li>
<li>@kaxilnaik (Kaxil Naik)</li>
+<li>@feng-tao (Tao Feng)</li>
</ul>
<p>For the full list of contributors, take a look at <a class="reference external" href="https://github.com/apache/incubator-airflow/graphs/contributors">Airflow’s Github
Contributor page:</a></p>
@@ -212,8 +206,8 @@ Contributor page:</a></p>
<ul class="simple">
<li><a class="reference external" href="http://airflow.apache.org/">Airflow’s official documentation</a></li>
<li>Mailing list (send emails to
-<code class="docutils literal"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and/or
-<code class="docutils literal"><span class="pre">commits-subscribe@airflow.incubator.apache.org</span></code>
+<code class="docutils literal notranslate"><span class="pre">dev-subscribe@airflow.incubator.apache.org</span></code> and/or
+<code class="docutils literal notranslate"><span class="pre">commits-subscribe@airflow.incubator.apache.org</span></code>
to subscribe to each)</li>
<li><a class="reference external" href="https://issues.apache.org/jira/browse/AIRFLOW">Issues on Apache’s Jira</a></li>
<li><a class="reference external" href="https://gitter.im/airbnb/airflow">Gitter (chat) Channel</a></li>
@@ -228,9 +222,7 @@ to subscribe to each)</li>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -251,7 +243,7 @@ to subscribe to each)</li>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -270,6 +262,7 @@ to subscribe to each)</li>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -282,19 +275,13 @@ to subscribe to each)</li>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/py-modindex.html
----------------------------------------------------------------------
diff --git a/py-modindex.html b/py-modindex.html
index a054a54..60a9c43 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -24,18 +24,10 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
@@ -45,7 +37,7 @@
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -118,7 +110,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -126,9 +118,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +149,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -188,6 +179,11 @@
<tr class="cg-1">
<td></td>
<td>   
+ <a href="code.html#module-airflow.hooks.hive_hooks"><code class="xref">airflow.hooks.hive_hooks</code></a></td><td>
+ <em></em></td></tr>
+ <tr class="cg-1">
+ <td></td>
+ <td>   
<a href="code.html#module-airflow.macros"><code class="xref">airflow.macros</code></a></td><td>
<em></em></td></tr>
<tr class="cg-1">
@@ -199,9 +195,7 @@
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -213,7 +207,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -232,6 +226,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -244,19 +239,13 @@
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/scheduler.html
----------------------------------------------------------------------
diff --git a/scheduler.html b/scheduler.html
index a8df02e..3fb2968 100644
--- a/scheduler.html
+++ b/scheduler.html
@@ -24,27 +24,19 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="search.html"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
- <link rel="next" title="Plugins" href="plugins.html"/>
- <link rel="prev" title="Command Line Interface" href="cli.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="search.html" />
+ <link rel="next" title="Plugins" href="plugins.html" />
+ <link rel="prev" title="Command Line Interface" href="cli.html" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -123,7 +115,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -131,9 +123,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -184,30 +177,30 @@ and periodically (every minute or so) inspects active tasks to see whether
they can be triggered.</p>
<p>The Airflow scheduler is designed to run as a persistent service in an
Airflow production environment. To kick it off, all you need to do is
-execute <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">scheduler</span></code>. It will use the configuration specified in
-<code class="docutils literal"><span class="pre">airflow.cfg</span></code>.</p>
-<p>Note that if you run a DAG on a <code class="docutils literal"><span class="pre">schedule_interval</span></code> of one day,
-the run stamped <code class="docutils literal"><span class="pre">2016-01-01</span></code> will be trigger soon after <code class="docutils literal"><span class="pre">2016-01-01T23:59</span></code>.
+execute <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">scheduler</span></code>. It will use the configuration specified in
+<code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code>.</p>
+<p>Note that if you run a DAG on a <code class="docutils literal notranslate"><span class="pre">schedule_interval</span></code> of one day,
+the run stamped <code class="docutils literal notranslate"><span class="pre">2016-01-01</span></code> will be triggered soon after <code class="docutils literal notranslate"><span class="pre">2016-01-01T23:59</span></code>.
In other words, the job instance is started once the period it covers
has ended.</p>
-<p><strong>Let’s Repeat That</strong> The scheduler runs your job one <code class="docutils literal"><span class="pre">schedule_interval</span></code> AFTER the
+<p><strong>Let’s Repeat That</strong> The scheduler runs your job one <code class="docutils literal notranslate"><span class="pre">schedule_interval</span></code> AFTER the
start date, at the END of the period.</p>
<p>The scheduler starts an instance of the executor specified in the your
-<code class="docutils literal"><span class="pre">airflow.cfg</span></code>. If it happens to be the <code class="docutils literal"><span class="pre">LocalExecutor</span></code>, tasks will be
-executed as subprocesses; in the case of <code class="docutils literal"><span class="pre">CeleryExecutor</span></code> and
-<code class="docutils literal"><span class="pre">MesosExecutor</span></code>, tasks are executed remotely.</p>
+<code class="docutils literal notranslate"><span class="pre">airflow.cfg</span></code>. If it happens to be the <code class="docutils literal notranslate"><span class="pre">LocalExecutor</span></code>, tasks will be
+executed as subprocesses; in the case of <code class="docutils literal notranslate"><span class="pre">CeleryExecutor</span></code> and
+<code class="docutils literal notranslate"><span class="pre">MesosExecutor</span></code>, tasks are executed remotely.</p>
<p>To start a scheduler, simply run the command:</p>
-<div class="code bash highlight-default"><div class="highlight"><pre><span></span><span class="n">airflow</span> <span class="n">scheduler</span>
+<div class="code bash highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">airflow</span> <span class="n">scheduler</span>
</pre></div>
</div>
<div class="section" id="dag-runs">
<h2>DAG Runs<a class="headerlink" href="#dag-runs" title="Permalink to this headline">¶</a></h2>
<p>A DAG Run is an object representing an instantiation of the DAG in time.</p>
-<p>Each DAG may or may not have a schedule, which informs how <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> are
-created. <code class="docutils literal"><span class="pre">schedule_interval</span></code> is defined as a DAG arguments, and receives
+<p>Each DAG may or may not have a schedule, which informs how <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Runs</span></code> are
+created. <code class="docutils literal notranslate"><span class="pre">schedule_interval</span></code> is defined as a DAG arguments, and receives
preferably a
<a class="reference external" href="https://en.wikipedia.org/wiki/Cron#CRON_expression">cron expression</a> as
-a <code class="docutils literal"><span class="pre">str</span></code>, or a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code> object. Alternatively, you can also
+a <code class="docutils literal notranslate"><span class="pre">str</span></code>, or a <code class="docutils literal notranslate"><span class="pre">datetime.timedelta</span></code> object. Alternatively, you can also
use one of these cron “preset”:</p>
<table border="1" class="docutils">
<colgroup>
@@ -222,39 +215,39 @@ use one of these cron “preset”:</p>
</tr>
</thead>
<tbody valign="top">
-<tr class="row-even"><td><code class="docutils literal"><span class="pre">None</span></code></td>
+<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">None</span></code></td>
<td>Don’t schedule, use for exclusively “externally triggered”
DAGs</td>
<td> </td>
</tr>
-<tr class="row-odd"><td><code class="docutils literal"><span class="pre">@once</span></code></td>
+<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">@once</span></code></td>
<td>Schedule once and only once</td>
<td> </td>
</tr>
-<tr class="row-even"><td><code class="docutils literal"><span class="pre">@hourly</span></code></td>
+<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">@hourly</span></code></td>
<td>Run once an hour at the beginning of the hour</td>
-<td><code class="docutils literal"><span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
+<td><code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
</tr>
-<tr class="row-odd"><td><code class="docutils literal"><span class="pre">@daily</span></code></td>
+<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">@daily</span></code></td>
<td>Run once a day at midnight</td>
-<td><code class="docutils literal"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
+<td><code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
</tr>
-<tr class="row-even"><td><code class="docutils literal"><span class="pre">@weekly</span></code></td>
+<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">@weekly</span></code></td>
<td>Run once a week at midnight on Sunday morning</td>
-<td><code class="docutils literal"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">0</span></code></td>
+<td><code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">*</span> <span class="pre">*</span> <span class="pre">0</span></code></td>
</tr>
-<tr class="row-odd"><td><code class="docutils literal"><span class="pre">@monthly</span></code></td>
+<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">@monthly</span></code></td>
<td>Run once a month at midnight of the first day of the month</td>
-<td><code class="docutils literal"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">1</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
+<td><code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">1</span> <span class="pre">*</span> <span class="pre">*</span></code></td>
</tr>
-<tr class="row-even"><td><code class="docutils literal"><span class="pre">@yearly</span></code></td>
+<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">@yearly</span></code></td>
<td>Run once a year at midnight of January 1</td>
-<td><code class="docutils literal"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">1</span> <span class="pre">1</span> <span class="pre">*</span></code></td>
+<td><code class="docutils literal notranslate"><span class="pre">0</span> <span class="pre">0</span> <span class="pre">1</span> <span class="pre">1</span> <span class="pre">*</span></code></td>
</tr>
</tbody>
</table>
<p>Your DAG will be instantiated
-for each schedule, while creating a <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Run</span></code> entry for each schedule.</p>
+for each schedule, while creating a <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Run</span></code> entry for each schedule.</p>
<p>DAG runs have a state associated to them (running, failed, success) and
informs the scheduler on which set of schedules should be evaluated for
task submissions. Without the metadata at the DAG run level, the Airflow
@@ -265,17 +258,17 @@ tasks.</p>
</div>
<div class="section" id="backfill-and-catchup">
<h2>Backfill and Catchup<a class="headerlink" href="#backfill-and-catchup" title="Permalink to this headline">¶</a></h2>
-<p>An Airflow DAG with a <code class="docutils literal"><span class="pre">start_date</span></code>, possibly an <code class="docutils literal"><span class="pre">end_date</span></code>, and a <code class="docutils literal"><span class="pre">schedule_interval</span></code> defines a
+<p>An Airflow DAG with a <code class="docutils literal notranslate"><span class="pre">start_date</span></code>, possibly an <code class="docutils literal notranslate"><span class="pre">end_date</span></code>, and a <code class="docutils literal notranslate"><span class="pre">schedule_interval</span></code> defines a
series of intervals which the scheduler turn into individual Dag Runs and execute. A key capability of
Airflow is that these DAG Runs are atomic, idempotent items, and the scheduler, by default, will examine
the lifetime of the DAG (from start to end/now, one interval at a time) and kick off a DAG Run for any
interval that has not been run (or has been cleared). This concept is called Catchup.</p>
<p>If your DAG is written to handle its own catchup (IE not limited to the interval, but instead to “Now”
-for instance.), then you will want to turn catchup off (Either on the DAG itself with <code class="docutils literal"><span class="pre">dag.catchup</span> <span class="pre">=</span>
-<span class="pre">False</span></code>) or by default at the configuration file level with <code class="docutils literal"><span class="pre">catchup_by_default</span> <span class="pre">=</span> <span class="pre">False</span></code>. What this
+for instance.), then you will want to turn catchup off (Either on the DAG itself with <code class="docutils literal notranslate"><span class="pre">dag.catchup</span> <span class="pre">=</span>
+<span class="pre">False</span></code>) or by default at the configuration file level with <code class="docutils literal notranslate"><span class="pre">catchup_by_default</span> <span class="pre">=</span> <span class="pre">False</span></code>. What this
will do, is to instruct the scheduler to only create a DAG Run for the most current instance of the DAG
interval series.</p>
-<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="sd">"""</span>
+<div class="code python highlight-default notranslate"><div class="highlight"><pre><span></span><span class="sd">"""</span>
<span class="sd">Code that goes along with the Airflow tutorial located at:</span>
<span class="sd">https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py</span>
<span class="sd">"""</span>
@@ -300,9 +293,9 @@ interval series.</p>
</pre></div>
</div>
<p>In the example above, if the DAG is picked up by the scheduler daemon on 2016-01-02 at 6 AM, (or from the
-command line), a single DAG Run will be created, with an <code class="docutils literal"><span class="pre">execution_date</span></code> of 2016-01-01, and the next
+command line), a single DAG Run will be created, with an <code class="docutils literal notranslate"><span class="pre">execution_date</span></code> of 2016-01-01, and the next
one will be created just after midnight on the morning of 2016-01-03 with an execution date of 2016-01-02.</p>
-<p>If the <code class="docutils literal"><span class="pre">dag.catchup</span></code> value had been True instead, the scheduler would have created a DAG Run for each
+<p>If the <code class="docutils literal notranslate"><span class="pre">dag.catchup</span></code> value had been True instead, the scheduler would have created a DAG Run for each
completed interval between 2015-12-01 and 2016-01-02 (but not yet one for 2016-01-02, as that interval
hasn’t completed) and the scheduler will execute them sequentially. This behavior is great for atomic
datasets that can easily be split into periods. Turning catchup off is great if your DAG Runs perform
@@ -310,21 +303,22 @@ backfill internally.</p>
</div>
<div class="section" id="external-triggers">
<h2>External Triggers<a class="headerlink" href="#external-triggers" title="Permalink to this headline">¶</a></h2>
-<p>Note that <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> can also be created manually through the CLI while
-running an <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">trigger_dag</span></code> command, where you can define a
-specific <code class="docutils literal"><span class="pre">run_id</span></code>. The <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> created externally to the
+<p>Note that <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Runs</span></code> can also be created manually through the CLI while
+running an <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">trigger_dag</span></code> command, where you can define a
+specific <code class="docutils literal notranslate"><span class="pre">run_id</span></code>. The <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Runs</span></code> created externally to the
scheduler get associated to the trigger’s timestamp, and will be displayed
-in the UI alongside scheduled <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">runs</span></code>.</p>
+in the UI alongside scheduled <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">runs</span></code>.</p>
+<p>In addition, you can also manually trigger a <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Run</span></code> using the web UI (tab “DAGs” -> column “Links” -> button “Trigger Dag”).</p>
</div>
<div class="section" id="to-keep-in-mind">
<h2>To Keep in Mind<a class="headerlink" href="#to-keep-in-mind" title="Permalink to this headline">¶</a></h2>
<ul class="simple">
-<li>The first <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Run</span></code> is created based on the minimum <code class="docutils literal"><span class="pre">start_date</span></code> for the
+<li>The first <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Run</span></code> is created based on the minimum <code class="docutils literal notranslate"><span class="pre">start_date</span></code> for the
tasks in your DAG.</li>
-<li>Subsequent <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Runs</span></code> are created by the scheduler process, based on
-your DAG’s <code class="docutils literal"><span class="pre">schedule_interval</span></code>, sequentially.</li>
+<li>Subsequent <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Runs</span></code> are created by the scheduler process, based on
+your DAG’s <code class="docutils literal notranslate"><span class="pre">schedule_interval</span></code>, sequentially.</li>
<li>When clearing a set of tasks’ state in hope of getting them to re-run,
-it is important to keep in mind the <code class="docutils literal"><span class="pre">DAG</span> <span class="pre">Run</span></code>’s state too as it defines
+it is important to keep in mind the <code class="docutils literal notranslate"><span class="pre">DAG</span> <span class="pre">Run</span></code>’s state too as it defines
whether the scheduler should look into triggering tasks for that run.</li>
</ul>
<p>Here are some of the ways you can <strong>unblock tasks</strong>:</p>
@@ -334,15 +328,16 @@ from the task instances dialog, while defining whether you want to includes the
and the upstream/downstream dependencies. Note that a confirmation window comes next and
allows you to see the set you are about to clear. You can also clear all task instances
associated with the dag.</li>
-<li>The CLI command <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">clear</span> <span class="pre">-h</span></code> has lots of options when it comes to clearing task instance
+<li>The CLI command <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">clear</span> <span class="pre">-h</span></code> has lots of options when it comes to clearing task instance
states, including specifying date ranges, targeting task_ids by specifying a regular expression,
flags for including upstream and downstream relatives, and targeting task instances in specific
-states (<code class="docutils literal"><span class="pre">failed</span></code>, or <code class="docutils literal"><span class="pre">success</span></code>)</li>
+states (<code class="docutils literal notranslate"><span class="pre">failed</span></code>, or <code class="docutils literal notranslate"><span class="pre">success</span></code>)</li>
<li>Clearing a task instance will no longer delete the task instance record. Instead it updates
max_tries and set the current task instance state to be None.</li>
+<li>Marking task instances as failed can be done through the UI. This can be used to stop running task instances.</li>
<li>Marking task instances as successful can be done through the UI. This is mostly to fix false negatives,
or for instance when the fix has been applied outside of Airflow.</li>
-<li>The <code class="docutils literal"><span class="pre">airflow</span> <span class="pre">backfill</span></code> CLI subcommand has a flag to <code class="docutils literal"><span class="pre">--mark_success</span></code> and allows selecting
+<li>The <code class="docutils literal notranslate"><span class="pre">airflow</span> <span class="pre">backfill</span></code> CLI subcommand has a flag to <code class="docutils literal notranslate"><span class="pre">--mark_success</span></code> and allows selecting
subsections of the DAG as well as specifying date ranges.</li>
</ul>
</div>
@@ -350,9 +345,7 @@ subsections of the DAG as well as specifying date ranges.</li>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -373,7 +366,7 @@ subsections of the DAG as well as specifying date ranges.</li>
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -392,6 +385,7 @@ subsections of the DAG as well as specifying date ranges.</li>
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -404,19 +398,13 @@ subsections of the DAG as well as specifying date ranges.</li>
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/search.html
----------------------------------------------------------------------
diff --git a/search.html b/search.html
index 665d00d..bbeec88 100644
--- a/search.html
+++ b/search.html
@@ -24,25 +24,17 @@
-
-
- <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="genindex.html"/>
- <link rel="search" title="Search" href="#"/>
- <link rel="top" title="Airflow Documentation" href="index.html"/>
+ <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="genindex.html" />
+ <link rel="search" title="Search" href="#" />
<script src="_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -115,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="index.html">Airflow</a>
@@ -123,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -153,6 +146,8 @@
<li class="wy-breadcrumbs-aside">
+
+
</li>
</ul>
@@ -178,9 +173,7 @@
</div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -192,7 +185,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -211,6 +204,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'./',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -224,19 +218,13 @@
-
-
- <script type="text/javascript" src="_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
</script>
-
<script type="text/javascript">
jQuery(function() { Search.loadIndex("searchindex.js"); });
</script>