You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2016/07/01 21:54:28 UTC
[2/5] incubator-airflow-site git commit: Updating the disclaimer
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/slack_operator.html
----------------------------------------------------------------------
diff --git a/_modules/slack_operator.html b/_modules/slack_operator.html
index f958c5b..152902b 100644
--- a/_modules/slack_operator.html
+++ b/_modules/slack_operator.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for slack_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">slackclient</span> <span class="kn">import</span> <span class="n">SlackClient</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/sqlite_hook.html
----------------------------------------------------------------------
diff --git a/_modules/sqlite_hook.html b/_modules/sqlite_hook.html
index 6ed5de8..70abb8c 100644
--- a/_modules/sqlite_hook.html
+++ b/_modules/sqlite_hook.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for sqlite_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">sqlite3</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">sqlite3</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.dbapi_hook</span> <span class="kn">import</span> <span class="n">DbApiHook</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/ssh_execute_operator.html
----------------------------------------------------------------------
diff --git a/_modules/ssh_execute_operator.html b/_modules/ssh_execute_operator.html
index 4346f31..c4c952d 100644
--- a/_modules/ssh_execute_operator.html
+++ b/_modules/ssh_execute_operator.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for ssh_execute_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">bytes</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">from</span> <span class="nn">subprocess</span> <span class="kn">import</span> <span class="n">STDOUT</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/vertica_operator.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_operator.html b/_modules/vertica_operator.html
index 2ae2681..46f8a92 100644
--- a/_modules/vertica_operator.html
+++ b/_modules/vertica_operator.html
@@ -137,9 +137,23 @@
<div itemprop="articleBody">
<h1>Source code for vertica_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/vertica_to_hive.html
----------------------------------------------------------------------
diff --git a/_modules/vertica_to_hive.html b/_modules/vertica_to_hive.html
index 57c83f1..a700374 100644
--- a/_modules/vertica_to_hive.html
+++ b/_modules/vertica_to_hive.html
@@ -137,14 +137,28 @@
<div itemprop="articleBody">
<h1>Source code for vertica_to_hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">OrderedDict</span>
<span class="kn">import</span> <span class="nn">unicodecsv</span> <span class="kn">as</span> <span class="nn">csv</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
-<span class="kn">from</span> <span class="nn">airflow.contrib.hooks</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/webhdfs_hook.html
----------------------------------------------------------------------
diff --git a/_modules/webhdfs_hook.html b/_modules/webhdfs_hook.html
index d54cb05..3a44028 100644
--- a/_modules/webhdfs_hook.html
+++ b/_modules/webhdfs_hook.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for webhdfs_hook</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.base_hook</span> <span class="kn">import</span> <span class="n">BaseHook</span>
<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
<span class="kn">import</span> <span class="nn">logging</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/code.txt
----------------------------------------------------------------------
diff --git a/_sources/code.txt b/_sources/code.txt
index 2876876..ba1cddb 100644
--- a/_sources/code.txt
+++ b/_sources/code.txt
@@ -133,6 +133,10 @@ Variable Description
``{{ latest_date }}`` same as ``{{ ds }}``
``{{ ti }}`` same as ``{{ task_instance }}``
``{{ params }}`` a reference to the user-defined params dictionary
+``{{ var.value.my_var }}`` global defined variables represented as a dictionary
+``{{ var.json.my_var.path }}`` global defined variables represented as a dictionary
+ with deserialized JSON object, append the path to the
+ key within the JSON object
``{{ task_instance_key_str }}`` a unique, human-readable key to the task instance
formatted ``{dag_id}_{task_id}_{ds}``
``conf`` the full configuration object located at
@@ -151,6 +155,11 @@ dot notation. Here are some examples of what is possible:
Refer to the models documentation for more information on the objects'
attributes and methods.
+The ``var`` template variable allows you to access variables defined in Airflow's
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+``{{ var.json.my_dict_var.key1 }}``
+
Macros
''''''
Macros are a way to expose objects to your templates and live under the
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/concepts.txt
----------------------------------------------------------------------
diff --git a/_sources/concepts.txt b/_sources/concepts.txt
index 811130d..8cfc8ab 100644
--- a/_sources/concepts.txt
+++ b/_sources/concepts.txt
@@ -485,7 +485,7 @@ the main UI. For example:
#dags/subdag.py
from airflow.models import DAG
- from airflow.operators import DummyOperator
+ from airflow.operators.dummy_operator import DummyOperator
# Dag is returned by a factory method
@@ -510,7 +510,7 @@ This SubDAG can then be referenced in your main DAG file:
# main_dag.py
from datetime import datetime, timedelta
from airflow.models import DAG
- from airflow.operators import SubDagOperator
+ from airflow.operators.subdag_operator import SubDagOperator
from dags.subdag import sub_dag
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/faq.txt
----------------------------------------------------------------------
diff --git a/_sources/faq.txt b/_sources/faq.txt
index 21623fc..e61c1bf 100644
--- a/_sources/faq.txt
+++ b/_sources/faq.txt
@@ -78,12 +78,17 @@ We recommend against using dynamic values as ``start_date``, especially
once the period closes, and in theory an ``@hourly`` DAG would never get to
an hour after now as ``now()`` moves along.
-We also recommend using rounded ``start_date`` in relation to your
-``schedule_interval``. This means an ``@hourly`` would be at ``00:00``
+
+Previously we also recommended using rounded ``start_date`` in relation to your
+``schedule_interval``. This meant an ``@hourly`` would be at ``00:00``
minutes:seconds, a ``@daily`` job at midnight, a ``@monthly`` job on the
-first of the month. You can use any sensor or a ``TimeDeltaSensor`` to delay
-the execution of tasks within that period. While ``schedule_interval``
-does allow specifying a ``datetime.timedelta``
+first of the month. This is no longer required. Airflow will not auto align
+the ``start_date`` and the ``schedule_interval``, by using the ``start_date``
+as the moment to start looking.
+
+You can use any sensor or a ``TimeDeltaSensor`` to delay
+the execution of tasks within the schedule interval.
+While ``schedule_interval`` does allow specifying a ``datetime.timedelta``
object, we recommend using the macros or cron expressions instead, as
it enforces this idea of rounded schedules.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/index.txt
----------------------------------------------------------------------
diff --git a/_sources/index.txt b/_sources/index.txt
index c98d619..133d5ab 100644
--- a/_sources/index.txt
+++ b/_sources/index.txt
@@ -7,10 +7,10 @@
Apache Airflow (incubating) Documentation
=========================================
-.. note ::
+.. important::
- Apache Airflow is an effort undergoing incubation at The Apache
- Software Foundation (ASF), sponsored by Chris Riccomini.
+ **Disclaimer**: Apache Airflow is an effort undergoing incubation at The
+ Apache Software Foundation (ASF), sponsored by the Apache Incubator.
Incubation is required of all newly accepted projects until a further
review indicates that the infrastructure, communications, and
decision making process have stabilized in a manner consistent with
@@ -19,6 +19,7 @@ Apache Airflow (incubating) Documentation
the code, it does indicate that the project has yet to be fully
endorsed by the ASF.
+
Airflow is a platform to programmatically author, schedule and monitor
workflows.
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_sources/tutorial.txt
----------------------------------------------------------------------
diff --git a/_sources/tutorial.txt b/_sources/tutorial.txt
index e9d382b..a93479c 100644
--- a/_sources/tutorial.txt
+++ b/_sources/tutorial.txt
@@ -18,7 +18,7 @@ complicated, a line by line explanation follows below.
https://github.com/airbnb/airflow/blob/master/airflow/example_dags/tutorial.py
"""
from airflow import DAG
- from airflow.operators import BashOperator
+ from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
@@ -100,7 +100,7 @@ Airflow DAG object. Let's start by importing the libraries we will need.
from airflow import DAG
# Operators; we need this to operate!
- from airflow.operators import BashOperator
+ from airflow.operators.bash_operator import BashOperator
Default Arguments
-----------------
@@ -270,7 +270,7 @@ something like this:
http://airflow.readthedocs.org/en/latest/tutorial.html
"""
from airflow import DAG
- from airflow.operators import BashOperator
+ from airflow.operators.bash_operator import BashOperator
from datetime import datetime, timedelta
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/code.html
----------------------------------------------------------------------
diff --git a/code.html b/code.html
index 06a0aab..592ec27 100644
--- a/code.html
+++ b/code.html
@@ -322,6 +322,15 @@ between each tries</li>
</div>
<div class="section" id="module-airflow.operators">
<span id="operator-api"></span><h3>Operator API<a class="headerlink" href="#module-airflow.operators" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
<dl class="class">
<dt id="airflow.operators.BashOperator">
<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">BashOperator</code><span class="sig-paren">(</span><em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>output_encoding='utf-8'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bash_operator.html#BashOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.BashOperator" title="Permalink to this definition">�</a></dt>
@@ -431,7 +440,7 @@ is allowed</li>
<dl class="class">
<dt id="airflow.operators.ExternalTaskSensor">
-<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.ExternalTaskSensor" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">ExternalTaskSensor</code><span class="sig-paren">(</span><em>external_dag_id</em>, <em>external_task_id</em>, <em>allowed_states=None</em>, <em>execution_delta=None</em>, <em>execution_date_fn=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#ExternalTaskSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.ExternalTaskSensor" title="Permalink to this definition">�</a></dt>
<dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
<p>Waits for a task to complete in a different DAG</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -446,7 +455,12 @@ wait for</li>
<li><strong>allowed_states</strong> (<em>list</em>) – list of allowed states, default is <code class="docutils literal"><span class="pre">['success']</span></code></li>
<li><strong>execution_delta</strong> (<em>datetime.timedelta</em>) – time difference with the previous execution to
look at, the default is the same execution_date as the current task.
-For yesterday, use [positive!] datetime.timedelta(days=1)</li>
+For yesterday, use [positive!] datetime.timedelta(days=1). Either
+execution_delta or execution_date_fn can be passed to
+ExternalTaskSensor, but not both.</li>
+<li><strong>execution_date_fn</strong> (<em>callable</em>) – function that receives the current execution date
+and returns the desired execution date to query. Either execution_delta
+or execution_date_fn can be passed to ExternalTaskSensor, but not both.</li>
</ul>
</td>
</tr>
@@ -539,7 +553,10 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
<dt id="airflow.operators.HivePartitionSensor">
<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">HivePartitionSensor</code><span class="sig-paren">(</span><em>table</em>, <em>partition="ds='{{ ds }}'"</em>, <em>metastore_conn_id='metastore_default'</em>, <em>schema='default'</em>, <em>poke_interval=180</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/sensors.html#HivePartitionSensor"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.HivePartitionSensor" title="Permalink to this definition">�</a></dt>
<dd><p>Bases: <a class="reference internal" href="#airflow.operators.sensors.BaseSensorOperator" title="airflow.operators.sensors.BaseSensorOperator"><code class="xref py py-class docutils literal"><span class="pre">sensors.BaseSensorOperator</span></code></a></p>
-<p>Waits for a partition to show up in Hive</p>
+<p>Waits for a partition to show up in Hive.</p>
+<p>Note: Because @partition supports general logical operators, it
+can be inefficient. Consider using NamedHivePartitionSensor instead if
+you don’t need the full flexibility of HivePartitionSensor.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
@@ -548,7 +565,7 @@ part of the script before the first occurrence of <cite>script_begin_tag</cite><
<li><strong>table</strong> (<em>string</em>) – The name of the table to wait for, supports the dot
notation (my_database.my_table)</li>
<li><strong>partition</strong> (<em>string</em>) – The partition clause to wait for. This is passed as
-is to the Metastore Thrift client “get_partitions_by_filter” method,
+is to the metastore Thrift client “get_partitions_by_filter” method,
and apparently supports SQL like notation as in <cite>ds=‘2015-01-01’
AND type=’value’</cite> and > < sings as in “ds>=2015-01-01”</li>
<li><strong>metastore_conn_id</strong> (<em>str</em>) – reference to the metastore thrift service
@@ -841,7 +858,7 @@ Template reference are recognized by str ending in '.sql'</em>) – the sql
<dl class="class">
<dt id="airflow.operators.PrestoCheckOperator">
<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.CheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.CheckOperator</span></code></p>
<p>Performs checks against Presto. The <code class="docutils literal"><span class="pre">PrestoCheckOperator</span></code> expects
a sql query that will return a single row. Each value on that
first row is evaluated using python <code class="docutils literal"><span class="pre">bool</span></code> casting. If any of the
@@ -880,7 +897,7 @@ without stopping the progress of the DAG.</p>
<dl class="class">
<dt id="airflow.operators.PrestoIntervalCheckOperator">
<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoIntervalCheckOperator</code><span class="sig-paren">(</span><em>table</em>, <em>metrics_thresholds</em>, <em>date_filter_column='ds'</em>, <em>days_back=-7</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoIntervalCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoIntervalCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.IntervalCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.IntervalCheckOperator</span></code></p>
<p>Checks that the values of metrics given as SQL expressions are within
a certain tolerance of the ones from days_back before.</p>
<table class="docutils field-list" frame="void" rules="none">
@@ -903,7 +920,7 @@ against. Defaults to 7 days</li>
<dl class="class">
<dt id="airflow.operators.PrestoValueCheckOperator">
<em class="property">class </em><code class="descclassname">airflow.operators.</code><code class="descname">PrestoValueCheckOperator</code><span class="sig-paren">(</span><em>sql</em>, <em>pass_value</em>, <em>tolerance=None</em>, <em>presto_conn_id='presto_default'</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/presto_check_operator.html#PrestoValueCheckOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.operators.PrestoValueCheckOperator" title="Permalink to this definition">�</a></dt>
-<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">check_operator.ValueCheckOperator</span></code></p>
+<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.operators.check_operator.ValueCheckOperator</span></code></p>
<p>Performs a simple value check using sql code.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
@@ -1188,6 +1205,15 @@ The default is False.</li>
</div>
<div class="section" id="module-airflow.contrib.operators">
<span id="community-contributed-operators"></span><h3>Community-contributed Operators<a class="headerlink" href="#module-airflow.contrib.operators" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
<dl class="class">
<dt id="airflow.contrib.operators.SSHExecuteOperator">
<em class="property">class </em><code class="descclassname">airflow.contrib.operators.</code><code class="descname">SSHExecuteOperator</code><span class="sig-paren">(</span><em>ssh_hook</em>, <em>bash_command</em>, <em>xcom_push=False</em>, <em>env=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ssh_execute_operator.html#SSHExecuteOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.SSHExecuteOperator" title="Permalink to this definition">�</a></dt>
@@ -1286,7 +1312,7 @@ specific database</li>
<dl class="class">
<dt id="airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator">
-<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">�</a></dt>
+<em class="property">class </em><code class="descclassname">airflow.contrib.operators.gcs_download_operator.</code><code class="descname">GoogleCloudStorageDownloadOperator</code><span class="sig-paren">(</span><em>bucket</em>, <em>object</em>, <em>filename=False</em>, <em>store_to_xcom_key=False</em>, <em>google_cloud_storage_conn_id='google_cloud_storage_default'</em>, <em>delegate_to=None</em>, <em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/contrib/operators/gcs_download_operator.html#GoogleCloudStorageDownloadOperator"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.operators.gcs_download_operator.GoogleCloudStorageDownloadOperator" title="Permalink to this definition">�</a></dt>
<dd><p>Downloads a file from Google Cloud Storage.</p>
</dd></dl>
@@ -1560,8 +1586,8 @@ More info: <a class="reference external" href="https://www.hipchat.com/docs/apiv
in all templates</p>
<table border="1" class="docutils">
<colgroup>
-<col width="40%" />
-<col width="60%" />
+<col width="38%" />
+<col width="62%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Variable</th>
@@ -1620,6 +1646,14 @@ in all templates</p>
<tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">params</span> <span class="pre">}}</span></code></td>
<td>a reference to the user-defined params dictionary</td>
</tr>
+<tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.value.my_var</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary</td>
+</tr>
+<tr class="row-even"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_var.path</span> <span class="pre">}}</span></code></td>
+<td>global defined variables represented as a dictionary
+with deserialized JSON object, append the path to the
+key within the JSON object</td>
+</tr>
<tr class="row-odd"><td><code class="docutils literal"><span class="pre">{{</span> <span class="pre">task_instance_key_str</span> <span class="pre">}}</span></code></td>
<td>a unique, human-readable key to the task instance
formatted <code class="docutils literal"><span class="pre">{dag_id}_{task_id}_{ds}</span></code></td>
@@ -1647,6 +1681,10 @@ dot notation. Here are some examples of what is possible:
<code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.owner</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">task.task_id</span> <span class="pre">}}</span></code>, <code class="docutils literal"><span class="pre">{{</span> <span class="pre">ti.hostname</span> <span class="pre">}}</span></code>, ...
Refer to the models documentation for more information on the objects’
attributes and methods.</p>
+<p>The <code class="docutils literal"><span class="pre">var</span></code> template variable allows you to access variables defined in Airflow’s
+UI. You can access them as either plain-text or JSON. If you use JSON, you are
+also able to walk nested structures, such as dictionaries like:
+<code class="docutils literal"><span class="pre">{{</span> <span class="pre">var.json.my_dict_var.key1</span> <span class="pre">}}</span></code></p>
</div>
<div class="section" id="id2">
<h3>Macros<a class="headerlink" href="#id2" title="Permalink to this headline">�</a></h3>
@@ -1736,12 +1774,6 @@ as specified in the output format</p>
</dd></dl>
<dl class="function">
-<dt id="airflow.macros.integrate_plugins">
-<code class="descclassname">airflow.macros.</code><code class="descname">integrate_plugins</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/macros.html#integrate_plugins"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.macros.integrate_plugins" title="Permalink to this definition">�</a></dt>
-<dd><p>Integrate plugins to the context</p>
-</dd></dl>
-
-<dl class="function">
<dt id="airflow.macros.random">
<code class="descclassname">airflow.macros.</code><code class="descname">random</code><span class="sig-paren">(</span><span class="sig-paren">)</span> → x in the interval [0, 1).<a class="headerlink" href="#airflow.macros.random" title="Permalink to this definition">�</a></dt>
<dd></dd></dl>
@@ -1972,6 +2004,12 @@ template_searchpath and user_defined_macros</p>
</dd></dl>
<dl class="method">
+<dt id="airflow.models.DAG.normalize_schedule">
+<code class="descname">normalize_schedule</code><span class="sig-paren">(</span><em>dttm</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.normalize_schedule"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.normalize_schedule" title="Permalink to this definition">�</a></dt>
+<dd><p>Returns dttm + interval unless dttm is first interval then it returns dttm</p>
+</dd></dl>
+
+<dl class="method">
<dt id="airflow.models.DAG.run">
<code class="descname">run</code><span class="sig-paren">(</span><em>start_date=None</em>, <em>end_date=None</em>, <em>mark_success=False</em>, <em>include_adhoc=False</em>, <em>local=False</em>, <em>executor=None</em>, <em>donot_pickle=False</em>, <em>ignore_dependencies=False</em>, <em>ignore_first_depends_on_past=False</em>, <em>pool=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#DAG.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.DAG.run" title="Permalink to this definition">�</a></dt>
<dd><p>Runs the DAG.</p>
@@ -2465,6 +2503,24 @@ session is committed.</td>
<dt id="airflow.models.TaskInstance.run">
<code class="descname">run</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/airflow/models.html#TaskInstance.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.models.TaskInstance.run" title="Permalink to this definition">�</a></dt>
<dd><p>Runs the task instance.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
+<li><strong>verbose</strong> (<em>boolean</em>) – whether to turn on more verbose loggin</li>
+<li><strong>ignore_dependencies</strong> (<em>boolean</em>) – Doesn’t check for deps, just runs</li>
+<li><strong>ignore_depends_on_past</strong> (<em>boolean</em>) – Ignore depends_on_past but respect
+other dependencies</li>
+<li><strong>force</strong> (<em>boolean</em>) – Forces a run regarless of previous success</li>
+<li><strong>mark_success</strong> (<em>boolean</em>) – Don’t run the task, mark its state as success</li>
+<li><strong>test_mode</strong> (<em>boolean</em>) – Doesn’t record success or failure in the DB</li>
+<li><strong>pool</strong> (<em>str</em>) – specifies the pool to use to run the task instance</li>
+</ul>
+</td>
+</tr>
+</tbody>
+</table>
</dd></dl>
<dl class="method">
@@ -2631,6 +2687,15 @@ passwords when using operators or hooks.</p>
</div>
<div class="section" id="module-airflow.hooks">
<span id="hooks"></span><h2>Hooks<a class="headerlink" href="#module-airflow.hooks" title="Permalink to this headline">�</a></h2>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
<dl class="class">
<dt id="airflow.hooks.DbApiHook">
<em class="property">class </em><code class="descclassname">airflow.hooks.</code><code class="descname">DbApiHook</code><span class="sig-paren">(</span><em>*args</em>, <em>**kwargs</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/dbapi_hook.html#DbApiHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.DbApiHook" title="Permalink to this definition">�</a></dt>
@@ -2799,7 +2864,8 @@ traditional CLI. To enable <code class="docutils literal"><span class="pre">beel
extra field of your connection as in <code class="docutils literal"><span class="pre">{</span> <span class="pre">"use_beeline":</span> <span class="pre">true</span> <span class="pre">}</span></code></p>
<p>Note that you can also set default hive CLI parameters using the
<code class="docutils literal"><span class="pre">hive_cli_params</span></code> to be used in your connection as in
-<code class="docutils literal"><span class="pre">{"hive_cli_params":</span> <span class="pre">"-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444"}</span></code></p>
+<code class="docutils literal"><span class="pre">{"hive_cli_params":</span> <span class="pre">"-hiveconf</span> <span class="pre">mapred.job.tracker=some.jobtracker:444"}</span></code>
+Parameters passed here can be overridden by run_cli’s hive_conf param</p>
<p>The extra connection parameter <code class="docutils literal"><span class="pre">auth</span></code> gets passed as in the <code class="docutils literal"><span class="pre">jdbc</span></code>
connection string as is.</p>
<dl class="method">
@@ -2834,8 +2900,20 @@ and values</li>
<dl class="method">
<dt id="airflow.hooks.HiveCliHook.run_cli">
-<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">�</a></dt>
-<dd><p>Run an hql statement using the hive cli</p>
+<code class="descname">run_cli</code><span class="sig-paren">(</span><em>hql</em>, <em>schema=None</em>, <em>verbose=True</em>, <em>hive_conf=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveCliHook.run_cli"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveCliHook.run_cli" title="Permalink to this definition">�</a></dt>
+<dd><p>Run an hql statement using the hive cli. If hive_conf is specified it should be a
+dict and the entries will be set as key/value pairs in HiveConf</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>hive_conf</strong> (<em>dict</em>) – if specified these key value pairs will be passed to hive as
+<code class="docutils literal"><span class="pre">-hiveconf</span> <span class="pre">"key"="value"</span></code>. Note that they will be passed after the
+<code class="docutils literal"><span class="pre">hive_cli_params</span></code> and thus will override whatever values are specified in
+the database.</td>
+</tr>
+</tbody>
+</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">result</span> <span class="o">=</span> <span class="n">hh</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="s2">"USE airflow;"</span><span class="p">)</span>
<span class="gp">>>> </span><span class="p">(</span><span class="s2">"OK"</span> <span class="ow">in</span> <span class="n">result</span><span class="p">)</span>
@@ -2858,9 +2936,60 @@ and values</li>
<dd><p>Bases: <code class="xref py py-class docutils literal"><span class="pre">airflow.hooks.base_hook.BaseHook</span></code></p>
<p>Wrapper to interact with the Hive Metastore</p>
<dl class="method">
+<dt id="airflow.hooks.HiveMetastoreHook.check_for_named_partition">
+<code class="descname">check_for_named_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition_name</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_named_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_named_partition" title="Permalink to this definition">�</a></dt>
+<dd><p>Checks whether a partition with a given name exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) – Name of hive schema (database) @table belongs to</li>
+<li><strong>table</strong> – Name of hive table @partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Name of the partitions to check for (eg <cite>a=b/c=d</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
+<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
+<span class="gp">>>> </span><span class="n">t</span> <span class="o">=</span> <span class="s1">'static_babynames_partitioned'</span>
+<span class="gp">>>> </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">'airflow'</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">"ds=2015-01-01"</span><span class="p">)</span>
+<span class="go">True</span>
+<span class="gp">>>> </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_named_partition</span><span class="p">(</span><span class="s1">'airflow'</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">"ds=xxx"</span><span class="p">)</span>
+<span class="go">False</span>
+</pre></div>
+</div>
+</dd></dl>
+
+<dl class="method">
<dt id="airflow.hooks.HiveMetastoreHook.check_for_partition">
<code class="descname">check_for_partition</code><span class="sig-paren">(</span><em>schema</em>, <em>table</em>, <em>partition</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/hive_hooks.html#HiveMetastoreHook.check_for_partition"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.hooks.HiveMetastoreHook.check_for_partition" title="Permalink to this definition">�</a></dt>
<dd><p>Checks whether a partition exists</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>schema</strong> (<em>string</em>) – Name of hive schema (database) @table belongs to</li>
+<li><strong>table</strong> – Name of hive table @partition belongs to</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Partition:</th><td class="field-body"><p class="first">Expression that matches the partitions to check for
+(eg <cite>a = ‘b’ AND c = ‘d’</cite>)</p>
+</td>
+</tr>
+<tr class="field-odd field"><th class="field-name">Return type:</th><td class="field-body"><p class="first last">boolean</p>
+</td>
+</tr>
+</tbody>
+</table>
<div class="highlight-default"><div class="highlight"><pre><span></span><span class="gp">>>> </span><span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">t</span> <span class="o">=</span> <span class="s1">'static_babynames_partitioned'</span>
<span class="gp">>>> </span><span class="n">hh</span><span class="o">.</span><span class="n">check_for_partition</span><span class="p">(</span><span class="s1">'airflow'</span><span class="p">,</span> <span class="n">t</span><span class="p">,</span> <span class="s2">"ds='2015-01-01'"</span><span class="p">)</span>
@@ -3377,6 +3506,15 @@ directory, files will be uploaded inside.</li>
<div class="section" id="module-airflow.contrib.hooks">
<span id="community-contributed-hooks"></span><h3>Community contributed hooks<a class="headerlink" href="#module-airflow.contrib.hooks" title="Permalink to this headline">�</a></h3>
+<p>Importer that dynamically loads a class and module from its parent. This
+allows Airflow to support <cite>from airflow.operators.bash_operator import
+BashOperator</cite> even though BashOperator is actually in
+airflow.operators.bash_operator.</p>
+<p>The importer also takes over for the parent_module by wrapping it. This is
+required to support attribute-based usage:</p>
+<blockquote>
+<div>from airflow import operators
+operators.BashOperator(...)</div></blockquote>
<dl class="class">
<dt id="airflow.contrib.hooks.BigQueryHook">
<em class="property">class </em><code class="descclassname">airflow.contrib.hooks.</code><code class="descname">BigQueryHook</code><span class="sig-paren">(</span><em>bigquery_conn_id='bigquery_default'</em>, <em>delegate_to=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/bigquery_hook.html#BigQueryHook"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.BigQueryHook" title="Permalink to this definition">�</a></dt>
@@ -3500,7 +3638,7 @@ downstream.</p>
<dt id="airflow.contrib.hooks.FTPHook.close_conn">
<code class="descname">close_conn</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.close_conn"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.close_conn" title="Permalink to this definition">�</a></dt>
<dd><p>Closes the connection. An error will occur if the
-connection wasnt ever opened.</p>
+connection wasn’t ever opened.</p>
</dd></dl>
<dl class="method">
@@ -3534,7 +3672,7 @@ connection wasnt ever opened.</p>
<dl class="method">
<dt id="airflow.contrib.hooks.FTPHook.delete_file">
<code class="descname">delete_file</code><span class="sig-paren">(</span><em>path</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/ftp_hook.html#FTPHook.delete_file"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#airflow.contrib.hooks.FTPHook.delete_file" title="Permalink to this definition">�</a></dt>
-<dd><p>Removes a file on the FTP Server</p>
+<dd><p>Removes a file on the FTP Server.</p>
<table class="docutils field-list" frame="void" rules="none">
<col class="field-name" />
<col class="field-body" />
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/concepts.html
----------------------------------------------------------------------
diff --git a/concepts.html b/concepts.html
index b41766f..15b2b4d 100644
--- a/concepts.html
+++ b/concepts.html
@@ -573,7 +573,7 @@ object. This will prevent the SubDAG from being treated like a separate DAG in
the main UI. For example:</p>
<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1">#dags/subdag.py</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">DummyOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.dummy_operator</span> <span class="k">import</span> <span class="n">DummyOperator</span>
<span class="c1"># Dag is returned by a factory method</span>
@@ -596,7 +596,7 @@ the main UI. For example:</p>
<div class="code python highlight-default"><div class="highlight"><pre><span></span><span class="c1"># main_dag.py</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">DAG</span>
-<span class="kn">from</span> <span class="nn">airflow.operators</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.subdag_operator</span> <span class="k">import</span> <span class="n">SubDagOperator</span>
<span class="kn">from</span> <span class="nn">dags.subdag</span> <span class="k">import</span> <span class="n">sub_dag</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/faq.html
----------------------------------------------------------------------
diff --git a/faq.html b/faq.html
index 050da7b..3d8cb5d 100644
--- a/faq.html
+++ b/faq.html
@@ -202,12 +202,15 @@ inactive DagRuns to get the new task to get onboarded properly.</p>
<code class="docutils literal"><span class="pre">datetime.now()</span></code> as it can be quite confusing. The task is triggered
once the period closes, and in theory an <code class="docutils literal"><span class="pre">@hourly</span></code> DAG would never get to
an hour after now as <code class="docutils literal"><span class="pre">now()</span></code> moves along.</p>
-<p>We also recommend using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
-<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This means an <code class="docutils literal"><span class="pre">@hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
+<p>Previously we also recommended using rounded <code class="docutils literal"><span class="pre">start_date</span></code> in relation to your
+<code class="docutils literal"><span class="pre">schedule_interval</span></code>. This meant an <code class="docutils literal"><span class="pre">@hourly</span></code> would be at <code class="docutils literal"><span class="pre">00:00</span></code>
minutes:seconds, a <code class="docutils literal"><span class="pre">@daily</span></code> job at midnight, a <code class="docutils literal"><span class="pre">@monthly</span></code> job on the
-first of the month. You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
-the execution of tasks within that period. While <code class="docutils literal"><span class="pre">schedule_interval</span></code>
-does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
+first of the month. This is no longer required. Airflow will not auto align
+the <code class="docutils literal"><span class="pre">start_date</span></code> and the <code class="docutils literal"><span class="pre">schedule_interval</span></code>, by using the <code class="docutils literal"><span class="pre">start_date</span></code>
+as the moment to start looking.</p>
+<p>You can use any sensor or a <code class="docutils literal"><span class="pre">TimeDeltaSensor</span></code> to delay
+the execution of tasks within the schedule interval.
+While <code class="docutils literal"><span class="pre">schedule_interval</span></code> does allow specifying a <code class="docutils literal"><span class="pre">datetime.timedelta</span></code>
object, we recommend using the macros or cron expressions instead, as
it enforces this idea of rounded schedules.</p>
<p>When using <code class="docutils literal"><span class="pre">depends_on_past=True</span></code> it’s important to pay special attention
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/genindex.html
----------------------------------------------------------------------
diff --git a/genindex.html b/genindex.html
index ecba6b9..20b510f 100644
--- a/genindex.html
+++ b/genindex.html
@@ -150,6 +150,7 @@
| <a href="#K"><strong>K</strong></a>
| <a href="#L"><strong>L</strong></a>
| <a href="#M"><strong>M</strong></a>
+ | <a href="#N"><strong>N</strong></a>
| <a href="#O"><strong>O</strong></a>
| <a href="#P"><strong>P</strong></a>
| <a href="#Q"><strong>Q</strong></a>
@@ -288,6 +289,10 @@
</dt>
+ <dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_named_partition">check_for_named_partition() (airflow.hooks.HiveMetastoreHook method)</a>
+ </dt>
+
+
<dt><a href="code.html#airflow.hooks.HiveMetastoreHook.check_for_partition">check_for_partition() (airflow.hooks.HiveMetastoreHook method)</a>
</dt>
@@ -774,19 +779,15 @@
</dl></dd>
- <dt><a href="code.html#airflow.macros.integrate_plugins">integrate_plugins() (in module airflow.macros)</a>
- </dt>
-
-
<dt><a href="code.html#airflow.models.DAG.is_paused">is_paused (airflow.models.DAG attribute)</a>
</dt>
- </dl></td>
- <td style="width: 33%" valign="top"><dl>
<dt><a href="code.html#airflow.models.TaskInstance.is_premature">is_premature() (airflow.models.TaskInstance method)</a>
</dt>
+ </dl></td>
+ <td style="width: 33%" valign="top"><dl>
<dt><a href="code.html#airflow.models.TaskInstance.is_queueable">is_queueable() (airflow.models.TaskInstance method)</a>
</dt>
@@ -914,6 +915,16 @@
</dl></td>
</tr></table>
+<h2 id="N">N</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+ <td style="width: 33%" valign="top"><dl>
+
+ <dt><a href="code.html#airflow.models.DAG.normalize_schedule">normalize_schedule() (airflow.models.DAG method)</a>
+ </dt>
+
+ </dl></td>
+</tr></table>
+
<h2 id="O">O</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%" valign="top"><dl>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 383bab0..5ac4744 100644
--- a/index.html
+++ b/index.html
@@ -140,10 +140,10 @@
<a class="reference internal image-reference" href="_images/incubator.jpg"><img alt="_images/incubator.jpg" src="_images/incubator.jpg" style="width: 150px;" /></a>
<div class="section" id="apache-airflow-incubating-documentation">
<h1>Apache Airflow (incubating) Documentation<a class="headerlink" href="#apache-airflow-incubating-documentation" title="Permalink to this headline">�</a></h1>
-<div class="admonition note">
-<p class="first admonition-title">Note</p>
-<p class="last">Apache Airflow is an effort undergoing incubation at The Apache
-Software Foundation (ASF), sponsored by Chris Riccomini.
+<div class="admonition important">
+<p class="first admonition-title">Important</p>
+<p class="last"><strong>Disclaimer</strong>: Apache Airflow is an effort undergoing incubation at The
+Apache Software Foundation (ASF), sponsored by the Apache Incubator.
Incubation is required of all newly accepted projects until a further
review indicates that the infrastructure, communications, and
decision making process have stabilized in a manner consistent with
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/objects.inv
----------------------------------------------------------------------
diff --git a/objects.inv b/objects.inv
index 36f1543..56e566e 100644
Binary files a/objects.inv and b/objects.inv differ