You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2016/07/01 21:54:31 UTC
[5/5] incubator-airflow-site git commit: Updating the disclaimer
Updating the disclaimer
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/457b698d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/457b698d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/457b698d
Branch: refs/heads/asf-site
Commit: 457b698d9f92af370061d322267d85886e6cbfe4
Parents: 502934c
Author: Maxime Beauchemin <ma...@apache.org>
Authored: Fri Jul 1 14:54:14 2016 -0700
Committer: Maxime Beauchemin <ma...@apache.org>
Committed: Fri Jul 1 14:54:14 2016 -0700
----------------------------------------------------------------------
.../contrib/executors/mesos_executor.html | 16 +-
.../contrib/operators/bigquery_operator.html | 16 +-
.../contrib/operators/bigquery_to_gcs.html | 16 +-
.../operators/gcs_download_operator.html | 37 +++-
_modules/airflow/executors/celery_executor.html | 16 +-
_modules/airflow/executors/local_executor.html | 16 +-
.../airflow/executors/sequential_executor.html | 16 +-
_modules/airflow/macros.html | 40 ++++-
_modules/airflow/macros/hive.html | 20 ++-
_modules/airflow/models.html | 111 +++++++++---
_modules/airflow/operators/docker_operator.html | 16 +-
_modules/airflow/operators/sensors.html | 137 ++++++++++++--
_modules/bash_operator.html | 16 +-
_modules/dagrun_operator.html | 16 +-
_modules/dbapi_hook.html | 24 ++-
_modules/druid_hook.html | 16 +-
_modules/dummy_operator.html | 16 +-
_modules/email_operator.html | 16 +-
_modules/ftp_hook.html | 22 ++-
_modules/generic_transfer.html | 16 +-
_modules/hive_hooks.html | 82 +++++++--
_modules/hive_operator.html | 22 ++-
_modules/hive_to_druid.html | 48 +++--
_modules/hive_to_mysql.html | 21 ++-
_modules/hive_to_samba_operator.html | 19 +-
_modules/http_hook.html | 2 +-
_modules/http_operator.html | 18 +-
_modules/mssql_hook.html | 25 ++-
_modules/mssql_operator.html | 20 ++-
_modules/mssql_to_hive.html | 19 +-
_modules/mysql_hook.html | 16 +-
_modules/mysql_operator.html | 20 ++-
_modules/mysql_to_hive.html | 19 +-
_modules/postgres_hook.html | 16 +-
_modules/postgres_operator.html | 20 ++-
_modules/presto_check_operator.html | 18 +-
_modules/presto_hook.html | 25 ++-
_modules/python_operator.html | 16 +-
_modules/qubole_operator.html | 20 ++-
_modules/s3_to_hive_operator.html | 19 +-
_modules/sensors.html | 137 ++++++++++++--
_modules/slack_operator.html | 16 +-
_modules/sqlite_hook.html | 16 +-
_modules/ssh_execute_operator.html | 16 +-
_modules/vertica_operator.html | 20 ++-
_modules/vertica_to_hive.html | 20 ++-
_modules/webhdfs_hook.html | 16 +-
_sources/code.txt | 9 +
_sources/concepts.txt | 4 +-
_sources/faq.txt | 15 +-
_sources/index.txt | 7 +-
_sources/tutorial.txt | 6 +-
code.html | 180 ++++++++++++++++---
concepts.html | 4 +-
faq.html | 13 +-
genindex.html | 23 ++-
index.html | 8 +-
objects.inv | Bin 2236 -> 2246 bytes
searchindex.js | 2 +-
tutorial.html | 6 +-
60 files changed, 1363 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index 65557d9..e2d8f61 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.contrib.executors.mesos_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
<span class="kn">import</span> <span class="nn">logging</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_operator.html b/_modules/airflow/contrib/operators/bigquery_operator.html
index 830fd57..6038f2c 100644
--- a/_modules/airflow/contrib/operators/bigquery_operator.html
+++ b/_modules/airflow/contrib/operators/bigquery_operator.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.contrib.operators.bigquery_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/bigquery_to_gcs.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/bigquery_to_gcs.html b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
index 465128c..a3b8d18 100644
--- a/_modules/airflow/contrib/operators/bigquery_to_gcs.html
+++ b/_modules/airflow/contrib/operators/bigquery_to_gcs.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.contrib.operators.bigquery_to_gcs</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/contrib/operators/gcs_download_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/gcs_download_operator.html b/_modules/airflow/contrib/operators/gcs_download_operator.html
index 4af6020..dbf56fb 100644
--- a/_modules/airflow/contrib/operators/gcs_download_operator.html
+++ b/_modules/airflow/contrib/operators/gcs_download_operator.html
@@ -137,7 +137,22 @@
<div itemprop="articleBody">
<h1>Source code for airflow.contrib.operators.gcs_download_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">logging</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">from</span> <span class="nn">airflow.contrib.hooks.gcs_hook</span> <span class="kn">import</span> <span class="n">GoogleCloudStorageHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
@@ -147,7 +162,7 @@
<span class="sd">"""</span>
<span class="sd"> Downloads a file from Google Cloud Storage.</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span><span class="s1">'object'</span><span class="p">,</span><span class="s1">'filename'</span><span class="p">,)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'bucket'</span><span class="p">,</span><span class="s1">'object'</span><span class="p">,</span><span class="s1">'filename'</span><span class="p">,</span><span class="s1">'store_to_xcom_key'</span><span class="p">,)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f0eee4'</span>
@@ -156,7 +171,8 @@
<span class="bp">self</span><span class="p">,</span>
<span class="n">bucket</span><span class="p">,</span>
<span class="nb">object</span><span class="p">,</span>
- <span class="n">filename</span><span class="p">,</span>
+ <span class="n">filename</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">store_to_xcom_key</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
<span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="s1">'google_cloud_storage_default'</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span>
@@ -171,7 +187,13 @@
<span class="sd"> :type object: string</span>
<span class="sd"> :param filename: The file path on the local file system (where the</span>
<span class="sd"> operator is being executed) that the file should be downloaded to.</span>
+<span class="sd"> If false, the downloaded data will not be stored on the local file</span>
+<span class="sd"> system.</span>
<span class="sd"> :type filename: string</span>
+<span class="sd"> :param store_to_xcom_key: If this param is set, the operator will push</span>
+<span class="sd"> the contents of the downloaded file to XCom with the key set in this</span>
+<span class="sd"> parameter. If false, the downloaded data will not be pushed to XCom.</span>
+<span class="sd"> :type store_to_xcom_key: string</span>
<span class="sd"> :param google_cloud_storage_conn_id: The connection ID to use when</span>
<span class="sd"> connecting to Google cloud storage.</span>
<span class="sd"> :type google_cloud_storage_conn_id: string</span>
@@ -183,6 +205,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
<span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
<span class="bp">self</span><span class="o">.</span><span class="n">filename</span> <span class="o">=</span> <span class="n">filename</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span> <span class="o">=</span> <span class="n">store_to_xcom_key</span>
<span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span> <span class="o">=</span> <span class="n">google_cloud_storage_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span> <span class="o">=</span> <span class="n">delegate_to</span>
@@ -190,7 +213,13 @@
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing download: </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
<span class="n">hook</span> <span class="o">=</span> <span class="n">GoogleCloudStorageHook</span><span class="p">(</span><span class="n">google_cloud_storage_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">google_cloud_storage_conn_id</span><span class="p">,</span>
<span class="n">delegate_to</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">delegate_to</span><span class="p">)</span>
- <span class="k">print</span><span class="p">(</span><span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">))</span></div>
+ <span class="n">file_bytes</span> <span class="o">=</span> <span class="n">hook</span><span class="o">.</span><span class="n">download</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">filename</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">getsizeof</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span> <span class="o"><</span> <span class="mi">48000</span><span class="p">:</span>
+ <span class="n">context</span><span class="p">[</span><span class="s1">'ti'</span><span class="p">]</span><span class="o">.</span><span class="n">xcom_push</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">store_to_xcom_key</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">file_bytes</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">'The size of the downloaded file is too large to push to XCom!'</span><span class="p">)</span>
+ <span class="k">print</span><span class="p">(</span><span class="n">file_bytes</span><span class="p">)</span></div>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/celery_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/celery_executor.html b/_modules/airflow/executors/celery_executor.html
index a231e28..9601c12 100644
--- a/_modules/airflow/executors/celery_executor.html
+++ b/_modules/airflow/executors/celery_executor.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.executors.celery_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 79c456d..1841cb4 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.executors.local_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">multiprocessing</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">multiprocessing</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/executors/sequential_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index a393b0f..ce84843 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.executors.sequential_executor</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 6b4e17f..78efd22 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.macros</h1><div class="highlight"><pre>
-<span></span><span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">import</span> <span class="nn">dateutil</span>
@@ -187,11 +201,29 @@
<span class="k">return</span> <span class="n">datetime</span><span class="o">.</span><span class="n">strptime</span><span class="p">(</span><span class="n">ds</span><span class="p">,</span> <span class="n">input_format</span><span class="p">)</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="n">output_format</span><span class="p">)</span></div>
-<div class="viewcode-block" id="integrate_plugins"><a class="viewcode-back" href="../../code.html#airflow.macros.integrate_plugins">[docs]</a><span class="k">def</span> <span class="nf">integrate_plugins</span><span class="p">():</span>
+<span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span>
<span class="sd">"""Integrate plugins to the context"""</span>
+ <span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span>
- <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
- <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span></div>
+ <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
+ <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+ <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+
+
+ <span class="c1">##########################################################</span>
+ <span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
+
+ <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AIRFLOW_USE_NEW_IMPORTS'</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span>
+ <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
+ <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
+ <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span>
+ <span class="n">_macro</span><span class="p">,</span>
+ <span class="s2">"Importing plugin macro '{i}' directly from "</span>
+ <span class="s2">"'airflow.macros' has been deprecated. Please "</span>
+ <span class="s2">"import from 'airflow.macros.[plugin_module]' "</span>
+ <span class="s2">"instead. Support for direct imports will be dropped "</span>
+ <span class="s2">"entirely in Airflow 2.0."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span>
</pre></div>
</div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/macros/hive.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index 43cd2a0..92000d3 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -139,7 +139,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.macros.hive</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">datetime</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
<div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
@@ -166,7 +180,7 @@
<span class="sd"> >>> max_partition('airflow.static_babynames_partitioned')</span>
<span class="sd"> '2015-01-01'</span>
<span class="sd"> '''</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
<span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
@@ -219,7 +233,7 @@
<span class="sd"> >>> closest_ds_partition(tbl, '2015-01-02')</span>
<span class="sd"> '2015-01-01'</span>
<span class="sd"> '''</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
<span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/models.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/models.html b/_modules/airflow/models.html
index aba90cd..d4ed8a6 100644
--- a/_modules/airflow/models.html
+++ b/_modules/airflow/models.html
@@ -735,33 +735,43 @@
<span class="n">descriptor</span><span class="o">=</span><span class="nb">property</span><span class="p">(</span><span class="n">cls</span><span class="o">.</span><span class="n">get_extra</span><span class="p">,</span> <span class="n">cls</span><span class="o">.</span><span class="n">set_extra</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">hooks</span>
- <span class="kn">from</span> <span class="nn">airflow.contrib</span> <span class="kn">import</span> <span class="n">hooks</span> <span class="k">as</span> <span class="n">contrib_hooks</span>
<span class="k">try</span><span class="p">:</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'mysql'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
+ <span class="k">return</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'google_cloud_platform'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.bigquery_hook</span> <span class="kn">import</span> <span class="n">BigQueryHook</span>
+ <span class="k">return</span> <span class="n">BigQueryHook</span><span class="p">(</span><span class="n">bigquery_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'postgres'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="kn">import</span> <span class="n">PostgresHook</span>
+ <span class="k">return</span> <span class="n">PostgresHook</span><span class="p">(</span><span class="n">postgres_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'hive_cli'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
+ <span class="k">return</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'presto'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="kn">import</span> <span class="n">PrestoHook</span>
+ <span class="k">return</span> <span class="n">PrestoHook</span><span class="p">(</span><span class="n">presto_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'hiveserver2'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
+ <span class="k">return</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'sqlite'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.sqlite_hook</span> <span class="kn">import</span> <span class="n">SqliteHook</span>
+ <span class="k">return</span> <span class="n">SqliteHook</span><span class="p">(</span><span class="n">sqlite_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'jdbc'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.jdbc_hook</span> <span class="kn">import</span> <span class="n">JdbcHook</span>
+ <span class="k">return</span> <span class="n">JdbcHook</span><span class="p">(</span><span class="n">jdbc_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'mssql'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.mssql_hook</span> <span class="kn">import</span> <span class="n">MsSqlHook</span>
+ <span class="k">return</span> <span class="n">MsSqlHook</span><span class="p">(</span><span class="n">mssql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'oracle'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">hooks</span><span class="o">.</span><span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.oracle_hook</span> <span class="kn">import</span> <span class="n">OracleHook</span>
+ <span class="k">return</span> <span class="n">OracleHook</span><span class="p">(</span><span class="n">oracle_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'vertica'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.vertica_hook</span> <span class="kn">import</span> <span class="n">VerticaHook</span>
+ <span class="k">return</span> <span class="n">VerticaHook</span><span class="p">(</span><span class="n">vertica_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">conn_type</span> <span class="o">==</span> <span class="s1">'cloudant'</span><span class="p">:</span>
- <span class="k">return</span> <span class="n">contrib_hooks</span><span class="o">.</span><span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
+ <span class="kn">from</span> <span class="nn">airflow.contrib.hooks.cloudant_hook</span> <span class="kn">import</span> <span class="n">CloudantHook</span>
+ <span class="k">return</span> <span class="n">CloudantHook</span><span class="p">(</span><span class="n">cloudant_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">conn_id</span><span class="p">)</span>
<span class="k">except</span><span class="p">:</span>
<span class="k">return</span> <span class="bp">None</span>
@@ -844,6 +854,7 @@
<span class="n">__table_args__</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">'ti_dag_state'</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
+ <span class="n">Index</span><span class="p">(</span><span class="s1">'ti_state'</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">'ti_state_lkp'</span><span class="p">,</span> <span class="n">dag_id</span><span class="p">,</span> <span class="n">task_id</span><span class="p">,</span> <span class="n">execution_date</span><span class="p">,</span> <span class="n">state</span><span class="p">),</span>
<span class="n">Index</span><span class="p">(</span><span class="s1">'ti_pool'</span><span class="p">,</span> <span class="n">pool</span><span class="p">,</span> <span class="n">state</span><span class="p">,</span> <span class="n">priority_weight</span><span class="p">),</span>
<span class="p">)</span>
@@ -1308,17 +1319,32 @@
<div class="viewcode-block" id="TaskInstance.run"><a class="viewcode-back" href="../../code.html#airflow.models.TaskInstance.run">[docs]</a> <span class="k">def</span> <span class="nf">run</span><span class="p">(</span>
<span class="bp">self</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
- <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="c1"># Doesn't check for deps, just runs</span>
- <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="c1"># Ignore depends_on_past but respect</span>
- <span class="c1"># other deps</span>
- <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="c1"># Disregards previous successes</span>
- <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="c1"># Don't run the task, act as if it succeeded</span>
- <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span> <span class="c1"># Doesn't record success or failure in the DB</span>
+ <span class="n">ignore_dependencies</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">ignore_depends_on_past</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">force</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">mark_success</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">test_mode</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
<span class="n">job_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">pool</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Runs the task instance.</span>
+
+<span class="sd"> :param verbose: whether to turn on more verbose loggin</span>
+<span class="sd"> :type verbose: boolean</span>
+<span class="sd"> :param ignore_dependencies: Doesn't check for deps, just runs</span>
+<span class="sd"> :type ignore_dependencies: boolean</span>
+<span class="sd"> :param ignore_depends_on_past: Ignore depends_on_past but respect</span>
+<span class="sd"> other dependencies</span>
+<span class="sd"> :type ignore_depends_on_past: boolean</span>
+<span class="sd"> :param force: Forces a run regarless of previous success</span>
+<span class="sd"> :type force: boolean</span>
+<span class="sd"> :param mark_success: Don't run the task, mark its state as success</span>
+<span class="sd"> :type mark_success: boolean</span>
+<span class="sd"> :param test_mode: Doesn't record success or failure in the DB</span>
+<span class="sd"> :type test_mode: boolean</span>
+<span class="sd"> :param pool: specifies the pool to use to run the task instance</span>
+<span class="sd"> :type pool: str</span>
<span class="sd"> """</span>
<span class="n">task</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span>
<span class="bp">self</span><span class="o">.</span><span class="n">pool</span> <span class="o">=</span> <span class="n">pool</span> <span class="ow">or</span> <span class="n">task</span><span class="o">.</span><span class="n">pool</span>
@@ -1560,6 +1586,24 @@
<span class="k">if</span> <span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">:</span>
<span class="n">params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">task</span><span class="o">.</span><span class="n">params</span><span class="p">)</span>
+ <span class="k">class</span> <span class="nc">VariableAccessor</span><span class="p">:</span>
+ <span class="sd">"""</span>
+<span class="sd"> Wrapper around Variable. This way you can get variables in templates by using</span>
+<span class="sd"> {var.variable_name}.</span>
+<span class="sd"> """</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">)</span>
+
+ <span class="k">class</span> <span class="nc">VariableJsonAccessor</span><span class="p">:</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">pass</span>
+
+ <span class="k">def</span> <span class="nf">__getattr__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">item</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">item</span><span class="p">,</span> <span class="n">deserialize_json</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+
<span class="k">return</span> <span class="p">{</span>
<span class="s1">'dag'</span><span class="p">:</span> <span class="n">task</span><span class="o">.</span><span class="n">dag</span><span class="p">,</span>
<span class="s1">'ds'</span><span class="p">:</span> <span class="n">ds</span><span class="p">,</span>
@@ -1585,6 +1629,10 @@
<span class="s1">'task_instance_key_str'</span><span class="p">:</span> <span class="n">ti_key_str</span><span class="p">,</span>
<span class="s1">'conf'</span><span class="p">:</span> <span class="n">configuration</span><span class="p">,</span>
<span class="s1">'test_mode'</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_mode</span><span class="p">,</span>
+ <span class="s1">'var'</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s1">'value'</span><span class="p">:</span> <span class="n">VariableAccessor</span><span class="p">(),</span>
+ <span class="s1">'json'</span><span class="p">:</span> <span class="n">VariableJsonAccessor</span><span class="p">()</span>
+ <span class="p">}</span>
<span class="p">}</span>
<span class="k">def</span> <span class="nf">render_templates</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -2706,6 +2754,21 @@
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span><span class="p">,</span> <span class="n">timedelta</span><span class="p">):</span>
<span class="k">return</span> <span class="n">dttm</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schedule_interval</span>
+<div class="viewcode-block" id="DAG.normalize_schedule"><a class="viewcode-back" href="../../code.html#airflow.models.DAG.normalize_schedule">[docs]</a> <span class="k">def</span> <span class="nf">normalize_schedule</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dttm</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Returns dttm + interval unless dttm is first interval then it returns dttm</span>
+<span class="sd"> """</span>
+ <span class="n">following</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">following_schedule</span><span class="p">(</span><span class="n">dttm</span><span class="p">)</span>
+
+ <span class="c1"># in case of @once</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">following</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">dttm</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">previous_schedule</span><span class="p">(</span><span class="n">following</span><span class="p">)</span> <span class="o">!=</span> <span class="n">dttm</span><span class="p">:</span>
+ <span class="k">return</span> <span class="n">following</span>
+
+ <span class="k">return</span> <span class="n">dttm</span></div>
+
<span class="nd">@property</span>
<span class="k">def</span> <span class="nf">tasks</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">task_dict</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
@@ -3485,10 +3548,12 @@
<span class="nd">@staticmethod</span>
<span class="nd">@provide_session</span>
- <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">find</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">run_id</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">execution_date</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">state</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">external_trigger</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">session</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Returns a set of dag runs for the given search criteria.</span>
+<span class="sd"> :param dag_id: the dag_id to find dag runs for</span>
+<span class="sd"> :type dag_id: integer, list</span>
<span class="sd"> :param run_id: defines the the run id for this dag run</span>
<span class="sd"> :type run_id: string</span>
<span class="sd"> :param execution_date: the execution date</span>
@@ -3502,7 +3567,9 @@
<span class="sd"> """</span>
<span class="n">DR</span> <span class="o">=</span> <span class="n">DagRun</span>
- <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
+ <span class="n">qry</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">DR</span><span class="p">)</span>
+ <span class="k">if</span> <span class="n">dag_id</span><span class="p">:</span>
+ <span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">dag_id</span> <span class="o">==</span> <span class="n">dag_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">run_id</span><span class="p">:</span>
<span class="n">qry</span> <span class="o">=</span> <span class="n">qry</span><span class="o">.</span><span class="n">filter</span><span class="p">(</span><span class="n">DR</span><span class="o">.</span><span class="n">run_id</span> <span class="o">==</span> <span class="n">run_id</span><span class="p">)</span>
<span class="k">if</span> <span class="n">execution_date</span><span class="p">:</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/457b698d/_modules/airflow/operators/docker_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
index 46fcf7b..06aa65e 100644
--- a/_modules/airflow/operators/docker_operator.html
+++ b/_modules/airflow/operators/docker_operator.html
@@ -137,7 +137,21 @@
<div itemprop="articleBody">
<h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
-<span></span><span class="kn">import</span> <span class="nn">json</span>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
+<span class="c1"># you may not use this file except in compliance with the License.</span>
+<span class="c1"># You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>