You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ma...@apache.org on 2017/03/06 17:10:23 UTC
[18/22] incubator-airflow-site git commit: Latest docs version as of
1.8.x
Latest docs version as of 1.8.x
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/commit/9c75ee9e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/tree/9c75ee9e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/diff/9c75ee9e
Branch: refs/heads/asf-site
Commit: 9c75ee9e46cc5ec459979dbfd2c911743c07fd63
Parents: 4af0850
Author: Maxime Beauchemin <ma...@apache.org>
Authored: Mon Mar 6 08:43:12 2017 -0800
Committer: Maxime Beauchemin <ma...@apache.org>
Committed: Mon Mar 6 08:43:12 2017 -0800
----------------------------------------------------------------------
.../contrib/executors/mesos_executor.html | 81 +-
.../contrib/operators/hipchat_operator.html | 72 +-
_modules/airflow/executors/local_executor.html | 73 +-
.../airflow/executors/sequential_executor.html | 59 +-
_modules/airflow/macros.html | 79 +-
_modules/airflow/macros/hive.html | 61 +-
_modules/airflow/models.html | 1993 ++++++++++--------
_modules/airflow/operators/sensors.html | 242 ++-
_modules/bash_operator.html | 77 +-
_modules/dagrun_operator.html | 67 +-
_modules/dbapi_hook.html | 130 +-
_modules/dummy_operator.html | 49 +-
_modules/email_operator.html | 61 +-
_modules/ftp_hook.html | 69 +-
_modules/generic_transfer.html | 57 +-
_modules/http_hook.html | 61 +-
_modules/http_operator.html | 71 +-
_modules/index.html | 74 +-
_modules/mysql_hook.html | 95 +-
_modules/mysql_operator.html | 61 +-
_modules/presto_check_operator.html | 60 +-
_modules/presto_hook.html | 63 +-
_modules/python_operator.html | 69 +-
_modules/sensors.html | 242 ++-
_modules/sqlite_hook.html | 49 +-
_modules/ssh_execute_operator.html | 75 +-
_modules/ssh_hook.html | 81 +-
_static/basic.css | 68 +-
_static/comment-bright.png | Bin 3500 -> 756 bytes
_static/comment-close.png | Bin 3578 -> 829 bytes
_static/comment.png | Bin 3445 -> 641 bytes
_static/css/badge_only.css | 2 +-
_static/css/theme.css | 4 +-
_static/down-pressed.png | Bin 347 -> 222 bytes
_static/down.png | Bin 347 -> 202 bytes
_static/file.png | Bin 358 -> 286 bytes
_static/fonts/Inconsolata-Bold.ttf | Bin 66352 -> 109948 bytes
_static/fonts/Inconsolata-Regular.ttf | Bin 84548 -> 96964 bytes
_static/fonts/Lato-Bold.ttf | Bin 121788 -> 656544 bytes
_static/fonts/Lato-Regular.ttf | Bin 120196 -> 656568 bytes
_static/fonts/fontawesome-webfont.eot | Bin 56006 -> 76518 bytes
_static/fonts/fontawesome-webfont.svg | 207 +-
_static/fonts/fontawesome-webfont.ttf | Bin 112160 -> 152796 bytes
_static/fonts/fontawesome-webfont.woff | Bin 65452 -> 90412 bytes
_static/jquery.js | 8 +-
_static/js/theme.js | 58 +-
_static/minus.png | Bin 173 -> 90 bytes
_static/plus.png | Bin 173 -> 90 bytes
_static/searchtools.js | 115 +-
_static/up-pressed.png | Bin 345 -> 214 bytes
_static/up.png | Bin 345 -> 203 bytes
cli.html | 482 +++--
code.html | 1933 +++--------------
concepts.html | 129 +-
configuration.html | 97 +-
faq.html | 61 +-
genindex.html | 1541 ++++----------
index.html | 115 +-
installation.html | 51 +-
license.html | 51 +-
objects.inv | Bin 2326 -> 2147 bytes
plugins.html | 64 +-
profiling.html | 51 +-
project.html | 76 +-
py-modindex.html | 63 +-
scheduler.html | 79 +-
search.html | 41 +-
searchindex.js | 2 +-
security.html | 147 +-
start.html | 51 +-
tutorial.html | 51 +-
ui.html | 51 +-
72 files changed, 5097 insertions(+), 4672 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/contrib/executors/mesos_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/executors/mesos_executor.html b/_modules/airflow/contrib/executors/mesos_executor.html
index e2d8f61..dbca37d 100644
--- a/_modules/airflow/contrib/executors/mesos_executor.html
+++ b/_modules/airflow/contrib/executors/mesos_executor.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
<link rel="up" title="Module code" href="../../../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
<li><a href="../../../index.html">Module code</a> »</li>
-
- <li>airflow.contrib.executors.mesos_executor</li>
+
+ <li>airflow.contrib.executors.mesos_executor</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,21 +176,21 @@
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">from</span> <span class="nn">future</span> <span class="kn">import</span> <span class="n">standard_library</span>
+<span class="kn">from</span> <span class="nn">future</span> <span class="k">import</span> <span class="n">standard_library</span>
<span class="n">standard_library</span><span class="o">.</span><span class="n">install_aliases</span><span class="p">()</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
<span class="kn">import</span> <span class="nn">logging</span>
-<span class="kn">from</span> <span class="nn">queue</span> <span class="kn">import</span> <span class="n">Queue</span>
+<span class="kn">from</span> <span class="nn">queue</span> <span class="k">import</span> <span class="n">Queue</span>
<span class="kn">import</span> <span class="nn">mesos.interface</span>
-<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="kn">import</span> <span class="n">mesos_pb2</span>
+<span class="kn">from</span> <span class="nn">mesos.interface</span> <span class="k">import</span> <span class="n">mesos_pb2</span>
<span class="kn">import</span> <span class="nn">mesos.native</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="kn">import</span> <span class="n">Session</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.settings</span> <span class="k">import</span> <span class="n">Session</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="n">DEFAULT_FRAMEWORK_NAME</span> <span class="o">=</span> <span class="s1">'Airflow'</span>
@@ -206,13 +231,13 @@
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">)</span> <span class="ow">and</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span>
+ <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
<span class="c1"># Update the Framework ID in the database.</span>
<span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
<span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">get_framework_name</span><span class="p">()</span>
<span class="n">connection</span> <span class="o">=</span> <span class="n">Session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">connection</span> <span class="o">=</span> <span class="n">Connection</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">,</span> <span class="n">conn_type</span><span class="o">=</span><span class="s1">'mesos_framework-id'</span><span class="p">,</span>
<span class="n">extra</span><span class="o">=</span><span class="n">frameworkId</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
@@ -286,7 +311,7 @@
<span class="n">mem</span><span class="o">.</span><span class="n">scalar</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_mem</span>
<span class="n">command</span> <span class="o">=</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">CommandInfo</span><span class="p">()</span>
- <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">command</span><span class="o">.</span><span class="n">shell</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">command</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">cmd</span>
<span class="n">task</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">MergeFrom</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
@@ -306,7 +331,7 @@
<span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
<span class="c1"># The map may not contain an item if the framework re-registered after a failover.</span>
<span class="c1"># Discard these tasks.</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+ <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">"Unrecognised task key </span><span class="si">%s</span><span class="s2">"</span> <span class="o">%</span> <span class="n">update</span><span class="o">.</span><span class="n">task_id</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<span class="k">return</span>
<span class="k">if</span> <span class="n">update</span><span class="o">.</span><span class="n">state</span> <span class="o">==</span> <span class="n">mesos_pb2</span><span class="o">.</span><span class="n">TASK_FINISHED</span><span class="p">:</span>
@@ -357,23 +382,23 @@
<span class="n">task_memory</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'TASK_MEMORY'</span><span class="p">)</span>
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getboolean</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'CHECKPOINT'</span><span class="p">):</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">if</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">):</span>
<span class="c1"># Import here to work around a circular import error</span>
- <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">Connection</span>
+ <span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">Connection</span>
<span class="c1"># Query the database to get the ID of the Mesos Framework, if available.</span>
<span class="n">conn_id</span> <span class="o">=</span> <span class="n">FRAMEWORK_CONNID_PREFIX</span> <span class="o">+</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span>
<span class="n">session</span> <span class="o">=</span> <span class="n">Session</span><span class="p">()</span>
<span class="n">connection</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">Connection</span><span class="p">)</span><span class="o">.</span><span class="n">filter_by</span><span class="p">(</span><span class="n">conn_id</span><span class="o">=</span><span class="n">conn_id</span><span class="p">)</span><span class="o">.</span><span class="n">first</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">connection</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># Set the Framework ID to let the scheduler reconnect with running tasks.</span>
<span class="n">framework</span><span class="o">.</span><span class="n">id</span><span class="o">.</span><span class="n">value</span> <span class="o">=</span> <span class="n">connection</span><span class="o">.</span><span class="n">extra</span>
<span class="n">framework</span><span class="o">.</span><span class="n">failover_timeout</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">getint</span><span class="p">(</span><span class="s1">'mesos'</span><span class="p">,</span> <span class="s1">'FAILOVER_TIMEOUT'</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
- <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="bp">False</span>
+ <span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span> <span class="o">=</span> <span class="kc">False</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'MesosFramework master : </span><span class="si">%s</span><span class="s1">, name : </span><span class="si">%s</span><span class="s1">, cpu : </span><span class="si">%s</span><span class="s1">, mem : </span><span class="si">%s</span><span class="s1">, checkpoint : </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span>
<span class="n">master</span><span class="p">,</span> <span class="n">framework</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_cpu</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">task_memory</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">framework</span><span class="o">.</span><span class="n">checkpoint</span><span class="p">))</span>
@@ -411,7 +436,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span> <span class="o">=</span> <span class="n">driver</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mesos_driver</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -425,6 +450,9 @@
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -457,7 +485,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/contrib/operators/hipchat_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/contrib/operators/hipchat_operator.html b/_modules/airflow/contrib/operators/hipchat_operator.html
index 1505a9b..194afad 100644
--- a/_modules/airflow/contrib/operators/hipchat_operator.html
+++ b/_modules/airflow/contrib/operators/hipchat_operator.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../../index.html"/>
<link rel="up" title="Module code" href="../../../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../../index.html">Docs</a> »</li>
+
<li><a href="../../../index.html">Module code</a> »</li>
-
- <li>airflow.contrib.operators.hipchat_operator</li>
+
+ <li>airflow.contrib.operators.hipchat_operator</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,9 +176,11 @@
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="kn">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
+
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">requests</span>
<span class="kn">import</span> <span class="nn">json</span>
@@ -166,7 +193,6 @@
<span class="sd"> at https://www.hipchat.com/docs/apiv2. Before using any HipChat API operators you need</span>
<span class="sd"> to get an authentication token at https://www.hipchat.com/docs/apiv2/auth.</span>
<span class="sd"> In the future additional HipChat operators will be derived from this class as well.</span>
-
<span class="sd"> :param token: HipChat REST API authentication token</span>
<span class="sd"> :type token: str</span>
<span class="sd"> :param base_url: HipChat REST API base url.</span>
@@ -181,15 +207,14 @@
<span class="nb">super</span><span class="p">(</span><span class="n">HipChatAPIOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">token</span> <span class="o">=</span> <span class="n">token</span>
<span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">base_url</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="bp">None</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="bp">None</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="bp">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Used by the execute function. Set the request method, url, and body of HipChat's</span>
<span class="sd"> REST API call.</span>
-
<span class="sd"> Override in child class. Each HipChatAPI child operator is responsible for having</span>
<span class="sd"> a prepare_request method call which sets self.method, self.url, and self.body.</span>
<span class="sd"> """</span>
@@ -215,7 +240,6 @@
<span class="sd">"""</span>
<span class="sd"> Send notification to a specific HipChat room.</span>
<span class="sd"> More info: https://www.hipchat.com/docs/apiv2/method/send_room_notification</span>
-
<span class="sd"> :param room_id: Room in which to send notification on HipChat</span>
<span class="sd"> :type room_id: str</span>
<span class="sd"> :param message: The message body</span>
@@ -245,11 +269,11 @@
<span class="s1">'message_format'</span><span class="p">:</span> <span class="s1">'html'</span><span class="p">,</span>
<span class="s1">'color'</span><span class="p">:</span> <span class="s1">'yellow'</span><span class="p">,</span>
<span class="s1">'frm'</span><span class="p">:</span> <span class="s1">'airflow'</span><span class="p">,</span>
- <span class="s1">'attach_to'</span><span class="p">:</span> <span class="bp">None</span><span class="p">,</span>
- <span class="s1">'notify'</span><span class="p">:</span> <span class="bp">False</span><span class="p">,</span>
- <span class="s1">'card'</span><span class="p">:</span> <span class="bp">None</span>
+ <span class="s1">'attach_to'</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
+ <span class="s1">'notify'</span><span class="p">:</span> <span class="kc">False</span><span class="p">,</span>
+ <span class="s1">'card'</span><span class="p">:</span> <span class="kc">None</span>
<span class="p">}</span>
- <span class="k">for</span> <span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="ow">in</span> <span class="n">default_options</span><span class="o">.</span><span class="n">iteritems</span><span class="p">():</span>
+ <span class="k">for</span> <span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">)</span> <span class="ow">in</span> <span class="n">default_options</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="nb">setattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prop</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">prop</span><span class="p">,</span> <span class="n">default</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">prepare_request</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -266,10 +290,13 @@
<span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="s1">'POST'</span>
<span class="bp">self</span><span class="o">.</span><span class="n">url</span> <span class="o">=</span> <span class="s1">'</span><span class="si">%s</span><span class="s1">/room/</span><span class="si">%s</span><span class="s1">/notification'</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">room_id</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">body</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="nb">dict</span><span class="p">(</span>
- <span class="p">(</span><span class="n">k</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">),</span> <span class="n">v</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">iteritems</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">))</span></div>
+ <span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">k</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">v</span><span class="p">))</span></div>
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -302,7 +329,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/executors/local_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/local_executor.html b/_modules/airflow/executors/local_executor.html
index 250eafd..2e35470 100644
--- a/_modules/airflow/executors/local_executor.html
+++ b/_modules/airflow/executors/local_executor.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../index.html"/>
<link rel="up" title="Module code" href="../../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
<li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.executors.local_executor</li>
+
+ <li>airflow.executors.local_executor</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -155,12 +180,12 @@
<span class="kn">import</span> <span class="nn">subprocess</span>
<span class="kn">import</span> <span class="nn">time</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">range</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">range</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="kn">import</span> <span class="n">configuration</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="kn">import</span> <span class="n">LoggingMixin</span>
+<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.logging</span> <span class="k">import</span> <span class="n">LoggingMixin</span>
<span class="n">PARALLELISM</span> <span class="o">=</span> <span class="n">configuration</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="s1">'PARALLELISM'</span><span class="p">)</span>
@@ -171,24 +196,24 @@
<span class="n">multiprocessing</span><span class="o">.</span><span class="n">Process</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span> <span class="o">=</span> <span class="n">task_queue</span>
<span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span> <span class="o">=</span> <span class="n">result_queue</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="bp">True</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">daemon</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">def</span> <span class="nf">run</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
- <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
+ <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
<span class="n">key</span><span class="p">,</span> <span class="n">command</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
- <span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">key</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="c1"># Received poison pill, no more tasks to run</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
<span class="k">break</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"{} running {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">{}</span><span class="s2"> running </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
<span class="bp">self</span><span class="o">.</span><span class="n">__class__</span><span class="o">.</span><span class="n">__name__</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
- <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '{0}'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
+ <span class="n">command</span> <span class="o">=</span> <span class="s2">"exec bash -c '</span><span class="si">{0}</span><span class="s2">'"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
- <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+ <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span>
<span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">state</span> <span class="o">=</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"failed to execute task {}:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"failed to execute task </span><span class="si">{}</span><span class="s2">:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
<span class="c1"># raise e</span>
<span class="bp">self</span><span class="o">.</span><span class="n">result_queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">state</span><span class="p">))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">task_queue</span><span class="o">.</span><span class="n">task_done</span><span class="p">()</span>
@@ -213,7 +238,7 @@
<span class="k">for</span> <span class="n">w</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
<span class="n">w</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
@@ -224,7 +249,7 @@
<span class="k">def</span> <span class="nf">end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="c1"># Sending poison pill to all worker</span>
<span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">workers</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="bp">None</span><span class="p">,</span> <span class="bp">None</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">put</span><span class="p">((</span><span class="kc">None</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
<span class="c1"># Wait for commands to finish</span>
<span class="bp">self</span><span class="o">.</span><span class="n">queue</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
@@ -232,6 +257,9 @@
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -264,7 +292,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/executors/sequential_executor.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/executors/sequential_executor.html b/_modules/airflow/executors/sequential_executor.html
index ce84843..d5644c2 100644
--- a/_modules/airflow/executors/sequential_executor.html
+++ b/_modules/airflow/executors/sequential_executor.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../index.html"/>
<link rel="up" title="Module code" href="../../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
<li><a href="../../index.html">Module code</a> »</li>
-
- <li>airflow.executors.sequential_executor</li>
+
+ <li>airflow.executors.sequential_executor</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,11 +176,11 @@
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">str</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
<span class="kn">import</span> <span class="nn">subprocess</span>
-<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="kn">import</span> <span class="n">BaseExecutor</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="kn">import</span> <span class="n">State</span>
+<span class="kn">from</span> <span class="nn">airflow.executors.base_executor</span> <span class="k">import</span> <span class="n">BaseExecutor</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
<div class="viewcode-block" id="SequentialExecutor"><a class="viewcode-back" href="../../../code.html#airflow.executors.SequentialExecutor">[docs]</a><span class="k">class</span> <span class="nc">SequentialExecutor</span><span class="p">(</span><span class="n">BaseExecutor</span><span class="p">):</span>
@@ -171,19 +196,19 @@
<span class="nb">super</span><span class="p">(</span><span class="n">SequentialExecutor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span> <span class="o">=</span> <span class="p">[]</span>
- <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">execute_async</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,</span> <span class="n">queue</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">key</span><span class="p">,</span> <span class="n">command</span><span class="p">,))</span>
<span class="k">def</span> <span class="nf">sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">command</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Executing command: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">command</span><span class="p">))</span>
<span class="k">try</span><span class="p">:</span>
- <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
+ <span class="n">subprocess</span><span class="o">.</span><span class="n">check_call</span><span class="p">(</span><span class="n">command</span><span class="p">,</span> <span class="n">shell</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">SUCCESS</span><span class="p">)</span>
<span class="k">except</span> <span class="n">subprocess</span><span class="o">.</span><span class="n">CalledProcessError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">change_state</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="n">State</span><span class="o">.</span><span class="n">FAILED</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task {}:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Failed to execute task </span><span class="si">{}</span><span class="s2">:"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)))</span>
<span class="bp">self</span><span class="o">.</span><span class="n">commands_to_run</span> <span class="o">=</span> <span class="p">[]</span>
@@ -192,6 +217,9 @@
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -224,7 +252,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/macros.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros.html b/_modules/airflow/macros.html
index 19c18ff..34826d7 100644
--- a/_modules/airflow/macros.html
+++ b/_modules/airflow/macros.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../genindex.html"/>
+ <link rel="search" title="Search" href="../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../index.html"/>
<link rel="up" title="Module code" href="../index.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../index.html">Airflow</a>
+
</nav>
@@ -118,19 +126,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../index.html">Docs</a> »</li>
+
<li><a href="../index.html">Module code</a> »</li>
-
- <li>airflow.macros</li>
+
+ <li>airflow.macros</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -151,12 +176,12 @@
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
-<span class="kn">from</span> <span class="nn">random</span> <span class="kn">import</span> <span class="n">random</span>
-<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="k">import</span> <span class="n">absolute_import</span>
+<span class="kn">from</span> <span class="nn">random</span> <span class="k">import</span> <span class="n">random</span>
+<span class="kn">from</span> <span class="nn">datetime</span> <span class="k">import</span> <span class="n">datetime</span><span class="p">,</span> <span class="n">timedelta</span>
<span class="kn">import</span> <span class="nn">dateutil</span>
<span class="kn">import</span> <span class="nn">time</span>
-<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">hive</span>
+<span class="kn">from</span> <span class="nn">.</span> <span class="k">import</span> <span class="n">hive</span>
<span class="kn">import</span> <span class="nn">uuid</span>
@@ -204,28 +229,33 @@
<span class="k">def</span> <span class="nf">_integrate_plugins</span><span class="p">():</span>
<span class="sd">"""Integrate plugins to the context"""</span>
<span class="kn">import</span> <span class="nn">sys</span>
- <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="kn">import</span> <span class="n">macros</span> <span class="k">as</span> <span class="n">_macros</span>
- <span class="k">for</span> <span class="n">_macro_module</span> <span class="ow">in</span> <span class="n">_macros</span><span class="p">:</span>
- <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
- <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro_module</span>
+ <span class="kn">from</span> <span class="nn">airflow.plugins_manager</span> <span class="k">import</span> <span class="n">macros_modules</span>
+ <span class="k">for</span> <span class="n">macros_module</span> <span class="ow">in</span> <span class="n">macros_modules</span><span class="p">:</span>
+ <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">macros_module</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">macros_module</span>
+ <span class="nb">globals</span><span class="p">()[</span><span class="n">macros_module</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">macros_module</span>
<span class="c1">##########################################################</span>
<span class="c1"># TODO FIXME Remove in Airflow 2.0</span>
- <span class="kn">import</span> <span class="nn">os</span> <span class="kn">as</span> <span class="nn">_os</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AIRFLOW_USE_NEW_IMPORTS'</span><span class="p">,</span> <span class="bp">False</span><span class="p">):</span>
- <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="kn">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
- <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">_macro_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
- <span class="nb">globals</span><span class="p">()[</span><span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span><span class="p">]</span> <span class="o">=</span> <span class="n">_deprecated</span><span class="p">(</span>
- <span class="n">_macro</span><span class="p">,</span>
- <span class="s2">"Importing plugin macro '{i}' directly from "</span>
+ <span class="kn">import</span> <span class="nn">os</span> <span class="k">as</span> <span class="nn">_os</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">_os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'AIRFLOW_USE_NEW_IMPORTS'</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+ <span class="kn">from</span> <span class="nn">zope.deprecation</span> <span class="k">import</span> <span class="n">deprecated</span> <span class="k">as</span> <span class="n">_deprecated</span>
+ <span class="k">for</span> <span class="n">_macro</span> <span class="ow">in</span> <span class="n">macros_module</span><span class="o">.</span><span class="n">_objects</span><span class="p">:</span>
+ <span class="n">macro_name</span> <span class="o">=</span> <span class="n">_macro</span><span class="o">.</span><span class="n">__name__</span>
+ <span class="nb">globals</span><span class="p">()[</span><span class="n">macro_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">_macro</span>
+ <span class="n">_deprecated</span><span class="p">(</span>
+ <span class="n">macro_name</span><span class="p">,</span>
+ <span class="s2">"Importing plugin macro '</span><span class="si">{i}</span><span class="s2">' directly from "</span>
<span class="s2">"'airflow.macros' has been deprecated. Please "</span>
<span class="s2">"import from 'airflow.macros.[plugin_module]' "</span>
<span class="s2">"instead. Support for direct imports will be dropped "</span>
- <span class="s2">"entirely in Airflow 2.0."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">_macro</span><span class="p">))</span>
+ <span class="s2">"entirely in Airflow 2.0."</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="n">macro_name</span><span class="p">))</span>
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -258,7 +288,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/9c75ee9e/_modules/airflow/macros/hive.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/macros/hive.html b/_modules/airflow/macros/hive.html
index 92000d3..8d13632 100644
--- a/_modules/airflow/macros/hive.html
+++ b/_modules/airflow/macros/hive.html
@@ -30,6 +30,9 @@
+ <link rel="index" title="Index"
+ href="../../../genindex.html"/>
+ <link rel="search" title="Search" href="../../../search.html"/>
<link rel="top" title="Airflow Documentation" href="../../../index.html"/>
<link rel="up" title="airflow.macros" href="../macros.html"/>
@@ -40,6 +43,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -90,6 +94,8 @@
<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
</ul>
@@ -104,8 +110,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../../../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
</nav>
@@ -118,21 +126,38 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../../../index.html">Docs</a> »</li>
-
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
<li><a href="../../index.html">Module code</a> »</li>
-
+
<li><a href="../macros.html">airflow.macros</a> »</li>
-
- <li>airflow.macros.hive</li>
+
+ <li>airflow.macros.hive</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -157,7 +182,7 @@
<div class="viewcode-block" id="max_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.max_partition">[docs]</a><span class="k">def</span> <span class="nf">max_partition</span><span class="p">(</span>
- <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">):</span>
<span class="sd">'''</span>
<span class="sd"> Gets the max partition for a table.</span>
@@ -180,7 +205,7 @@
<span class="sd"> >>> max_partition('airflow.static_babynames_partitioned')</span>
<span class="sd"> '2015-01-01'</span>
<span class="sd"> '''</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
<span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
@@ -188,7 +213,7 @@
<span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">,</span> <span class="n">field</span><span class="o">=</span><span class="n">field</span><span class="p">,</span> <span class="nb">filter</span><span class="o">=</span><span class="nb">filter</span><span class="p">)</span></div>
-<span class="k">def</span> <span class="nf">_closest_date</span><span class="p">(</span><span class="n">target_dt</span><span class="p">,</span> <span class="n">date_list</span><span class="p">,</span> <span class="n">before_target</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
+<span class="k">def</span> <span class="nf">_closest_date</span><span class="p">(</span><span class="n">target_dt</span><span class="p">,</span> <span class="n">date_list</span><span class="p">,</span> <span class="n">before_target</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="sd">'''</span>
<span class="sd"> This function finds the date in a list closest to the target date.</span>
<span class="sd"> An optional parameter can be given to get the closest before or after.</span>
@@ -205,7 +230,7 @@
<span class="n">fb</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span> <span class="k">if</span> <span class="n">d</span> <span class="o">>=</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="o">.</span><span class="n">max</span>
<span class="n">fa</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span> <span class="k">if</span> <span class="n">d</span> <span class="o"><=</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="o">.</span><span class="n">min</span>
<span class="n">fnone</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">target_dt</span> <span class="o">-</span> <span class="n">d</span> <span class="k">if</span> <span class="n">d</span> <span class="o"><</span> <span class="n">target_dt</span> <span class="k">else</span> <span class="n">d</span> <span class="o">-</span> <span class="n">target_dt</span>
- <span class="k">if</span> <span class="n">before_target</span> <span class="ow">is</span> <span class="bp">None</span><span class="p">:</span>
+ <span class="k">if</span> <span class="n">before_target</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">min</span><span class="p">(</span><span class="n">date_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="n">fnone</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
<span class="k">if</span> <span class="n">before_target</span><span class="p">:</span>
<span class="k">return</span> <span class="nb">min</span><span class="p">(</span><span class="n">date_list</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="n">fb</span><span class="p">)</span><span class="o">.</span><span class="n">date</span><span class="p">()</span>
@@ -214,7 +239,7 @@
<div class="viewcode-block" id="closest_ds_partition"><a class="viewcode-back" href="../../../code.html#airflow.macros.hive.closest_ds_partition">[docs]</a><span class="k">def</span> <span class="nf">closest_ds_partition</span><span class="p">(</span>
- <span class="n">table</span><span class="p">,</span> <span class="n">ds</span><span class="p">,</span> <span class="n">before</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
+ <span class="n">table</span><span class="p">,</span> <span class="n">ds</span><span class="p">,</span> <span class="n">before</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
<span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">):</span>
<span class="sd">'''</span>
<span class="sd"> This function finds the date in a list closest to the target date.</span>
@@ -233,13 +258,13 @@
<span class="sd"> >>> closest_ds_partition(tbl, '2015-01-02')</span>
<span class="sd"> '2015-01-01'</span>
<span class="sd"> '''</span>
- <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveMetastoreHook</span>
+ <span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveMetastoreHook</span>
<span class="k">if</span> <span class="s1">'.'</span> <span class="ow">in</span> <span class="n">table</span><span class="p">:</span>
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">hh</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="n">metastore_conn_id</span><span class="o">=</span><span class="n">metastore_conn_id</span><span class="p">)</span>
<span class="n">partitions</span> <span class="o">=</span> <span class="n">hh</span><span class="o">.</span><span class="n">get_partitions</span><span class="p">(</span><span class="n">schema</span><span class="o">=</span><span class="n">schema</span><span class="p">,</span> <span class="n">table_name</span><span class="o">=</span><span class="n">table</span><span class="p">)</span>
<span class="k">if</span> <span class="ow">not</span> <span class="n">partitions</span><span class="p">:</span>
- <span class="k">return</span> <span class="bp">None</span>
+ <span class="k">return</span> <span class="kc">None</span>
<span class="n">part_vals</span> <span class="o">=</span> <span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">p</span><span class="o">.</span><span class="n">values</span><span class="p">())[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">partitions</span><span class="p">]</span>
<span class="k">if</span> <span class="n">ds</span> <span class="ow">in</span> <span class="n">part_vals</span><span class="p">:</span>
<span class="k">return</span> <span class="n">ds</span>
@@ -252,6 +277,9 @@
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -284,7 +312,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../../../_static/jquery.js"></script>