You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/08/29 20:24:50 UTC
[22/48] incubator-airflow-site git commit: 1.10.0 with Updated Api
Reference
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/presto_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/presto_to_mysql.html b/_modules/airflow/operators/presto_to_mysql.html
index 0f27e92..7e86afb 100644
--- a/_modules/airflow/operators/presto_to_mysql.html
+++ b/_modules/airflow/operators/presto_to_mysql.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,15 +168,16 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
+
<span class="kn">from</span> <span class="nn">airflow.hooks.presto_hook</span> <span class="k">import</span> <span class="n">PrestoHook</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -199,19 +190,19 @@
<span class="sd"> into memory before being pushed to MySQL, so this operator should</span>
<span class="sd"> be used for smallish amount of data.</span>
-<span class="sd"> :param sql: SQL query to execute against Presto</span>
+<span class="sd"> :param sql: SQL query to execute against Presto. (templated)</span>
<span class="sd"> :type sql: str</span>
<span class="sd"> :param mysql_table: target MySQL table, use dot notation to target a</span>
-<span class="sd"> specific database</span>
+<span class="sd"> specific database. (templated)</span>
<span class="sd"> :type mysql_table: str</span>
<span class="sd"> :param mysql_conn_id: source mysql connection</span>
<span class="sd"> :type mysql_conn_id: str</span>
<span class="sd"> :param presto_conn_id: source presto connection</span>
<span class="sd"> :type presto_conn_id: str</span>
<span class="sd"> :param mysql_preoperator: sql statement to run against mysql prior to</span>
-<span class="sd"> import, typically use to truncate of delete in place of the data</span>
-<span class="sd"> coming in, allowing the task to be idempotent (running the task</span>
-<span class="sd"> twice won't double load data)</span>
+<span class="sd"> import, typically use to truncate of delete in place</span>
+<span class="sd"> of the data coming in, allowing the task to be idempotent (running</span>
+<span class="sd"> the task twice won't double load data). (templated)</span>
<span class="sd"> :type mysql_preoperator: str</span>
<span class="sd"> """</span>
@@ -251,9 +242,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -265,7 +254,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -284,6 +273,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -296,19 +286,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/python_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/python_operator.html b/_modules/airflow/operators/python_operator.html
index fb1e29d..d5ff59b 100644
--- a/_modules/airflow/operators/python_operator.html
+++ b/_modules/airflow/operators/python_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -226,7 +216,7 @@
<span class="sd"> :param templates_dict: a dictionary where the values are templates that</span>
<span class="sd"> will get templated by the Airflow engine sometime between</span>
<span class="sd"> ``__init__`` and ``execute`` takes place and are made available</span>
-<span class="sd"> in your callable's context after the template has been applied</span>
+<span class="sd"> in your callable's context after the template has been applied. (templated)</span>
<span class="sd"> :type templates_dict: dict of str</span>
<span class="sd"> :param templates_exts: a list of file extensions to resolve while</span>
<span class="sd"> processing templated fields, for examples ``['.sql', '.hql']``</span>
@@ -335,14 +325,16 @@
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
+
<div class="viewcode-block" id="PythonVirtualenvOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.python_operator.PythonVirtualenvOperator">[docs]</a><span class="k">class</span> <span class="nc">PythonVirtualenvOperator</span><span class="p">(</span><span class="n">PythonOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Allows one to run a function in a virtualenv that is created and destroyed</span>
<span class="sd"> automatically (with certain caveats).</span>
-<span class="sd"> The function must be defined using def, and not be part of a class. All imports</span>
-<span class="sd"> must happen inside the function and no variables outside of the scope may be referenced.</span>
-<span class="sd"> A global scope variable named virtualenv_string_args will be available (populated by</span>
+<span class="sd"> The function must be defined using def, and not be</span>
+<span class="sd"> part of a class. All imports must happen inside the function</span>
+<span class="sd"> and no variables outside of the scope may be referenced. A global scope</span>
+<span class="sd"> variable named virtualenv_string_args will be available (populated by</span>
<span class="sd"> string_args). In addition, one can pass stuff through op_args and op_kwargs, and one</span>
<span class="sd"> can use a return value.</span>
@@ -357,10 +349,12 @@
<span class="sd"> :param python_version: The Python version to run the virtualenv with. Note that</span>
<span class="sd"> both 2 and 2.7 are acceptable forms.</span>
<span class="sd"> :type python_version: str</span>
-<span class="sd"> :param use_dill: Whether to use dill to serialize the args and result (pickle is default).</span>
-<span class="sd"> This allow more complex types but requires you to include dill in your requirements.</span>
+<span class="sd"> :param use_dill: Whether to use dill to serialize</span>
+<span class="sd"> the args and result (pickle is default). This allow more complex types</span>
+<span class="sd"> but requires you to include dill in your requirements.</span>
<span class="sd"> :type use_dill: bool</span>
-<span class="sd"> :param system_site_packages: Whether to include system_site_packages in your virtualenv.</span>
+<span class="sd"> :param system_site_packages: Whether to include</span>
+<span class="sd"> system_site_packages in your virtualenv.</span>
<span class="sd"> See virtualenv documentation for more information.</span>
<span class="sd"> :type system_site_packages: bool</span>
<span class="sd"> :param op_args: A list of positional arguments to pass to python_callable.</span>
@@ -380,8 +374,11 @@
<span class="sd"> processing templated fields, for examples ``['.sql', '.hql']``</span>
<span class="sd"> :type templates_exts: list(str)</span>
<span class="sd"> """</span>
- <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">python_callable</span><span class="p">,</span> <span class="n">requirements</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">python_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_dill</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
- <span class="n">system_site_packages</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">op_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">op_kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">string_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">python_callable</span><span class="p">,</span>
+ <span class="n">requirements</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">python_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">use_dill</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">system_site_packages</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">op_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">op_kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">string_args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">templates_dict</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">templates_exts</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">PythonVirtualenvOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
<span class="n">python_callable</span><span class="o">=</span><span class="n">python_callable</span><span class="p">,</span>
@@ -398,22 +395,25 @@
<span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span> <span class="o">=</span> <span class="n">use_dill</span>
<span class="bp">self</span><span class="o">.</span><span class="n">system_site_packages</span> <span class="o">=</span> <span class="n">system_site_packages</span>
<span class="c1"># check that dill is present if needed</span>
- <span class="n">dill_in_requirements</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'dill'</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span><span class="p">)</span>
+ <span class="n">dill_in_requirements</span> <span class="o">=</span> <span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'dill'</span><span class="p">),</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span><span class="p">)</span>
<span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="n">system_site_packages</span><span class="p">)</span> <span class="ow">and</span> <span class="n">use_dill</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">any</span><span class="p">(</span><span class="n">dill_in_requirements</span><span class="p">):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'If using dill, dill must be in the environment '</span> <span class="o">+</span>
<span class="s1">'either via system_site_packages or requirements'</span><span class="p">)</span>
<span class="c1"># check that a function is passed, and that it is not a lambda</span>
- <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">)</span>
- <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span> <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">):</span>
+ <span class="k">if</span> <span class="p">(</span><span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">,</span>
+ <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">==</span>
+ <span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="mi">0</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'</span><span class="si">{}</span><span class="s1"> only supports functions for python_callable arg'</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
<span class="c1"># check that args are passed iff python major version matches</span>
- <span class="k">if</span> <span class="p">(</span><span class="n">python_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
- <span class="ow">and</span> <span class="nb">str</span><span class="p">(</span><span class="n">python_version</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="nb">str</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
- <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">()):</span>
+ <span class="k">if</span> <span class="p">(</span><span class="n">python_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
+ <span class="nb">str</span><span class="p">(</span><span class="n">python_version</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span> <span class="o">!=</span> <span class="nb">str</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="ow">and</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">()):</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"Passing op_args or op_kwargs is not supported across "</span>
<span class="s2">"different Python major versions "</span>
- <span class="s2">"for PythonVirtualenvOperator. Please use string_args."</span><span class="p">)</span>
+ <span class="s2">"for PythonVirtualenvOperator. "</span>
+ <span class="s2">"Please use string_args."</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">execute_callable</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'venv'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
@@ -485,8 +485,9 @@
<span class="k">else</span><span class="p">:</span>
<span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
<span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Error deserializing result. Note that result deserialization "</span>
- <span class="s2">"is not supported across major Python versions."</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"Error deserializing result. "</span>
+ <span class="s2">"Note that result deserialization "</span>
+ <span class="s2">"is not supported across major Python versions."</span><span class="p">)</span>
<span class="k">raise</span>
<span class="k">def</span> <span class="nf">_write_script</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">script_filename</span><span class="p">):</span>
@@ -511,9 +512,11 @@
<span class="n">cmd</span> <span class="o">=</span> <span class="p">[</span><span class="s1">'</span><span class="si">{}</span><span class="s1">/bin/pip'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">),</span> <span class="s1">'install'</span><span class="p">]</span>
<span class="k">return</span> <span class="n">cmd</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">requirements</span>
- <span class="k">def</span> <span class="nf">_generate_python_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">,</span> <span class="n">script_filename</span><span class="p">,</span> <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">):</span>
+ <span class="k">def</span> <span class="nf">_generate_python_cmd</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tmp_dir</span><span class="p">,</span> <span class="n">script_filename</span><span class="p">,</span>
+ <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">):</span>
<span class="c1"># direct path alleviates need to activate</span>
- <span class="k">return</span> <span class="p">[</span><span class="s1">'</span><span class="si">{}</span><span class="s1">/bin/python'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">),</span> <span class="n">script_filename</span><span class="p">,</span> <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">]</span>
+ <span class="k">return</span> <span class="p">[</span><span class="s1">'</span><span class="si">{}</span><span class="s1">/bin/python'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">tmp_dir</span><span class="p">),</span> <span class="n">script_filename</span><span class="p">,</span>
+ <span class="n">input_filename</span><span class="p">,</span> <span class="n">output_filename</span><span class="p">,</span> <span class="n">string_args_filename</span><span class="p">]</span>
<span class="k">def</span> <span class="nf">_generate_python_code</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_dill</span><span class="p">:</span>
@@ -523,11 +526,13 @@
<span class="n">fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span>
<span class="c1"># dont try to read pickle if we didnt pass anything</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pass_op_args</span><span class="p">():</span>
- <span class="n">load_args_line</span> <span class="o">=</span> <span class="s1">'with open(sys.argv[1], "rb") as f: arg_dict = </span><span class="si">{}</span><span class="s1">.load(f)'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pickling_library</span><span class="p">)</span>
+ <span class="n">load_args_line</span> <span class="o">=</span> <span class="s1">'with open(sys.argv[1], "rb") as f: arg_dict = </span><span class="si">{}</span><span class="s1">.load(f)'</span>\
+ <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pickling_library</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">load_args_line</span> <span class="o">=</span> <span class="s1">'arg_dict = {"args": [], "kwargs": </span><span class="si">{}</span><span class="s1">}'</span>
- <span class="c1"># no indents in original code so we can accept any type of indents in the original function</span>
+ <span class="c1"># no indents in original code so we can accept</span>
+ <span class="c1"># any type of indents in the original function</span>
<span class="c1"># we deserialize args, call function, serialize result if necessary</span>
<span class="k">return</span> <span class="n">dedent</span><span class="p">(</span><span class="s2">"""</span><span class="se">\</span>
<span class="s2"> import </span><span class="si">{pickling_library}</span><span class="s2"></span>
@@ -535,24 +540,22 @@
<span class="s2"> </span><span class="si">{load_args_code}</span><span class="s2"></span>
<span class="s2"> args = arg_dict["args"]</span>
<span class="s2"> kwargs = arg_dict["kwargs"]</span>
-<span class="s2"> with open(sys.argv[3], 'r') as f: virtualenv_string_args = list(map(lambda x: x.strip(), list(f)))</span>
+<span class="s2"> with open(sys.argv[3], 'r') as f:</span>
+<span class="s2"> virtualenv_string_args = list(map(lambda x: x.strip(), list(f)))</span>
<span class="s2"> </span><span class="si">{python_callable_lines}</span><span class="s2"></span>
<span class="s2"> res = </span><span class="si">{python_callable_name}</span><span class="s2">(*args, **kwargs)</span>
-<span class="s2"> with open(sys.argv[2], 'wb') as f: res is not None and </span><span class="si">{pickling_library}</span><span class="s2">.dump(res, f)</span>
-<span class="s2"> """</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
- <span class="n">load_args_code</span><span class="o">=</span><span class="n">load_args_line</span><span class="p">,</span>
- <span class="n">python_callable_lines</span><span class="o">=</span><span class="n">dedent</span><span class="p">(</span><span class="n">inspect</span><span class="o">.</span><span class="n">getsource</span><span class="p">(</span><span class="n">fn</span><span class="p">)),</span>
- <span class="n">python_callable_name</span><span class="o">=</span><span class="n">fn</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span>
- <span class="n">pickling_library</span><span class="o">=</span><span class="n">pickling_library</span><span class="p">)</span>
+<span class="s2"> with open(sys.argv[2], 'wb') as f:</span>
+<span class="s2"> res is not None and </span><span class="si">{pickling_library}</span><span class="s2">.dump(res, f)</span>
+<span class="s2"> """</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">load_args_code</span><span class="o">=</span><span class="n">load_args_line</span><span class="p">,</span>
+ <span class="n">python_callable_lines</span><span class="o">=</span><span class="n">dedent</span><span class="p">(</span><span class="n">inspect</span><span class="o">.</span><span class="n">getsource</span><span class="p">(</span><span class="n">fn</span><span class="p">)),</span>
+ <span class="n">python_callable_name</span><span class="o">=</span><span class="n">fn</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span>
+ <span class="n">pickling_library</span><span class="o">=</span><span class="n">pickling_library</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
-
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -564,7 +567,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -583,6 +586,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -595,19 +599,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/redshift_to_s3_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/redshift_to_s3_operator.html b/_modules/airflow/operators/redshift_to_s3_operator.html
index a68f20e..729f0c9 100644
--- a/_modules/airflow/operators/redshift_to_s3_operator.html
+++ b/_modules/airflow/operators/redshift_to_s3_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,15 +168,16 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
<span class="c1"># KIND, either express or implied. See the License for the</span>
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
+
<span class="kn">from</span> <span class="nn">airflow.hooks.postgres_hook</span> <span class="k">import</span> <span class="n">PostgresHook</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.S3_hook</span> <span class="k">import</span> <span class="n">S3Hook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -310,9 +301,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -324,7 +313,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -343,6 +332,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -355,19 +345,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/s3_file_transform_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/s3_file_transform_operator.html b/_modules/airflow/operators/s3_file_transform_operator.html
index 366400e..f9b2a59 100644
--- a/_modules/airflow/operators/s3_file_transform_operator.html
+++ b/_modules/airflow/operators/s3_file_transform_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -214,11 +204,11 @@
<span class="sd"> S3 Select is also available to filter the source contents. Users can</span>
<span class="sd"> omit the transformation script if S3 Select expression is specified.</span>
-<span class="sd"> :param source_s3_key: The key to be retrieved from S3</span>
+<span class="sd"> :param source_s3_key: The key to be retrieved from S3. (templated)</span>
<span class="sd"> :type source_s3_key: str</span>
<span class="sd"> :param source_aws_conn_id: source s3 connection</span>
<span class="sd"> :type source_aws_conn_id: str</span>
-<span class="sd"> :param dest_s3_key: The key to be written from S3</span>
+<span class="sd"> :param dest_s3_key: The key to be written from S3. (templated)</span>
<span class="sd"> :type dest_s3_key: str</span>
<span class="sd"> :param dest_aws_conn_id: destination s3 connection</span>
<span class="sd"> :type dest_aws_conn_id: str</span>
@@ -311,9 +301,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -325,7 +313,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -344,6 +332,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -356,19 +345,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file