You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/08/29 20:24:54 UTC
[26/48] incubator-airflow-site git commit: 1.10.0 with Updated Api
Reference
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/bash_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/bash_operator.html b/_modules/airflow/operators/bash_operator.html
index d6d7dc6..da3c883 100644
--- a/_modules/airflow/operators/bash_operator.html
+++ b/_modules/airflow/operators/bash_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -195,28 +185,18 @@
<span class="kn">from</span> <span class="nn">subprocess</span> <span class="k">import</span> <span class="n">Popen</span><span class="p">,</span> <span class="n">STDOUT</span><span class="p">,</span> <span class="n">PIPE</span>
<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">gettempdir</span><span class="p">,</span> <span class="n">NamedTemporaryFile</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">configuration</span> <span class="k">as</span> <span class="n">conf</span>
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
-<span class="c1"># These variables are required in cases when BashOperator tasks use airflow specific code,</span>
-<span class="c1"># e.g. they import packages in the airflow context and the possibility of impersonation</span>
-<span class="c1"># gives not guarantee that these variables are available in the impersonated environment.</span>
-<span class="c1"># Hence, we need to propagate them in the Bash script used as a wrapper of commands in</span>
-<span class="c1"># this BashOperator.</span>
-<span class="n">PYTHONPATH_VAR</span> <span class="o">=</span> <span class="s1">'PYTHONPATH'</span>
-<span class="n">AIRFLOW_HOME_VAR</span> <span class="o">=</span> <span class="s1">'AIRFLOW_HOME'</span>
-
-
<div class="viewcode-block" id="BashOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.bash_operator.BashOperator">[docs]</a><span class="k">class</span> <span class="nc">BashOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
<span class="sd">"""</span>
<span class="sd"> Execute a Bash script, command or set of commands.</span>
<span class="sd"> :param bash_command: The command, set of commands or reference to a</span>
-<span class="sd"> bash script (must be '.sh') to be executed.</span>
+<span class="sd"> bash script (must be '.sh') to be executed. (templated)</span>
<span class="sd"> :type bash_command: string</span>
<span class="sd"> :param xcom_push: If xcom_push is True, the last line written to stdout</span>
<span class="sd"> will also be pushed to an XCom when the bash command completes.</span>
@@ -254,18 +234,12 @@
<span class="sd"> """</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Tmp dir root location: </span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">gettempdir</span><span class="p">())</span>
- <span class="n">airflow_home_value</span> <span class="o">=</span> <span class="n">conf</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'core'</span><span class="p">,</span> <span class="n">AIRFLOW_HOME_VAR</span><span class="p">)</span>
- <span class="n">pythonpath_value</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">PYTHONPATH_VAR</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
-
- <span class="n">bash_command</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'export </span><span class="si">{}</span><span class="s1">=</span><span class="si">{}</span><span class="s1">; '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">AIRFLOW_HOME_VAR</span><span class="p">,</span> <span class="n">airflow_home_value</span><span class="p">)</span> <span class="o">+</span>
- <span class="s1">'export </span><span class="si">{}</span><span class="s1">=</span><span class="si">{}</span><span class="s1">; '</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">PYTHONPATH_VAR</span><span class="p">,</span> <span class="n">pythonpath_value</span><span class="p">)</span> <span class="o">+</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">lineage_data</span> <span class="o">=</span> <span class="n">bash_command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">lineage_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span>
<span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">tmp_dir</span><span class="p">:</span>
<span class="k">with</span> <span class="n">NamedTemporaryFile</span><span class="p">(</span><span class="nb">dir</span><span class="o">=</span><span class="n">tmp_dir</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_id</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
- <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="n">bash_command</span><span class="p">,</span> <span class="s1">'utf_8'</span><span class="p">))</span>
+ <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="nb">bytes</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span><span class="p">,</span> <span class="s1">'utf_8'</span><span class="p">))</span>
<span class="n">f</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
<span class="n">fname</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span>
<span class="n">script_location</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="n">fname</span><span class="p">)</span>
@@ -281,7 +255,7 @@
<span class="n">signal</span><span class="o">.</span><span class="n">signal</span><span class="p">(</span><span class="nb">getattr</span><span class="p">(</span><span class="n">signal</span><span class="p">,</span> <span class="n">sig</span><span class="p">),</span> <span class="n">signal</span><span class="o">.</span><span class="n">SIG_DFL</span><span class="p">)</span>
<span class="n">os</span><span class="o">.</span><span class="n">setsid</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">bash_command</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bash_command</span><span class="p">)</span>
<span class="n">sp</span> <span class="o">=</span> <span class="n">Popen</span><span class="p">(</span>
<span class="p">[</span><span class="s1">'bash'</span><span class="p">,</span> <span class="n">fname</span><span class="p">],</span>
<span class="n">stdout</span><span class="o">=</span><span class="n">PIPE</span><span class="p">,</span> <span class="n">stderr</span><span class="o">=</span><span class="n">STDOUT</span><span class="p">,</span>
@@ -313,9 +287,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -327,7 +299,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -346,6 +318,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -358,19 +331,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/check_operator.html b/_modules/airflow/operators/check_operator.html
index 7e367ba..4d3fd89 100644
--- a/_modules/airflow/operators/check_operator.html
+++ b/_modules/airflow/operators/check_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -229,7 +219,7 @@
<span class="sd"> needs to be defined. Whereas a get_db_hook is hook that gets a</span>
<span class="sd"> single record from an external source.</span>
-<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :param sql: the sql to be executed. (templated)</span>
<span class="sd"> :type sql: string</span>
<span class="sd"> """</span>
@@ -262,13 +252,13 @@
<span class="k">def</span> <span class="nf">_convert_to_float_if_possible</span><span class="p">(</span><span class="n">s</span><span class="p">):</span>
- <span class="sd">'''</span>
+ <span class="sd">"""</span>
<span class="sd"> A small helper function to convert a string to a numeric value</span>
<span class="sd"> if appropriate</span>
<span class="sd"> :param s: the string to be converted</span>
<span class="sd"> :type s: str</span>
-<span class="sd"> '''</span>
+<span class="sd"> """</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">ret</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
<span class="k">except</span> <span class="p">(</span><span class="ne">ValueError</span><span class="p">,</span> <span class="ne">TypeError</span><span class="p">):</span>
@@ -284,7 +274,7 @@
<span class="sd"> needs to be defined. Whereas a get_db_hook is hook that gets a</span>
<span class="sd"> single record from an external source.</span>
-<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :param sql: the sql to be executed. (templated)</span>
<span class="sd"> :type sql: string</span>
<span class="sd"> """</span>
@@ -388,7 +378,7 @@
<span class="n">sqlt</span> <span class="o">=</span> <span class="p">(</span><span class="s2">"SELECT </span><span class="si">{sqlexp}</span><span class="s2"> FROM </span><span class="si">{table}</span><span class="s2">"</span>
<span class="s2">" WHERE </span><span class="si">{date_filter_column}</span><span class="s2">="</span><span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql1</span> <span class="o">=</span> <span class="n">sqlt</span> <span class="o">+</span> <span class="s2">"'{{ ds }}'"</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">sql2</span> <span class="o">=</span> <span class="n">sqlt</span> <span class="o">+</span> <span class="s2">"'{{ macros.ds_add(ds, "</span><span class="o">+</span><span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">days_back</span><span class="p">)</span><span class="o">+</span><span class="s2">") }}'"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql2</span> <span class="o">=</span> <span class="n">sqlt</span> <span class="o">+</span> <span class="s2">"'{{ macros.ds_add(ds, "</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">days_back</span><span class="p">)</span> <span class="o">+</span> <span class="s2">") }}'"</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
<span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_db_hook</span><span class="p">()</span>
@@ -434,9 +424,7 @@
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -448,7 +436,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -467,6 +455,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -479,19 +468,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/dagrun_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/dagrun_operator.html b/_modules/airflow/operators/dagrun_operator.html
index b77ac68..3a3036c 100644
--- a/_modules/airflow/operators/dagrun_operator.html
+++ b/_modules/airflow/operators/dagrun_operator.html
@@ -24,26 +24,17 @@
-
-
- <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
-
-
-
-
-
- <link rel="index" title="Index"
- href="../../../genindex.html"/>
- <link rel="search" title="Search" href="../../../search.html"/>
- <link rel="top" title="Airflow Documentation" href="../../../index.html"/>
- <link rel="up" title="Module code" href="../../index.html"/>
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
<script src="../../../_static/js/modernizr.min.js"></script>
</head>
-<body class="wy-body-for-nav" role="document">
+<body class="wy-body-for-nav">
<div class="wy-grid-for-nav">
@@ -116,7 +107,7 @@
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
- <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+ <nav class="wy-nav-top" aria-label="top navigation">
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
<a href="../../../index.html">Airflow</a>
@@ -124,9 +115,10 @@
</nav>
-
<div class="wy-nav-content">
+
<div class="rst-content">
+
@@ -156,8 +148,6 @@
<li class="wy-breadcrumbs-aside">
-
-
</li>
</ul>
@@ -178,9 +168,9 @@
<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
<span class="c1"># "License"); you may not use this file except in compliance</span>
<span class="c1"># with the License. You may obtain a copy of the License at</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
-<span class="c1"># </span>
+<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
<span class="c1"># software distributed under the License is distributed on an</span>
<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
@@ -188,12 +178,12 @@
<span class="c1"># specific language governing permissions and limitations</span>
<span class="c1"># under the License.</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span><span class="p">,</span> <span class="n">DagBag</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
<span class="kn">from</span> <span class="nn">airflow.utils</span> <span class="k">import</span> <span class="n">timezone</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.db</span> <span class="k">import</span> <span class="n">create_session</span>
<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.state</span> <span class="k">import</span> <span class="n">State</span>
-<span class="kn">from</span> <span class="nn">airflow</span> <span class="k">import</span> <span class="n">settings</span>
+<span class="kn">from</span> <span class="nn">airflow.api.common.experimental.trigger_dag</span> <span class="k">import</span> <span class="n">trigger_dag</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
<span class="k">class</span> <span class="nc">DagRunOrder</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
@@ -218,6 +208,8 @@
<span class="sd"> to your tasks while executing that DAG run. Your function header</span>
<span class="sd"> should look like ``def foo(context, dag_run_obj):``</span>
<span class="sd"> :type python_callable: python callable</span>
+<span class="sd"> :param execution_date: Execution date for the dag</span>
+<span class="sd"> :type execution_date: datetime.datetime</span>
<span class="sd"> """</span>
<span class="n">template_fields</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">()</span>
@@ -228,35 +220,29 @@
<span class="bp">self</span><span class="p">,</span>
<span class="n">trigger_dag_id</span><span class="p">,</span>
<span class="n">python_callable</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">execution_date</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
<span class="nb">super</span><span class="p">(</span><span class="n">TriggerDagRunOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span> <span class="o">=</span> <span class="n">python_callable</span>
<span class="bp">self</span><span class="o">.</span><span class="n">trigger_dag_id</span> <span class="o">=</span> <span class="n">trigger_dag_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span> <span class="o">=</span> <span class="n">execution_date</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">dro</span> <span class="o">=</span> <span class="n">DagRunOrder</span><span class="p">(</span><span class="n">run_id</span><span class="o">=</span><span class="s1">'trig__'</span> <span class="o">+</span> <span class="n">timezone</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">())</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<span class="n">dro</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">python_callable</span><span class="p">(</span><span class="n">context</span><span class="p">,</span> <span class="n">dro</span><span class="p">)</span>
<span class="k">if</span> <span class="n">dro</span><span class="p">:</span>
- <span class="k">with</span> <span class="n">create_session</span><span class="p">()</span> <span class="k">as</span> <span class="n">session</span><span class="p">:</span>
- <span class="n">dbag</span> <span class="o">=</span> <span class="n">DagBag</span><span class="p">(</span><span class="n">settings</span><span class="o">.</span><span class="n">DAGS_FOLDER</span><span class="p">)</span>
- <span class="n">trigger_dag</span> <span class="o">=</span> <span class="n">dbag</span><span class="o">.</span><span class="n">get_dag</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trigger_dag_id</span><span class="p">)</span>
- <span class="n">dr</span> <span class="o">=</span> <span class="n">trigger_dag</span><span class="o">.</span><span class="n">create_dagrun</span><span class="p">(</span>
- <span class="n">run_id</span><span class="o">=</span><span class="n">dro</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span>
- <span class="n">state</span><span class="o">=</span><span class="n">State</span><span class="o">.</span><span class="n">RUNNING</span><span class="p">,</span>
- <span class="n">conf</span><span class="o">=</span><span class="n">dro</span><span class="o">.</span><span class="n">payload</span><span class="p">,</span>
- <span class="n">external_trigger</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Creating DagRun </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">dr</span><span class="p">)</span>
- <span class="n">session</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">dr</span><span class="p">)</span>
- <span class="n">session</span><span class="o">.</span><span class="n">commit</span><span class="p">()</span>
+ <span class="n">trigger_dag</span><span class="p">(</span><span class="n">dag_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trigger_dag_id</span><span class="p">,</span>
+ <span class="n">run_id</span><span class="o">=</span><span class="n">dro</span><span class="o">.</span><span class="n">run_id</span><span class="p">,</span>
+ <span class="n">conf</span><span class="o">=</span><span class="n">json</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">dro</span><span class="o">.</span><span class="n">payload</span><span class="p">),</span>
+ <span class="n">execution_date</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">execution_date</span><span class="p">,</span>
+ <span class="n">replace_microseconds</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Criteria not met, moving on"</span><span class="p">)</span></div>
</pre></div>
</div>
- <div class="articleComments">
-
- </div>
+
</div>
<footer>
@@ -268,7 +254,7 @@
</p>
</div>
- Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</footer>
@@ -287,6 +273,7 @@
var DOCUMENTATION_OPTIONS = {
URL_ROOT:'../../../',
VERSION:'',
+ LANGUAGE:'None',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
HAS_SOURCE: true,
@@ -299,19 +286,13 @@
-
-
- <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
-
-
<script type="text/javascript">
jQuery(function () {
- SphinxRtdTheme.StickyNav.enable();
+ SphinxRtdTheme.Navigation.enable(true);
});
- </script>
-
+ </script>
</body>
</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/docker_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/docker_operator.html b/_modules/airflow/operators/docker_operator.html
new file mode 100644
index 0000000..3ecbd9f
--- /dev/null
+++ b/_modules/airflow/operators/docker_operator.html
@@ -0,0 +1,470 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.docker_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.docker_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.docker_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">import</span> <span class="nn">json</span>
+
+<span class="kn">from</span> <span class="nn">airflow.hooks.docker_hook</span> <span class="k">import</span> <span class="n">DockerHook</span>
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.file</span> <span class="k">import</span> <span class="n">TemporaryDirectory</span>
+<span class="kn">from</span> <span class="nn">docker</span> <span class="k">import</span> <span class="n">APIClient</span><span class="p">,</span> <span class="n">tls</span>
+<span class="kn">import</span> <span class="nn">ast</span>
+
+
+<div class="viewcode-block" id="DockerOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.docker_operator.DockerOperator">[docs]</a><span class="k">class</span> <span class="nc">DockerOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Execute a command inside a docker container.</span>
+
+<span class="sd"> A temporary directory is created on the host and</span>
+<span class="sd"> mounted into a container to allow storing files</span>
+<span class="sd"> that together exceed the default disk size of 10GB in a container.</span>
+<span class="sd"> The path to the mounted directory can be accessed</span>
+<span class="sd"> via the environment variable ``AIRFLOW_TMP_DIR``.</span>
+
+<span class="sd"> If a login to a private registry is required prior to pulling the image, a</span>
+<span class="sd"> Docker connection needs to be configured in Airflow and the connection ID</span>
+<span class="sd"> be provided with the parameter ``docker_conn_id``.</span>
+
+<span class="sd"> :param image: Docker image from which to create the container.</span>
+<span class="sd"> :type image: str</span>
+<span class="sd"> :param api_version: Remote API version. Set to ``auto`` to automatically</span>
+<span class="sd"> detect the server's version.</span>
+<span class="sd"> :type api_version: str</span>
+<span class="sd"> :param command: Command to be run in the container. (templated)</span>
+<span class="sd"> :type command: str or list</span>
+<span class="sd"> :param cpus: Number of CPUs to assign to the container.</span>
+<span class="sd"> This value gets multiplied with 1024. See</span>
+<span class="sd"> https://docs.docker.com/engine/reference/run/#cpu-share-constraint</span>
+<span class="sd"> :type cpus: float</span>
+<span class="sd"> :param docker_url: URL of the host running the docker daemon.</span>
+<span class="sd"> Default is unix://var/run/docker.sock</span>
+<span class="sd"> :type docker_url: str</span>
+<span class="sd"> :param environment: Environment variables to set in the container. (templated)</span>
+<span class="sd"> :type environment: dict</span>
+<span class="sd"> :param force_pull: Pull the docker image on every run. Default is false.</span>
+<span class="sd"> :type force_pull: bool</span>
+<span class="sd"> :param mem_limit: Maximum amount of memory the container can use.</span>
+<span class="sd"> Either a float value, which represents the limit in bytes,</span>
+<span class="sd"> or a string like ``128m`` or ``1g``.</span>
+<span class="sd"> :type mem_limit: float or str</span>
+<span class="sd"> :param network_mode: Network mode for the container.</span>
+<span class="sd"> :type network_mode: str</span>
+<span class="sd"> :param tls_ca_cert: Path to a PEM-encoded certificate authority</span>
+<span class="sd"> to secure the docker connection.</span>
+<span class="sd"> :type tls_ca_cert: str</span>
+<span class="sd"> :param tls_client_cert: Path to the PEM-encoded certificate</span>
+<span class="sd"> used to authenticate docker client.</span>
+<span class="sd"> :type tls_client_cert: str</span>
+<span class="sd"> :param tls_client_key: Path to the PEM-encoded key used to authenticate docker client.</span>
+<span class="sd"> :type tls_client_key: str</span>
+<span class="sd"> :param tls_hostname: Hostname to match against</span>
+<span class="sd"> the docker server certificate or False to disable the check.</span>
+<span class="sd"> :type tls_hostname: str or bool</span>
+<span class="sd"> :param tls_ssl_version: Version of SSL to use when communicating with docker daemon.</span>
+<span class="sd"> :type tls_ssl_version: str</span>
+<span class="sd"> :param tmp_dir: Mount point inside the container to</span>
+<span class="sd"> a temporary directory created on the host by the operator.</span>
+<span class="sd"> The path is also made available via the environment variable</span>
+<span class="sd"> ``AIRFLOW_TMP_DIR`` inside the container.</span>
+<span class="sd"> :type tmp_dir: str</span>
+<span class="sd"> :param user: Default user inside the docker container.</span>
+<span class="sd"> :type user: int or str</span>
+<span class="sd"> :param volumes: List of volumes to mount into the container, e.g.</span>
+<span class="sd"> ``['/host/path:/container/path', '/host/path2:/container/path2:ro']``.</span>
+<span class="sd"> :param working_dir: Working directory to</span>
+<span class="sd"> set on the container (equivalent to the -w switch the docker client)</span>
+<span class="sd"> :type working_dir: str</span>
+<span class="sd"> :param xcom_push: Does the stdout will be pushed to the next step using XCom.</span>
+<span class="sd"> The default is False.</span>
+<span class="sd"> :type xcom_push: bool</span>
+<span class="sd"> :param xcom_all: Push all the stdout or just the last line.</span>
+<span class="sd"> The default is False (last line).</span>
+<span class="sd"> :type xcom_all: bool</span>
+<span class="sd"> :param docker_conn_id: ID of the Airflow connection to use</span>
+<span class="sd"> :type docker_conn_id: str</span>
+<span class="sd"> """</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'command'</span><span class="p">,</span> <span class="s1">'environment'</span><span class="p">,)</span>
+ <span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sh'</span><span class="p">,</span> <span class="s1">'.bash'</span><span class="p">,)</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span>
+ <span class="n">image</span><span class="p">,</span>
+ <span class="n">api_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">command</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">cpus</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span>
+ <span class="n">docker_url</span><span class="o">=</span><span class="s1">'unix://var/run/docker.sock'</span><span class="p">,</span>
+ <span class="n">environment</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">force_pull</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">mem_limit</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">network_mode</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tls_ca_cert</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tls_client_cert</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tls_client_key</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tls_hostname</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tls_ssl_version</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">tmp_dir</span><span class="o">=</span><span class="s1">'/tmp/airflow'</span><span class="p">,</span>
+ <span class="n">user</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">volumes</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">working_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">xcom_push</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">xcom_all</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">docker_conn_id</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span>
+ <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+
+ <span class="nb">super</span><span class="p">(</span><span class="n">DockerOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">api_version</span> <span class="o">=</span> <span class="n">api_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="o">=</span> <span class="n">command</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cpus</span> <span class="o">=</span> <span class="n">cpus</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span> <span class="o">=</span> <span class="n">docker_url</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">environment</span> <span class="o">=</span> <span class="n">environment</span> <span class="ow">or</span> <span class="p">{}</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">force_pull</span> <span class="o">=</span> <span class="n">force_pull</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">=</span> <span class="n">image</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">mem_limit</span> <span class="o">=</span> <span class="n">mem_limit</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">network_mode</span> <span class="o">=</span> <span class="n">network_mode</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span> <span class="o">=</span> <span class="n">tls_ca_cert</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span> <span class="o">=</span> <span class="n">tls_client_cert</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span> <span class="o">=</span> <span class="n">tls_client_key</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tls_hostname</span> <span class="o">=</span> <span class="n">tls_hostname</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tls_ssl_version</span> <span class="o">=</span> <span class="n">tls_ssl_version</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span> <span class="o">=</span> <span class="n">tmp_dir</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">user</span> <span class="o">=</span> <span class="n">user</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span> <span class="o">=</span> <span class="n">volumes</span> <span class="ow">or</span> <span class="p">[]</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">working_dir</span> <span class="o">=</span> <span class="n">working_dir</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span> <span class="o">=</span> <span class="n">xcom_push</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">xcom_all</span> <span class="o">=</span> <span class="n">xcom_all</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span> <span class="o">=</span> <span class="n">docker_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">shm_size</span> <span class="o">=</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">'shm_size'</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">container</span> <span class="o">=</span> <span class="kc">None</span>
+
+ <span class="k">def</span> <span class="nf">get_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">return</span> <span class="n">DockerHook</span><span class="p">(</span>
+ <span class="n">docker_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span><span class="p">,</span>
+ <span class="n">base_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span><span class="p">,</span>
+ <span class="n">version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
+ <span class="n">tls</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">__get_tls_config</span><span class="p">()</span>
+ <span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Starting docker container from image </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">)</span>
+
+ <span class="n">tls_config</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">__get_tls_config</span><span class="p">()</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">docker_conn_id</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="o">=</span> <span class="n">APIClient</span><span class="p">(</span>
+ <span class="n">base_url</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span><span class="p">,</span>
+ <span class="n">version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">api_version</span><span class="p">,</span>
+ <span class="n">tls</span><span class="o">=</span><span class="n">tls_config</span>
+ <span class="p">)</span>
+
+ <span class="k">if</span> <span class="s1">':'</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span><span class="p">:</span>
+ <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span> <span class="o">+</span> <span class="s1">':latest'</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">image</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">image</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">force_pull</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">images</span><span class="p">(</span><span class="n">name</span><span class="o">=</span><span class="n">image</span><span class="p">))</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Pulling docker image </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="n">image</span><span class="p">)</span>
+ <span class="k">for</span> <span class="n">l</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">pull</span><span class="p">(</span><span class="n">image</span><span class="p">,</span> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="n">output</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">l</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"</span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">output</span><span class="p">[</span><span class="s1">'status'</span><span class="p">])</span>
+
+ <span class="n">cpu_shares</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="nb">round</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cpus</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">))</span>
+
+ <span class="k">with</span> <span class="n">TemporaryDirectory</span><span class="p">(</span><span class="n">prefix</span><span class="o">=</span><span class="s1">'airflowtmp'</span><span class="p">)</span> <span class="k">as</span> <span class="n">host_tmp_dir</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">environment</span><span class="p">[</span><span class="s1">'AIRFLOW_TMP_DIR'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s1">'</span><span class="si">{0}</span><span class="s1">:</span><span class="si">{1}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">host_tmp_dir</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">tmp_dir</span><span class="p">))</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">container</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">create_container</span><span class="p">(</span>
+ <span class="n">command</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">get_command</span><span class="p">(),</span>
+ <span class="n">cpu_shares</span><span class="o">=</span><span class="n">cpu_shares</span><span class="p">,</span>
+ <span class="n">environment</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">environment</span><span class="p">,</span>
+ <span class="n">host_config</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">create_host_config</span><span class="p">(</span>
+ <span class="n">binds</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">volumes</span><span class="p">,</span>
+ <span class="n">network_mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">network_mode</span><span class="p">,</span>
+ <span class="n">shm_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shm_size</span><span class="p">),</span>
+ <span class="n">image</span><span class="o">=</span><span class="n">image</span><span class="p">,</span>
+ <span class="n">mem_limit</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mem_limit</span><span class="p">,</span>
+ <span class="n">user</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">user</span><span class="p">,</span>
+ <span class="n">working_dir</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">working_dir</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">start</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
+
+ <span class="n">line</span> <span class="o">=</span> <span class="s1">''</span>
+ <span class="k">for</span> <span class="n">line</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">logs</span><span class="p">(</span><span class="n">container</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">],</span> <span class="n">stream</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+ <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">line</span><span class="p">,</span> <span class="s1">'decode'</span><span class="p">):</span>
+ <span class="n">line</span> <span class="o">=</span> <span class="n">line</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">'utf-8'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+
+ <span class="n">exit_code</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">wait</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
+ <span class="k">if</span> <span class="n">exit_code</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s1">'docker container failed'</span><span class="p">)</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_push_flag</span><span class="p">:</span>
+ <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">logs</span><span class="p">(</span><span class="n">container</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span> \
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">xcom_all</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">line</span><span class="p">)</span>
+
+ <span class="k">def</span> <span class="nf">get_command</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'['</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+ <span class="n">commands</span> <span class="o">=</span> <span class="n">ast</span><span class="o">.</span><span class="n">literal_eval</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">command</span><span class="p">)</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">commands</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">command</span>
+ <span class="k">return</span> <span class="n">commands</span>
+
+ <span class="k">def</span> <span class="nf">on_kill</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cli</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Stopping docker container'</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">cli</span><span class="o">.</span><span class="n">stop</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">container</span><span class="p">[</span><span class="s1">'Id'</span><span class="p">])</span>
+
+ <span class="k">def</span> <span class="nf">__get_tls_config</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="n">tls_config</span> <span class="o">=</span> <span class="kc">None</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span><span class="p">:</span>
+ <span class="n">tls_config</span> <span class="o">=</span> <span class="n">tls</span><span class="o">.</span><span class="n">TLSConfig</span><span class="p">(</span>
+ <span class="n">ca_cert</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_ca_cert</span><span class="p">,</span>
+ <span class="n">client_cert</span><span class="o">=</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_client_cert</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">tls_client_key</span><span class="p">),</span>
+ <span class="n">verify</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+ <span class="n">ssl_version</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_ssl_version</span><span class="p">,</span>
+ <span class="n">assert_hostname</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">tls_hostname</span>
+ <span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">docker_url</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'tcp://'</span><span class="p">,</span> <span class="s1">'https://'</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">tls_config</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/7d4d7628/_modules/airflow/operators/druid_check_operator.html
----------------------------------------------------------------------
diff --git a/_modules/airflow/operators/druid_check_operator.html b/_modules/airflow/operators/druid_check_operator.html
new file mode 100644
index 0000000..aac2c6e
--- /dev/null
+++ b/_modules/airflow/operators/druid_check_operator.html
@@ -0,0 +1,309 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
+
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+ <title>airflow.operators.druid_check_operator — Airflow Documentation</title>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+ <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
+ <link rel="index" title="Index" href="../../../genindex.html" />
+ <link rel="search" title="Search" href="../../../search.html" />
+
+
+ <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav">
+
+
+ <div class="wy-grid-for-nav">
+
+
+ <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+ <div class="wy-side-scroll">
+ <div class="wy-side-nav-search">
+
+
+
+ <a href="../../../index.html" class="icon icon-home"> Airflow
+
+
+
+ </a>
+
+
+
+
+
+
+
+<div role="search">
+ <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+ <input type="text" name="q" placeholder="Search docs" />
+ <input type="hidden" name="check_keywords" value="yes" />
+ <input type="hidden" name="area" value="default" />
+ </form>
+</div>
+
+
+ </div>
+
+ <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+
+
+
+
+
+
+ <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../project.html">Project</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../license.html">License</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../start.html">Quick Start</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../installation.html">Installation</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../tutorial.html">Tutorial</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../howto/index.html">How-to Guides</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../ui.html">UI / Screenshots</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../concepts.html">Concepts</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../profiling.html">Data Profiling</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../cli.html">Command Line Interface</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../scheduler.html">Scheduling & Triggers</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../plugins.html">Plugins</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../timezone.html">Time zones</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../integration.html">Integration</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../lineage.html">Lineage</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../faq.html">FAQ</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../code.html">API Reference</a></li>
+</ul>
+
+
+
+ </div>
+ </div>
+ </nav>
+
+ <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+
+ <nav class="wy-nav-top" aria-label="top navigation">
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../../../index.html">Airflow</a>
+
+ </nav>
+
+
+ <div class="wy-nav-content">
+
+ <div class="rst-content">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+ <ul class="wy-breadcrumbs">
+
+ <li><a href="../../../index.html">Docs</a> »</li>
+
+ <li><a href="../../index.html">Module code</a> »</li>
+
+ <li>airflow.operators.druid_check_operator</li>
+
+
+ <li class="wy-breadcrumbs-aside">
+
+ </li>
+
+ </ul>
+
+
+ <hr/>
+</div>
+ <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+ <div itemprop="articleBody">
+
+ <h1>Source code for airflow.operators.druid_check_operator</h1><div class="highlight"><pre>
+<span></span><span class="c1"># -*- coding: utf-8 -*-</span>
+<span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one</span>
+<span class="c1"># or more contributor license agreements. See the NOTICE file</span>
+<span class="c1"># distributed with this work for additional information</span>
+<span class="c1"># regarding copyright ownership. The ASF licenses this file</span>
+<span class="c1"># to you under the Apache License, Version 2.0 (the</span>
+<span class="c1"># "License"); you may not use this file except in compliance</span>
+<span class="c1"># with the License. You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1"># http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing,</span>
+<span class="c1"># software distributed under the License is distributed on an</span>
+<span class="c1"># "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span>
+<span class="c1"># KIND, either express or implied. See the License for the</span>
+<span class="c1"># specific language governing permissions and limitations</span>
+<span class="c1"># under the License.</span>
+
+<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="k">import</span> <span class="n">DruidDbApiHook</span>
+<span class="kn">from</span> <span class="nn">airflow.operators.check_operator</span> <span class="k">import</span> <span class="n">CheckOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+
+
+<div class="viewcode-block" id="DruidCheckOperator"><a class="viewcode-back" href="../../../code.html#airflow.operators.druid_check_operator.DruidCheckOperator">[docs]</a><span class="k">class</span> <span class="nc">DruidCheckOperator</span><span class="p">(</span><span class="n">CheckOperator</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Performs checks against Druid. The ``DruidCheckOperator`` expects</span>
+<span class="sd"> a sql query that will return a single row. Each value on that</span>
+<span class="sd"> first row is evaluated using python ``bool`` casting. If any of the</span>
+<span class="sd"> values return ``False`` the check is failed and errors out.</span>
+
+<span class="sd"> Note that Python bool casting evals the following as ``False``:</span>
+
+<span class="sd"> * ``False``</span>
+<span class="sd"> * ``0``</span>
+<span class="sd"> * Empty string (``""``)</span>
+<span class="sd"> * Empty list (``[]``)</span>
+<span class="sd"> * Empty dictionary or set (``{}``)</span>
+
+<span class="sd"> Given a query like ``SELECT COUNT(*) FROM foo``, it will fail only if</span>
+<span class="sd"> the count ``== 0``. You can craft much more complex query that could,</span>
+<span class="sd"> for instance, check that the table has the same number of rows as</span>
+<span class="sd"> the source table upstream, or that the count of today's partition is</span>
+<span class="sd"> greater than yesterday's partition, or that a set of metrics are less</span>
+<span class="sd"> than 3 standard deviation for the 7 day average.</span>
+<span class="sd"> This operator can be used as a data quality check in your pipeline, and</span>
+<span class="sd"> depending on where you put it in your DAG, you have the choice to</span>
+<span class="sd"> stop the critical path, preventing from</span>
+<span class="sd"> publishing dubious data, or on the side and receive email alterts</span>
+<span class="sd"> without stopping the progress of the DAG.</span>
+
+<span class="sd"> :param sql: the sql to be executed</span>
+<span class="sd"> :type sql: string</span>
+<span class="sd"> :param druid_broker_conn_id: reference to the druid broker</span>
+<span class="sd"> :type druid_broker_conn_id: string</span>
+<span class="sd"> """</span>
+
+ <span class="nd">@apply_defaults</span>
+ <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
+ <span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">,</span>
+ <span class="n">druid_broker_conn_id</span><span class="o">=</span><span class="s1">'druid_broker_default'</span><span class="p">,</span>
+ <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">DruidCheckOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">sql</span><span class="o">=</span><span class="n">sql</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">druid_broker_conn_id</span> <span class="o">=</span> <span class="n">druid_broker_conn_id</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
+
+<div class="viewcode-block" id="DruidCheckOperator.get_db_hook"><a class="viewcode-back" href="../../../code.html#airflow.operators.druid_check_operator.DruidCheckOperator.get_db_hook">[docs]</a> <span class="k">def</span> <span class="nf">get_db_hook</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Return the druid db api hook.</span>
+<span class="sd"> """</span>
+ <span class="k">return</span> <span class="n">DruidDbApiHook</span><span class="p">(</span><span class="n">druid_broker_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_broker_conn_id</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="DruidCheckOperator.get_first"><a class="viewcode-back" href="../../../code.html#airflow.operators.druid_check_operator.DruidCheckOperator.get_first">[docs]</a> <span class="k">def</span> <span class="nf">get_first</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sql</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Executes the druid sql to druid broker and returns the first resulting row.</span>
+
+<span class="sd"> :param sql: the sql statement to be executed (str)</span>
+<span class="sd"> :type sql: str</span>
+<span class="sd"> """</span>
+ <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_db_hook</span><span class="p">()</span><span class="o">.</span><span class="n">get_conn</span><span class="p">()</span> <span class="k">as</span> <span class="n">cur</span><span class="p">:</span>
+ <span class="n">cur</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sql</span><span class="p">)</span>
+ <span class="k">return</span> <span class="n">cur</span><span class="o">.</span><span class="n">fetchone</span><span class="p">()</span></div>
+
+ <span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing SQL check: </span><span class="si">{}</span><span class="s1">'</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">))</span>
+ <span class="n">record</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_first</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Record: </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">record</span><span class="p">)))</span>
+ <span class="k">if</span> <span class="ow">not</span> <span class="n">record</span><span class="p">:</span>
+ <span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="s2">"The query returned None"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Success."</span><span class="p">)</span></div>
+</pre></div>
+
+ </div>
+
+ </div>
+ <footer>
+
+
+ <hr/>
+
+ <div role="contentinfo">
+ <p>
+
+ </p>
+ </div>
+ Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+
+</footer>
+
+ </div>
+ </div>
+
+ </section>
+
+ </div>
+
+
+
+
+
+ <script type="text/javascript">
+ var DOCUMENTATION_OPTIONS = {
+ URL_ROOT:'../../../',
+ VERSION:'',
+ LANGUAGE:'None',
+ COLLAPSE_INDEX:false,
+ FILE_SUFFIX:'.html',
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
+ };
+ </script>
+ <script type="text/javascript" src="../../../_static/jquery.js"></script>
+ <script type="text/javascript" src="../../../_static/underscore.js"></script>
+ <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+
+
+ <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+
+ <script type="text/javascript">
+ jQuery(function () {
+ SphinxRtdTheme.Navigation.enable(true);
+ });
+ </script>
+
+</body>
+</html>
\ No newline at end of file