You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by cr...@apache.org on 2018/01/03 17:48:14 UTC
[14/35] incubator-airflow-site git commit: 1.9.0
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_operator.html
----------------------------------------------------------------------
diff --git a/_modules/hive_operator.html b/_modules/hive_operator.html
index 4d8c62c..b0e3265 100644
--- a/_modules/hive_operator.html
+++ b/_modules/hive_operator.html
@@ -13,6 +13,8 @@
+
+
@@ -30,6 +32,9 @@
+ <link rel="index" title="Index"
+ href="../genindex.html"/>
+ <link rel="search" title="Search" href="../search.html"/>
<link rel="top" title="Airflow Documentation" href="../index.html"/>
<link rel="up" title="Module code" href="index.html"/>
@@ -40,6 +45,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -76,7 +82,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
</ul>
@@ -104,8 +115,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../index.html">Airflow</a>
+
</nav>
@@ -118,19 +131,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../index.html">Docs</a> »</li>
-
+
+ <li><a href="../index.html">Docs</a> »</li>
+
<li><a href="index.html">Module code</a> »</li>
-
- <li>hive_operator</li>
+
+ <li>hive_operator</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -150,14 +180,12 @@
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-
-<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">re</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="kn">import</span> <span class="n">context_to_airflow_vars</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.operator_helpers</span> <span class="k">import</span> <span class="n">context_to_airflow_vars</span>
<div class="viewcode-block" id="HiveOperator"><a class="viewcode-back" href="../code.html#airflow.operators.HiveOperator">[docs]</a><span class="k">class</span> <span class="nc">HiveOperator</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -196,21 +224,21 @@
<span class="bp">self</span><span class="p">,</span> <span class="n">hql</span><span class="p">,</span>
<span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span>
<span class="n">schema</span><span class="o">=</span><span class="s1">'default'</span><span class="p">,</span>
- <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
- <span class="n">script_begin_tag</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">run_as_owner</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
- <span class="n">mapred_queue</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">mapred_job_name</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">hiveconf_jinja_translate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">script_begin_tag</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">run_as_owner</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+ <span class="n">mapred_queue</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">mapred_queue_priority</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">mapred_job_name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HiveOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hiveconf_jinja_translate</span> <span class="o">=</span> <span class="n">hiveconf_jinja_translate</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hql</span> <span class="o">=</span> <span class="n">hql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span> <span class="o">=</span> <span class="n">hive_cli_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">script_begin_tag</span> <span class="o">=</span> <span class="n">script_begin_tag</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">None</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="kc">None</span>
<span class="k">if</span> <span class="n">run_as_owner</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">run_as</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dag</span><span class="o">.</span><span class="n">owner</span>
@@ -234,7 +262,7 @@
<span class="bp">self</span><span class="o">.</span><span class="n">hql</span> <span class="o">=</span> <span class="s2">"</span><span class="se">\n</span><span class="s2">"</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">script_begin_tag</span><span class="p">)[</span><span class="mi">1</span><span class="p">:])</span>
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: '</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'Executing: </span><span class="si">%s</span><span class="s1">'</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_hook</span><span class="p">()</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hook</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hql</span><span class="p">,</span> <span class="n">schema</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span>
<span class="n">hive_conf</span><span class="o">=</span><span class="n">context_to_airflow_vars</span><span class="p">(</span><span class="n">context</span><span class="p">))</span>
@@ -248,6 +276,9 @@
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -280,7 +311,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_to_druid.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_druid.html b/_modules/hive_to_druid.html
index d85bfb3..35dc4ff 100644
--- a/_modules/hive_to_druid.html
+++ b/_modules/hive_to_druid.html
@@ -13,6 +13,8 @@
+
+
@@ -30,6 +32,9 @@
+ <link rel="index" title="Index"
+ href="../genindex.html"/>
+ <link rel="search" title="Search" href="../search.html"/>
<link rel="top" title="Airflow Documentation" href="../index.html"/>
<link rel="up" title="Module code" href="index.html"/>
@@ -40,6 +45,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -76,7 +82,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
</ul>
@@ -104,8 +115,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../index.html">Airflow</a>
+
</nav>
@@ -118,19 +131,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../index.html">Docs</a> »</li>
-
+
+ <li><a href="../index.html">Docs</a> »</li>
+
<li><a href="index.html">Module code</a> »</li>
-
- <li>hive_to_druid</li>
+
+ <li>hive_to_druid</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -150,13 +180,13 @@
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="k">import</span> <span class="n">DruidHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveCliHook</span><span class="p">,</span> <span class="n">HiveMetastoreHook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.druid_hook</span> <span class="kn">import</span> <span class="n">DruidHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
+<span class="n">LOAD_CHECK_INTERVAL</span> <span class="o">=</span> <span class="mi">5</span>
+<span class="n">DEFAULT_TARGET_PARTITION_SIZE</span> <span class="o">=</span> <span class="mi">5000000</span>
<div class="viewcode-block" id="HiveToDruidTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToDruidTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToDruidTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -189,7 +219,6 @@
<span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'sql'</span><span class="p">,</span> <span class="s1">'intervals'</span><span class="p">)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'.sql'</span><span class="p">,)</span>
- <span class="c1">#ui_color = '#a0e08c'</span>
<span class="nd">@apply_defaults</span>
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span>
@@ -197,22 +226,26 @@
<span class="n">sql</span><span class="p">,</span>
<span class="n">druid_datasource</span><span class="p">,</span>
<span class="n">ts_dim</span><span class="p">,</span>
- <span class="n">metric_spec</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">metric_spec</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="s1">'hive_cli_default'</span><span class="p">,</span>
<span class="n">druid_ingest_conn_id</span><span class="o">=</span><span class="s1">'druid_ingest_default'</span><span class="p">,</span>
<span class="n">metastore_conn_id</span><span class="o">=</span><span class="s1">'metastore_default'</span><span class="p">,</span>
- <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">intervals</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
+ <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">intervals</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="n">num_shards</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
<span class="n">target_partition_size</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
+ <span class="n">query_granularity</span><span class="o">=</span><span class="s2">"NONE"</span><span class="p">,</span>
+ <span class="n">segment_granularity</span><span class="o">=</span><span class="s2">"DAY"</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HiveToDruidTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HiveToDruidTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span> <span class="o">=</span> <span class="n">druid_datasource</span>
<span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span> <span class="o">=</span> <span class="n">ts_dim</span>
<span class="bp">self</span><span class="o">.</span><span class="n">intervals</span> <span class="o">=</span> <span class="n">intervals</span> <span class="ow">or</span> <span class="p">[</span><span class="s1">'{{ ds }}/{{ tomorrow_ds }}'</span><span class="p">]</span>
<span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span> <span class="o">=</span> <span class="n">num_shards</span>
<span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span> <span class="o">=</span> <span class="n">target_partition_size</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">query_granularity</span> <span class="o">=</span> <span class="n">query_granularity</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">segment_granularity</span> <span class="o">=</span> <span class="n">segment_granularity</span>
<span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span> <span class="o">=</span> <span class="n">metric_spec</span> <span class="ow">or</span> <span class="p">[{</span>
<span class="s2">"name"</span><span class="p">:</span> <span class="s2">"count"</span><span class="p">,</span>
<span class="s2">"type"</span><span class="p">:</span> <span class="s2">"count"</span><span class="p">}]</span>
@@ -223,28 +256,30 @@
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hive</span> <span class="o">=</span> <span class="n">HiveCliHook</span><span class="p">(</span><span class="n">hive_cli_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hive_cli_conn_id</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span>
<span class="n">hive_table</span> <span class="o">=</span> <span class="s1">'druid.'</span> <span class="o">+</span> <span class="n">context</span><span class="p">[</span><span class="s1">'task_instance_key_str'</span><span class="p">]</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s1">'.'</span><span class="p">,</span> <span class="s1">'_'</span><span class="p">)</span>
<span class="n">sql</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">strip</span><span class="p">(</span><span class="s1">';'</span><span class="p">)</span>
<span class="n">hql</span> <span class="o">=</span> <span class="s2">"""</span><span class="se">\</span>
-<span class="s2"> set mapred.output.compress=false;</span>
-<span class="s2"> set hive.exec.compress.output=false;</span>
-<span class="s2"> DROP TABLE IF EXISTS {hive_table};</span>
-<span class="s2"> CREATE TABLE {hive_table}</span>
+<span class="s2"> SET mapred.output.compress=false;</span>
+<span class="s2"> SET hive.exec.compress.output=false;</span>
+<span class="s2"> DROP TABLE IF EXISTS </span><span class="si">{hive_table}</span><span class="s2">;</span>
+<span class="s2"> CREATE TABLE </span><span class="si">{hive_table}</span><span class="s2"></span>
<span class="s2"> ROW FORMAT DELIMITED FIELDS TERMINATED BY '</span><span class="se">\t</span><span class="s2">'</span>
<span class="s2"> STORED AS TEXTFILE</span>
<span class="s2"> TBLPROPERTIES ('serialization.null.format' = '')</span>
<span class="s2"> AS</span>
-<span class="s2"> {sql}</span>
+<span class="s2"> </span><span class="si">{sql}</span><span class="s2"></span>
<span class="s2"> """</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="nb">locals</span><span class="p">())</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command:</span><span class="se">\n</span><span class="s2"> {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hql</span><span class="p">))</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running command:</span><span class="se">\n</span><span class="s2"> </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">hql</span><span class="p">)</span>
<span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span>
<span class="n">m</span> <span class="o">=</span> <span class="n">HiveMetastoreHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">metastore_conn_id</span><span class="p">)</span>
- <span class="n">t</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
+ <span class="c1"># Get the Hive table and extract the columns</span>
+ <span class="n">t</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
<span class="n">columns</span> <span class="o">=</span> <span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">t</span><span class="o">.</span><span class="n">sd</span><span class="o">.</span><span class="n">cols</span><span class="p">]</span>
+ <span class="c1"># Get the path on hdfs</span>
<span class="n">hdfs_uri</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="n">get_table</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span><span class="o">.</span><span class="n">sd</span><span class="o">.</span><span class="n">location</span>
<span class="n">pos</span> <span class="o">=</span> <span class="n">hdfs_uri</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="s1">'/user'</span><span class="p">)</span>
<span class="n">static_path</span> <span class="o">=</span> <span class="n">hdfs_uri</span><span class="p">[</span><span class="n">pos</span><span class="p">:]</span>
@@ -252,26 +287,113 @@
<span class="n">schema</span><span class="p">,</span> <span class="n">table</span> <span class="o">=</span> <span class="n">hive_table</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'.'</span><span class="p">)</span>
<span class="n">druid</span> <span class="o">=</span> <span class="n">DruidHook</span><span class="p">(</span><span class="n">druid_ingest_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_ingest_conn_id</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into Druid"</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"HDFS path: "</span> <span class="o">+</span> <span class="n">static_path</span><span class="p">)</span>
<span class="k">try</span><span class="p">:</span>
- <span class="n">druid</span><span class="o">.</span><span class="n">load_from_hdfs</span><span class="p">(</span>
- <span class="n">datasource</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span><span class="p">,</span>
- <span class="n">intervals</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
- <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span> <span class="n">ts_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
- <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span> <span class="n">num_shards</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span><span class="p">,</span> <span class="n">target_partition_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span><span class="p">,</span>
- <span class="n">metric_spec</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span> <span class="n">hadoop_dependency_coordinates</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span>
+ <span class="n">index_spec</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">construct_ingest_query</span><span class="p">(</span>
+ <span class="n">static_path</span><span class="o">=</span><span class="n">static_path</span><span class="p">,</span>
+ <span class="n">columns</span><span class="o">=</span><span class="n">columns</span><span class="p">,</span>
+ <span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into Druid, hdfs path: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">static_path</span><span class="p">)</span>
+
+ <span class="n">druid</span><span class="o">.</span><span class="n">submit_indexing_job</span><span class="p">(</span><span class="n">index_spec</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Load seems to have succeeded!"</span><span class="p">)</span>
<span class="k">finally</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
- <span class="s2">"Cleaning up by dropping the temp "</span>
- <span class="s2">"Hive table {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">))</span>
- <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
- <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+ <span class="s2">"Cleaning up by dropping the temp Hive table </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span>
+ <span class="n">hive_table</span>
+ <span class="p">)</span>
+ <span class="n">hql</span> <span class="o">=</span> <span class="s2">"DROP TABLE IF EXISTS </span><span class="si">{}</span><span class="s2">"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hive_table</span><span class="p">)</span>
+ <span class="n">hive</span><span class="o">.</span><span class="n">run_cli</span><span class="p">(</span><span class="n">hql</span><span class="p">)</span>
+
+<div class="viewcode-block" id="HiveToDruidTransfer.construct_ingest_query"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToDruidTransfer.construct_ingest_query">[docs]</a> <span class="k">def</span> <span class="nf">construct_ingest_query</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">static_path</span><span class="p">,</span> <span class="n">columns</span><span class="p">):</span>
+ <span class="sd">"""</span>
+<span class="sd"> Builds an ingest query for an HDFS TSV load.</span>
+
+<span class="sd"> :param static_path: The path on hdfs where the data is</span>
+<span class="sd"> :type static_path: str</span>
+<span class="sd"> :param columns: List of all the columns that are available</span>
+<span class="sd"> :type columns: list</span>
+<span class="sd"> """</span>
+
+ <span class="c1"># backward compatibilty for num_shards, but target_partition_size is the default setting</span>
+ <span class="c1"># and overwrites the num_shards</span>
+ <span class="n">num_shards</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span>
+ <span class="n">target_partition_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_partition_size</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_shards</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
+ <span class="n">target_partition_size</span> <span class="o">=</span> <span class="n">DEFAULT_TARGET_PARTITION_SIZE</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="n">num_shards</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
+
+ <span class="n">metric_names</span> <span class="o">=</span> <span class="p">[</span><span class="n">m</span><span class="p">[</span><span class="s1">'fieldName'</span><span class="p">]</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span> <span class="k">if</span> <span class="n">m</span><span class="p">[</span><span class="s1">'type'</span><span class="p">]</span> <span class="o">!=</span> <span class="s1">'count'</span><span class="p">]</span>
+
+ <span class="c1"># Take all the columns, which are not the time dimension or a metric, as the dimension columns</span>
+ <span class="n">dimensions</span> <span class="o">=</span> <span class="p">[</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">columns</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">metric_names</span> <span class="ow">and</span> <span class="n">c</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">]</span>
+
+ <span class="n">ingest_query_dict</span> <span class="o">=</span> <span class="p">{</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"index_hadoop"</span><span class="p">,</span>
+ <span class="s2">"spec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"dataSchema"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"metricsSpec"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">metric_spec</span><span class="p">,</span>
+ <span class="s2">"granularitySpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"queryGranularity"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">query_granularity</span><span class="p">,</span>
+ <span class="s2">"intervals"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">intervals</span><span class="p">,</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"uniform"</span><span class="p">,</span>
+ <span class="s2">"segmentGranularity"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">segment_granularity</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="s2">"parser"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"string"</span><span class="p">,</span>
+ <span class="s2">"parseSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"columns"</span><span class="p">:</span> <span class="n">columns</span><span class="p">,</span>
+ <span class="s2">"dimensionsSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"dimensionExclusions"</span><span class="p">:</span> <span class="p">[],</span>
+ <span class="s2">"dimensions"</span><span class="p">:</span> <span class="n">dimensions</span><span class="p">,</span> <span class="c1"># list of names</span>
+ <span class="s2">"spatialDimensions"</span><span class="p">:</span> <span class="p">[]</span>
+ <span class="p">},</span>
+ <span class="s2">"timestampSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"column"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">ts_dim</span><span class="p">,</span>
+ <span class="s2">"format"</span><span class="p">:</span> <span class="s2">"auto"</span>
+ <span class="p">},</span>
+ <span class="s2">"format"</span><span class="p">:</span> <span class="s2">"tsv"</span>
+ <span class="p">}</span>
+ <span class="p">},</span>
+ <span class="s2">"dataSource"</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">druid_datasource</span>
+ <span class="p">},</span>
+ <span class="s2">"tuningConfig"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hadoop"</span><span class="p">,</span>
+ <span class="s2">"jobProperties"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"mapreduce.job.user.classpath.first"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span>
+ <span class="s2">"mapreduce.map.output.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span>
+ <span class="s2">"mapreduce.output.fileoutputformat.compress"</span><span class="p">:</span> <span class="s2">"false"</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="s2">"partitionsSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hashed"</span><span class="p">,</span>
+ <span class="s2">"targetPartitionSize"</span><span class="p">:</span> <span class="n">target_partition_size</span><span class="p">,</span>
+ <span class="s2">"numShards"</span><span class="p">:</span> <span class="n">num_shards</span><span class="p">,</span>
+ <span class="p">},</span>
+ <span class="p">},</span>
+ <span class="s2">"ioConfig"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"inputSpec"</span><span class="p">:</span> <span class="p">{</span>
+ <span class="s2">"paths"</span><span class="p">:</span> <span class="n">static_path</span><span class="p">,</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"static"</span>
+ <span class="p">},</span>
+ <span class="s2">"type"</span><span class="p">:</span> <span class="s2">"hadoop"</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+ <span class="p">}</span>
+
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span><span class="p">:</span>
+ <span class="n">ingest_query_dict</span><span class="p">[</span><span class="s1">'hadoopDependencyCoordinates'</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">hadoop_dependency_coordinates</span>
+
+ <span class="k">return</span> <span class="n">ingest_query_dict</span></div></div>
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -304,7 +426,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/hive_to_mysql.html
----------------------------------------------------------------------
diff --git a/_modules/hive_to_mysql.html b/_modules/hive_to_mysql.html
index fd8b342..6262205 100644
--- a/_modules/hive_to_mysql.html
+++ b/_modules/hive_to_mysql.html
@@ -13,6 +13,8 @@
+
+
@@ -30,6 +32,9 @@
+ <link rel="index" title="Index"
+ href="../genindex.html"/>
+ <link rel="search" title="Search" href="../search.html"/>
<link rel="top" title="Airflow Documentation" href="../index.html"/>
<link rel="up" title="Module code" href="index.html"/>
@@ -40,6 +45,7 @@
<body class="wy-body-for-nav" role="document">
+
<div class="wy-grid-for-nav">
@@ -76,7 +82,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -90,6 +99,8 @@
<li class="toctree-l1"><a class="reference internal" href="../scheduler.html">Scheduling & Triggers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../plugins.html">Plugins</a></li>
<li class="toctree-l1"><a class="reference internal" href="../security.html">Security</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../api.html">Experimental Rest API</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../integration.html">Integration</a></li>
<li class="toctree-l1"><a class="reference internal" href="../faq.html">FAQ</a></li>
<li class="toctree-l1"><a class="reference internal" href="../code.html">API Reference</a></li>
</ul>
@@ -104,8 +115,10 @@
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
- <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
- <a href="../index.html">Airflow</a>
+
+ <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+ <a href="../index.html">Airflow</a>
+
</nav>
@@ -118,19 +131,36 @@
+
+
+
+
+
+
+
+
+
+
<div role="navigation" aria-label="breadcrumbs navigation">
+
<ul class="wy-breadcrumbs">
- <li><a href="../index.html">Docs</a> »</li>
-
+
+ <li><a href="../index.html">Docs</a> »</li>
+
<li><a href="index.html">Module code</a> »</li>
-
- <li>hive_to_mysql</li>
+
+ <li>hive_to_mysql</li>
+
+
<li class="wy-breadcrumbs-aside">
-
+
</li>
+
</ul>
+
+
<hr/>
</div>
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
@@ -150,15 +180,12 @@
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="k">import</span> <span class="n">HiveServer2Hook</span>
+<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="k">import</span> <span class="n">MySqlHook</span>
+<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
+<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="k">import</span> <span class="n">apply_defaults</span>
-<span class="kn">import</span> <span class="nn">logging</span>
-
-<span class="kn">from</span> <span class="nn">airflow.hooks.hive_hooks</span> <span class="kn">import</span> <span class="n">HiveServer2Hook</span>
-<span class="kn">from</span> <span class="nn">airflow.hooks.mysql_hook</span> <span class="kn">import</span> <span class="n">MySqlHook</span>
-<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="kn">import</span> <span class="n">BaseOperator</span>
-<span class="kn">from</span> <span class="nn">airflow.utils.decorators</span> <span class="kn">import</span> <span class="n">apply_defaults</span>
-
-<span class="kn">from</span> <span class="nn">tempfile</span> <span class="kn">import</span> <span class="n">NamedTemporaryFile</span>
+<span class="kn">from</span> <span class="nn">tempfile</span> <span class="k">import</span> <span class="n">NamedTemporaryFile</span>
<div class="viewcode-block" id="HiveToMySqlTransfer"><a class="viewcode-back" href="../code.html#airflow.operators.HiveToMySqlTransfer">[docs]</a><span class="k">class</span> <span class="nc">HiveToMySqlTransfer</span><span class="p">(</span><span class="n">BaseOperator</span><span class="p">):</span>
@@ -204,11 +231,11 @@
<span class="n">mysql_table</span><span class="p">,</span>
<span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="s1">'hiveserver2_default'</span><span class="p">,</span>
<span class="n">mysql_conn_id</span><span class="o">=</span><span class="s1">'mysql_default'</span><span class="p">,</span>
- <span class="n">mysql_preoperator</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">mysql_postoperator</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
- <span class="n">bulk_load</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
+ <span class="n">mysql_preoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">mysql_postoperator</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+ <span class="n">bulk_load</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">HiveToMySqlTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">HiveToMySqlTransfer</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">sql</span> <span class="o">=</span> <span class="n">sql</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span> <span class="o">=</span> <span class="n">mysql_table</span>
<span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span> <span class="o">=</span> <span class="n">mysql_conn_id</span>
@@ -219,22 +246,21 @@
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">hive</span> <span class="o">=</span> <span class="n">HiveServer2Hook</span><span class="p">(</span><span class="n">hiveserver2_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">hiveserver2_conn_id</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive"</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Extracting data from Hive: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">:</span>
<span class="n">tmpfile</span> <span class="o">=</span> <span class="n">NamedTemporaryFile</span><span class="p">()</span>
<span class="n">hive</span><span class="o">.</span><span class="n">to_csv</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">,</span> <span class="n">tmpfile</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s1">'</span><span class="se">\t</span><span class="s1">'</span><span class="p">,</span>
- <span class="n">lineterminator</span><span class="o">=</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="n">output_header</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
+ <span class="n">lineterminator</span><span class="o">=</span><span class="s1">'</span><span class="se">\n</span><span class="s1">'</span><span class="p">,</span> <span class="n">output_header</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">results</span> <span class="o">=</span> <span class="n">hive</span><span class="o">.</span><span class="n">get_records</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sql</span><span class="p">)</span>
<span class="n">mysql</span> <span class="o">=</span> <span class="n">MySqlHook</span><span class="p">(</span><span class="n">mysql_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_conn_id</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL preoperator"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL preoperator"</span><span class="p">)</span>
<span class="n">mysql</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_preoperator</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into MySQL"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Inserting rows into MySQL"</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">:</span>
<span class="n">mysql</span><span class="o">.</span><span class="n">bulk_load</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span><span class="p">,</span> <span class="n">tmp_file</span><span class="o">=</span><span class="n">tmpfile</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
@@ -243,13 +269,16 @@
<span class="n">mysql</span><span class="o">.</span><span class="n">insert_rows</span><span class="p">(</span><span class="n">table</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_table</span><span class="p">,</span> <span class="n">rows</span><span class="o">=</span><span class="n">results</span><span class="p">)</span>
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mysql_postoperator</span><span class="p">:</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL postoperator"</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Running MySQL postoperator"</span><span class="p">)</span>
<span class="n">mysql</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">mysql_postoperator</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Done."</span><span class="p">)</span></div>
</pre></div>
</div>
+ <div class="articleComments">
+
+ </div>
</div>
<footer>
@@ -282,7 +311,8 @@
VERSION:'',
COLLAPSE_INDEX:false,
FILE_SUFFIX:'.html',
- HAS_SOURCE: true
+ HAS_SOURCE: true,
+ SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/http_hook.html
----------------------------------------------------------------------
diff --git a/_modules/http_hook.html b/_modules/http_hook.html
index 5458ec1..ec71e54 100644
--- a/_modules/http_hook.html
+++ b/_modules/http_hook.html
@@ -13,6 +13,8 @@
+
+
@@ -80,7 +82,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -177,7 +182,6 @@
<span class="c1"># limitations under the License.</span>
<span class="kn">from</span> <span class="nn">builtins</span> <span class="k">import</span> <span class="nb">str</span>
-<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">requests</span>
@@ -201,9 +205,13 @@
<span class="sd"> """</span>
<span class="n">conn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_connection</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span>
<span class="n">session</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Session</span><span class="p">()</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
- <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'http'</span><span class="p">):</span>
- <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="s1">'http://'</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span>
+
+ <span class="k">if</span> <span class="s2">"://"</span> <span class="ow">in</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
+ <span class="k">else</span><span class="p">:</span>
+ <span class="c1"># schema defaults to HTTP</span>
+ <span class="n">schema</span> <span class="o">=</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">schema</span> <span class="k">else</span> <span class="s2">"http"</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="n">schema</span> <span class="o">+</span> <span class="s2">"://"</span> <span class="o">+</span> <span class="n">conn</span><span class="o">.</span><span class="n">host</span>
<span class="k">if</span> <span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_url</span> <span class="o">+</span> <span class="s2">":"</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">conn</span><span class="o">.</span><span class="n">port</span><span class="p">)</span> <span class="o">+</span> <span class="s2">"/"</span>
@@ -230,6 +238,11 @@
<span class="n">url</span><span class="p">,</span>
<span class="n">params</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
+ <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">==</span> <span class="s1">'HEAD'</span><span class="p">:</span>
+ <span class="c1"># HEAD doesn't use params</span>
+ <span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
+ <span class="n">url</span><span class="p">,</span>
+ <span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
<span class="k">else</span><span class="p">:</span>
<span class="c1"># Others use data</span>
<span class="n">req</span> <span class="o">=</span> <span class="n">requests</span><span class="o">.</span><span class="n">Request</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span>
@@ -238,7 +251,7 @@
<span class="n">headers</span><span class="o">=</span><span class="n">headers</span><span class="p">)</span>
<span class="n">prepped_request</span> <span class="o">=</span> <span class="n">session</span><span class="o">.</span><span class="n">prepare_request</span><span class="p">(</span><span class="n">req</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sending '"</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">+</span> <span class="s2">"' to url: "</span> <span class="o">+</span> <span class="n">url</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Sending '</span><span class="si">%s</span><span class="s2">' to url: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">url</span><span class="p">)</span>
<span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">run_and_check</span><span class="p">(</span><span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">)</span></div>
<div class="viewcode-block" id="HttpHook.run_and_check"><a class="viewcode-back" href="../code.html#airflow.hooks.HttpHook.run_and_check">[docs]</a> <span class="k">def</span> <span class="nf">run_and_check</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">session</span><span class="p">,</span> <span class="n">prepped_request</span><span class="p">,</span> <span class="n">extra_options</span><span class="p">):</span>
@@ -263,13 +276,13 @@
<span class="c1"># Tried rewrapping, but not supported. This way, it's possible</span>
<span class="c1"># to get reason and code for failure by checking first 3 chars</span>
<span class="c1"># for the code, or do a split on ':'</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"HTTP error: "</span> <span class="o">+</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
- <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">!=</span> <span class="s1">'GET'</span><span class="p">:</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="s2">"HTTP error: </span><span class="si">%s</span><span class="s2">"</span><span class="p">,</span> <span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
+ <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">'GET'</span><span class="p">,</span> <span class="s1">'HEAD'</span><span class="p">):</span>
<span class="c1"># The sensor uses GET, so this prevents filling up the log</span>
<span class="c1"># with the body every time the GET 'misses'.</span>
<span class="c1"># That's ok to do, because GETs should be repeatable and</span>
<span class="c1"># all data should be visible in the log (no post data)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">error</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="p">)</span>
<span class="k">raise</span> <span class="n">AirflowException</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="p">)</span><span class="o">+</span><span class="s2">":"</span><span class="o">+</span><span class="n">response</span><span class="o">.</span><span class="n">reason</span><span class="p">)</span>
<span class="k">return</span> <span class="n">response</span></div></div>
</pre></div>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/http_operator.html
----------------------------------------------------------------------
diff --git a/_modules/http_operator.html b/_modules/http_operator.html
index a22b710..8a42e96 100644
--- a/_modules/http_operator.html
+++ b/_modules/http_operator.html
@@ -13,6 +13,8 @@
+
+
@@ -80,7 +82,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -176,8 +181,6 @@
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
-<span class="kn">import</span> <span class="nn">logging</span>
-
<span class="kn">from</span> <span class="nn">airflow.exceptions</span> <span class="k">import</span> <span class="n">AirflowException</span>
<span class="kn">from</span> <span class="nn">airflow.hooks.http_hook</span> <span class="k">import</span> <span class="n">HttpHook</span>
<span class="kn">from</span> <span class="nn">airflow.models</span> <span class="k">import</span> <span class="n">BaseOperator</span>
@@ -209,7 +212,7 @@
<span class="sd"> depends on the option that's being modified.</span>
<span class="sd"> """</span>
- <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span><span class="s1">'data'</span><span class="p">,)</span>
+ <span class="n">template_fields</span> <span class="o">=</span> <span class="p">(</span><span class="s1">'endpoint'</span><span class="p">,</span> <span class="s1">'data'</span><span class="p">,)</span>
<span class="n">template_ext</span> <span class="o">=</span> <span class="p">()</span>
<span class="n">ui_color</span> <span class="o">=</span> <span class="s1">'#f4a460'</span>
@@ -227,7 +230,7 @@
<span class="sd"> If xcom_push is True, response of an HTTP request will also</span>
<span class="sd"> be pushed to an XCom.</span>
<span class="sd"> """</span>
- <span class="nb">super</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+ <span class="nb">super</span><span class="p">(</span><span class="n">SimpleHttpOperator</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span> <span class="o">=</span> <span class="n">http_conn_id</span>
<span class="bp">self</span><span class="o">.</span><span class="n">method</span> <span class="o">=</span> <span class="n">method</span>
<span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span> <span class="o">=</span> <span class="n">endpoint</span>
@@ -239,7 +242,9 @@
<span class="k">def</span> <span class="nf">execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
<span class="n">http</span> <span class="o">=</span> <span class="n">HttpHook</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">method</span><span class="p">,</span> <span class="n">http_conn_id</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">http_conn_id</span><span class="p">)</span>
- <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Calling HTTP method"</span><span class="p">)</span>
+
+ <span class="bp">self</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">"Calling HTTP method"</span><span class="p">)</span>
+
<span class="n">response</span> <span class="o">=</span> <span class="n">http</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">endpoint</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">,</span>
<span class="bp">self</span><span class="o">.</span><span class="n">headers</span><span class="p">,</span>
http://git-wip-us.apache.org/repos/asf/incubator-airflow-site/blob/28a3eb60/_modules/index.html
----------------------------------------------------------------------
diff --git a/_modules/index.html b/_modules/index.html
index b1faeb5..6bce09a 100644
--- a/_modules/index.html
+++ b/_modules/index.html
@@ -13,6 +13,8 @@
+
+
@@ -79,7 +81,10 @@
- <ul>
+
+
+
+ <ul>
<li class="toctree-l1"><a class="reference internal" href="../project.html">Project</a></li>
<li class="toctree-l1"><a class="reference internal" href="../license.html">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="../start.html">Quick Start</a></li>
@@ -159,32 +164,72 @@
<div itemprop="articleBody">
<h1>All modules for which code is available</h1>
-<ul><li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li>
+<ul><li><a href="S3_hook.html">S3_hook</a></li>
+<li><a href="airflow/contrib/executors/mesos_executor.html">airflow.contrib.executors.mesos_executor</a></li>
+<li><a href="airflow/contrib/hooks/bigquery_hook.html">airflow.contrib.hooks.bigquery_hook</a></li>
+<li><a href="airflow/contrib/hooks/datastore_hook.html">airflow.contrib.hooks.datastore_hook</a></li>
+<li><a href="airflow/contrib/hooks/emr_hook.html">airflow.contrib.hooks.emr_hook</a></li>
+<li><a href="airflow/contrib/hooks/gcp_dataflow_hook.html">airflow.contrib.hooks.gcp_dataflow_hook</a></li>
+<li><a href="airflow/contrib/hooks/gcp_mlengine_hook.html">airflow.contrib.hooks.gcp_mlengine_hook</a></li>
+<li><a href="airflow/contrib/hooks/gcs_hook.html">airflow.contrib.hooks.gcs_hook</a></li>
+<li><a href="airflow/contrib/hooks/wasb_hook.html">airflow.contrib.hooks.wasb_hook</a></li>
+<li><a href="airflow/contrib/operators/bigquery_check_operator.html">airflow.contrib.operators.bigquery_check_operator</a></li>
+<li><a href="airflow/contrib/operators/bigquery_operator.html">airflow.contrib.operators.bigquery_operator</a></li>
+<li><a href="airflow/contrib/operators/bigquery_to_bigquery.html">airflow.contrib.operators.bigquery_to_bigquery</a></li>
+<li><a href="airflow/contrib/operators/bigquery_to_gcs.html">airflow.contrib.operators.bigquery_to_gcs</a></li>
+<li><a href="airflow/contrib/operators/databricks_operator.html">airflow.contrib.operators.databricks_operator</a></li>
+<li><a href="airflow/contrib/operators/dataflow_operator.html">airflow.contrib.operators.dataflow_operator</a></li>
+<li><a href="airflow/contrib/operators/dataproc_operator.html">airflow.contrib.operators.dataproc_operator</a></li>
+<li><a href="airflow/contrib/operators/ecs_operator.html">airflow.contrib.operators.ecs_operator</a></li>
+<li><a href="airflow/contrib/operators/emr_add_steps_operator.html">airflow.contrib.operators.emr_add_steps_operator</a></li>
+<li><a href="airflow/contrib/operators/emr_create_job_flow_operator.html">airflow.contrib.operators.emr_create_job_flow_operator</a></li>
+<li><a href="airflow/contrib/operators/emr_terminate_job_flow_operator.html">airflow.contrib.operators.emr_terminate_job_flow_operator</a></li>
+<li><a href="airflow/contrib/operators/file_to_wasb.html">airflow.contrib.operators.file_to_wasb</a></li>
+<li><a href="airflow/contrib/operators/gcs_download_operator.html">airflow.contrib.operators.gcs_download_operator</a></li>
+<li><a href="airflow/contrib/operators/gcs_to_bq.html">airflow.contrib.operators.gcs_to_bq</a></li>
<li><a href="airflow/contrib/operators/hipchat_operator.html">airflow.contrib.operators.hipchat_operator</a></li>
+<li><a href="airflow/contrib/operators/mlengine_operator.html">airflow.contrib.operators.mlengine_operator</a></li>
+<li><a href="airflow/contrib/sensors/wasb_sensor.html">airflow.contrib.sensors.wasb_sensor</a></li>
<li><a href="airflow/executors/local_executor.html">airflow.executors.local_executor</a></li>
<li><a href="airflow/executors/sequential_executor.html">airflow.executors.sequential_executor</a></li>
+<li><a href="airflow/hooks/S3_hook.html">airflow.hooks.S3_hook</a></li>
<li><a href="airflow/macros.html">airflow.macros</a></li>
<ul><li><a href="airflow/macros/hive.html">airflow.macros.hive</a></li>
</ul><li><a href="airflow/models.html">airflow.models</a></li>
+<li><a href="airflow/operators/docker_operator.html">airflow.operators.docker_operator</a></li>
+<li><a href="airflow/operators/redshift_to_s3_operator.html">airflow.operators.redshift_to_s3_operator</a></li>
+<li><a href="airflow/operators/s3_file_transform_operator.html">airflow.operators.s3_file_transform_operator</a></li>
+<li><a href="airflow/operators/s3_to_hive_operator.html">airflow.operators.s3_to_hive_operator</a></li>
<li><a href="airflow/operators/sensors.html">airflow.operators.sensors</a></li>
<li><a href="bash_operator.html">bash_operator</a></li>
+<li><a href="bigquery_hook.html">bigquery_hook</a></li>
<li><a href="dagrun_operator.html">dagrun_operator</a></li>
<li><a href="dbapi_hook.html">dbapi_hook</a></li>
+<li><a href="druid_hook.html">druid_hook</a></li>
<li><a href="dummy_operator.html">dummy_operator</a></li>
<li><a href="email_operator.html">email_operator</a></li>
<li><a href="ftp_hook.html">ftp_hook</a></li>
+<li><a href="gcs_hook.html">gcs_hook</a></li>
<li><a href="generic_transfer.html">generic_transfer</a></li>
+<li><a href="hive_hooks.html">hive_hooks</a></li>
+<li><a href="hive_operator.html">hive_operator</a></li>
+<li><a href="hive_to_druid.html">hive_to_druid</a></li>
+<li><a href="hive_to_mysql.html">hive_to_mysql</a></li>
<li><a href="http_hook.html">http_hook</a></li>
<li><a href="http_operator.html">http_operator</a></li>
<li><a href="mysql_hook.html">mysql_hook</a></li>
<li><a href="mysql_operator.html">mysql_operator</a></li>
+<li><a href="mysql_to_hive.html">mysql_to_hive</a></li>
+<li><a href="postgres_hook.html">postgres_hook</a></li>
+<li><a href="postgres_operator.html">postgres_operator</a></li>
<li><a href="presto_check_operator.html">presto_check_operator</a></li>
<li><a href="presto_hook.html">presto_hook</a></li>
<li><a href="python_operator.html">python_operator</a></li>
+<li><a href="qubole_operator.html">qubole_operator</a></li>
+<li><a href="s3_to_hive_operator.html">s3_to_hive_operator</a></li>
<li><a href="sensors.html">sensors</a></li>
+<li><a href="slack_operator.html">slack_operator</a></li>
<li><a href="sqlite_hook.html">sqlite_hook</a></li>
-<li><a href="ssh_execute_operator.html">ssh_execute_operator</a></li>
-<li><a href="ssh_hook.html">ssh_hook</a></li>
</ul>
</div>